This commit is contained in:
Michael Sharp 2021-11-04 21:48:50 -07:00 коммит произвёл GitHub
Родитель cc5e6395e0
Коммит 0577957256
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
332 изменённых файлов: 1265 добавлений и 1181 удалений

Просмотреть файл

@ -306,6 +306,10 @@ dotnet_diagnostic.IDE2004.severity = warning
dotnet_diagnostic.CA1822.severity = suggestion
[test/**/*.cs]
# IDE0044: Don't force readonly for tests
dotnet_diagnostic.IDE0044.severity = none
dotnet_style_readonly_field = false
# MSML_GeneralName: This name should be PascalCased
dotnet_diagnostic.MSML_GeneralName.severity = none
@ -322,4 +326,18 @@ dotnet_diagnostic.MSML_ExtendBaseTestClass.severity = none
# The MSML_RelaxTestNaming suppressor for VSTHRD200 is not active for CodeAnalyzer.Tests, so we disable it altogether.
# VSTHRD200: Use "Async" suffix for async methods
dotnet_diagnostic.VSTHRD200.severity = none
dotnet_diagnostic.VSTHRD200.severity = none
[docs/**/*.cs]
# IDE0073: Dont want license file header in samples
dotnet_diagnostic.IDE0073.severity = none
file_header_template = unset
# IDE0044: Don't force readonly for samples
dotnet_diagnostic.IDE0044.severity = none
dotnet_style_readonly_field = false
[test/Microsoft.ML.TestFrameworkCommon/Utility/*.cs]
# IDE0073: Dont want license file header in code we are using from elsewhere
dotnet_diagnostic.IDE0073.severity = none
file_header_template = unset

Просмотреть файл

@ -17,6 +17,7 @@
<DebugType>portable</DebugType>
<DebugSymbols>true</DebugSymbols>
<LangVersion>latest</LangVersion>
<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
</PropertyGroup>
<PropertyGroup>

Просмотреть файл

@ -1,10 +1,10 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using System.Linq;
using Microsoft.ML.Data;
using System.Text;
using Microsoft.ML.AutoML.Samples.DataStructures;
using Microsoft.ML.Data;
namespace Microsoft.ML.AutoML.Samples
{

Просмотреть файл

@ -2,15 +2,15 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.ML;
using static Microsoft.ML.DataOperationsCatalog;
using System.Linq;
using Microsoft.ML.Data;
using System.IO.Compression;
using System.Threading;
using System.Net;
using Microsoft.ML.Vision;
using static Microsoft.ML.DataOperationsCatalog;
namespace Samples.Dynamic
{

Просмотреть файл

@ -18,7 +18,7 @@ namespace Samples.Dynamic
// AlienBody members to produce a SuperAlienHero entity with a "Name" member of type
// string and a single "Merged" member of type AlienBody, where the merger is just
// the addition of the various members of AlienBody.
static public void Example()
public static void Example()
{
var mlContext = new MLContext();
// Build in-memory data.

Просмотреть файл

@ -7,7 +7,7 @@ namespace Samples.Dynamic
{
class ConvertToGrayScaleInMemory
{
static public void Example()
public static void Example()
{
var mlContext = new MLContext();
// Create an image list.

Просмотреть файл

@ -1,8 +1,8 @@
using System;
using System.Collections.Generic;
using System.IO;
using Microsoft.ML;
using Microsoft.ML.Transforms.TimeSeries;
using System.IO;
namespace Samples.Dynamic
{

Просмотреть файл

@ -20,9 +20,9 @@ namespace Microsoft.Data.Analysis
/// </summary>
public partial class ArrowStringDataFrameColumn : DataFrameColumn, IEnumerable<string>
{
private IList<ReadOnlyDataFrameBuffer<byte>> _dataBuffers;
private IList<ReadOnlyDataFrameBuffer<int>> _offsetsBuffers;
private IList<ReadOnlyDataFrameBuffer<byte>> _nullBitMapBuffers;
private readonly IList<ReadOnlyDataFrameBuffer<byte>> _dataBuffers;
private readonly IList<ReadOnlyDataFrameBuffer<int>> _offsetsBuffers;
private readonly IList<ReadOnlyDataFrameBuffer<byte>> _nullBitMapBuffers;
/// <summary>
/// Constructs an empty <see cref="ArrowStringDataFrameColumn"/> with the given <paramref name="name"/>.
@ -40,7 +40,7 @@ namespace Microsoft.Data.Analysis
/// </summary>
/// <param name="name">The name of the column.</param>
/// <param name="values">The Arrow formatted string values in this column.</param>
/// <param name="offsets">The Arrow formatted offets in this column.</param>
/// <param name="offsets">The Arrow formatted offsets in this column.</param>
/// <param name="nullBits">The Arrow formatted null bits in this column.</param>
/// <param name="length">The length of the column.</param>
/// <param name="nullCount">The number of <see langword="null" /> values in this column.</param>

Просмотреть файл

@ -293,10 +293,10 @@ namespace Microsoft.Data.Analysis
private class WrappedStreamReaderOrStringReader
{
private Stream _stream;
private long _initialPosition;
private Encoding _encoding;
private string _csvString;
private readonly Stream _stream;
private readonly long _initialPosition;
private readonly Encoding _encoding;
private readonly string _csvString;
public WrappedStreamReaderOrStringReader(Stream stream, Encoding encoding)
{

Просмотреть файл

@ -142,7 +142,7 @@ namespace Microsoft.Data.Analysis
return ret;
}
private static bool IsAnyNullValueInColumns (IReadOnlyCollection<DataFrameColumn> columns, long index)
private static bool IsAnyNullValueInColumns(IReadOnlyCollection<DataFrameColumn> columns, long index)
{
foreach (var column in columns)
{
@ -176,19 +176,19 @@ namespace Microsoft.Data.Analysis
throw new ArgumentNullException(nameof(supplemetaryJoinColumnNames));
if (retainedJoinColumnNames.Length != supplemetaryJoinColumnNames.Length)
throw new ArgumentException(Strings.MismatchedArrayLengths, nameof(retainedJoinColumnNames));
throw new ArgumentException(Strings.MismatchedArrayLengths, nameof(retainedJoinColumnNames));
HashSet<long> intersection = calculateIntersection ? new HashSet<long>() : null;
// Get occurrences of values in columns used for join in the retained and supplementary dataframes
Dictionary<long, ICollection<long>> occurrences = null;
Dictionary<long, long> retainedIndicesReverseMapping = null;
HashSet<long> supplementaryJoinColumnsNullIndices = new HashSet<long>();
for (int colNameIndex = 0; colNameIndex < retainedJoinColumnNames.Length; colNameIndex++)
for (int colNameIndex = 0; colNameIndex < retainedJoinColumnNames.Length; colNameIndex++)
{
DataFrameColumn shrinkedRetainedColumn = retainedDataFrame.Columns[retainedJoinColumnNames[colNameIndex]];
@ -211,7 +211,7 @@ namespace Microsoft.Data.Analysis
retainedIndicesReverseMapping = newRetainedIndicesReverseMapping;
shrinkedRetainedColumn = shrinkedRetainedColumn.Clone(new Int64DataFrameColumn("Indices", shrinkedRetainedIndices));
}
DataFrameColumn supplementaryColumn = supplementaryDataFrame.Columns[supplemetaryJoinColumnNames[colNameIndex]];
//Find occurrenses on current step (join column)
@ -222,7 +222,7 @@ namespace Microsoft.Data.Analysis
newOccurrences = newOccurrences.ToDictionary(kvp => retainedIndicesReverseMapping[kvp.Key], kvp => kvp.Value);
supplementaryJoinColumnsNullIndices.UnionWith(supplementaryColumnNullIndices);
// shrink join result on current column by previous join columns (if any)
// (we have to remove occurrences that doesn't exist in previous columns, because JOIN happens only if ALL left and right columns in JOIN are matched)
if (occurrences != null)
@ -242,7 +242,7 @@ namespace Microsoft.Data.Analysis
occurrences = newOccurrences;
}
retainedRowIndices = new Int64DataFrameColumn("RetainedIndices");
supplementaryRowIndices = new Int64DataFrameColumn("SupplementaryIndices");
@ -280,7 +280,7 @@ namespace Microsoft.Data.Analysis
}
}
else
{
{
foreach (long row in supplementaryJoinColumnsNullIndices)
{
retainedRowIndices.Append(i);
@ -288,10 +288,10 @@ namespace Microsoft.Data.Analysis
}
}
}
return intersection;
}
public DataFrame Merge(DataFrame other, string[] leftJoinColumns, string[] rightJoinColumns, string leftSuffix = "_left", string rightSuffix = "_right", JoinAlgorithm joinAlgorithm = JoinAlgorithm.Left)
{
if (other == null)
@ -335,7 +335,7 @@ namespace Microsoft.Data.Analysis
else if (joinAlgorithm == JoinAlgorithm.FullOuter)
{
//In full outer join we would like to retain data from both side, so we do it into 2 steps: one first we do LEFT JOIN and then add lost data from the RIGHT side
//Step 1
//Do LEFT JOIN
isLeftDataFrameRetained = true;
@ -347,7 +347,7 @@ namespace Microsoft.Data.Analysis
var retainedJoinColumns = isLeftDataFrameRetained ? leftJoinColumns : rightJoinColumns;
var intersection = Merge(retainedDataFrame, supplementaryDataFrame, retainedJoinColumns, supplementaryJoinColumns, out retainedRowIndices, out supplementaryRowIndices, calculateIntersection: true);
//Step 2
//Do RIGHT JOIN to retain all data from supplementary DataFrame too (take into account data intersection from the first step to avoid duplicates)
for (long i = 0; i < supplementaryDataFrame.Columns.RowCount; i++)
@ -365,9 +365,9 @@ namespace Microsoft.Data.Analysis
}
else
throw new NotImplementedException(nameof(joinAlgorithm));
DataFrame ret = new DataFrame();
//insert columns from left dataframe (this)
for (int i = 0; i < this.Columns.Count; i++)
{

Просмотреть файл

@ -244,7 +244,7 @@ namespace Microsoft.Data.Analysis
ret.Add(i, otherRowIndices);
}
}
return ret;
}

Просмотреть файл

@ -13,11 +13,11 @@ namespace Microsoft.Data.Analysis
/// </summary>
public class DataFrameColumnCollection : Collection<DataFrameColumn>
{
private Action ColumnsChanged;
private readonly Action ColumnsChanged;
private List<string> _columnNames = new List<string>();
private readonly List<string> _columnNames = new List<string>();
private Dictionary<string, int> _columnNameToIndexDictionary = new Dictionary<string, int>(StringComparer.Ordinal);
private readonly Dictionary<string, int> _columnNameToIndexDictionary = new Dictionary<string, int>(StringComparer.Ordinal);
internal long RowCount { get; set; }

Просмотреть файл

@ -101,9 +101,9 @@ namespace Microsoft.Data.Analysis
#endregion
private int _groupByColumnIndex;
private IDictionary<TKey, ICollection<long>> _keyToRowIndicesMap;
private DataFrame _dataFrame;
private readonly int _groupByColumnIndex;
private readonly IDictionary<TKey, ICollection<long>> _keyToRowIndicesMap;
private readonly DataFrame _dataFrame;
public GroupBy(DataFrame dataFrame, int groupByColumnIndex, IDictionary<TKey, ICollection<long>> keyToRowIndices)
{

Просмотреть файл

@ -20,7 +20,7 @@ namespace Microsoft.Data.Analysis
public partial class PrimitiveDataFrameColumn<T> : DataFrameColumn, IEnumerable<T?>
where T : unmanaged
{
private PrimitiveColumnContainer<T> _columnContainer;
private readonly PrimitiveColumnContainer<T> _columnContainer;
internal PrimitiveColumnContainer<T> ColumnContainer => _columnContainer;

Просмотреть файл

@ -17,7 +17,7 @@ namespace Microsoft.Data.Analysis
/// <remarks> Is NOT Arrow compatible </remarks>
public partial class StringDataFrameColumn : DataFrameColumn, IEnumerable<string>
{
private List<List<string>> _stringBuffers = new List<List<string>>(); // To store more than intMax number of strings
private readonly List<List<string>> _stringBuffers = new List<List<string>>(); // To store more than intMax number of strings
public StringDataFrameColumn(string name, long length = 0) : base(name, length, typeof(string))
{

Просмотреть файл

@ -22,12 +22,12 @@ namespace Microsoft.Data.Analysis
internal class QuoteDelimitedFieldBuilder
{
private StringBuilder _field;
private readonly StringBuilder _field;
private bool _fieldFinished;
private int _index;
private int _delimiterLength;
private Regex _delimiterRegex;
private string _spaceChars;
private readonly Regex _delimiterRegex;
private readonly string _spaceChars;
private bool _malformedLine;
public QuoteDelimitedFieldBuilder(Regex delimiterRegex, string spaceChars)
@ -148,7 +148,7 @@ namespace Microsoft.Data.Analysis
private Regex _delimiterWithEndCharsRegex;
private int[] _whitespaceCodes = new int[] { '\u0009', '\u000B', '\u000C', '\u0020', '\u0085', '\u00A0', '\u1680', '\u2000', '\u2001', '\u2002', '\u2003', '\u2004', '\u2005', '\u2006', '\u2007', '\u2008', '\u2009', '\u200A', '\u200B', '\u2028', '\u2029', '\u3000', '\uFEFF' };
private readonly int[] _whitespaceCodes = new int[] { '\u0009', '\u000B', '\u000C', '\u0020', '\u0085', '\u00A0', '\u1680', '\u2000', '\u2001', '\u2002', '\u2003', '\u2004', '\u2005', '\u2006', '\u2007', '\u2008', '\u2009', '\u200A', '\u200B', '\u2028', '\u2029', '\u3000', '\uFEFF' };
private Regex _beginQuotesRegex;
@ -172,13 +172,13 @@ namespace Microsoft.Data.Analysis
private string _spaceChars;
private int _maxLineSize = 10000000;
private readonly int _maxLineSize = 10000000;
private int _maxBufferSize = 10000000;
private readonly int _maxBufferSize = 10000000;
private bool _leaveOpen;
private readonly bool _leaveOpen;
private char[] newLineChars = Environment.NewLine.ToCharArray();
private readonly char[] _newLineChars = Environment.NewLine.ToCharArray();
public string[] CommentTokens
{
@ -419,7 +419,7 @@ namespace Microsoft.Data.Analysis
}
_lineNumber++;
return line.TrimEnd(newLineChars);
return line.TrimEnd(_newLineChars);
}
public string[] ReadFields()
@ -470,7 +470,7 @@ namespace Microsoft.Data.Analysis
return null;
}
line = line.TrimEnd(newLineChars);
line = line.TrimEnd(_newLineChars);
if (line.Length < numberOfChars)
{
return line;
@ -799,7 +799,7 @@ namespace Microsoft.Data.Analysis
endHelper.BuildField(line, index);
if (endHelper.MalformedLine)
{
_errorLine = line.TrimEnd(newLineChars);
_errorLine = line.TrimEnd(_newLineChars);
_errorLineNumber = currentLineNumber;
throw new Exception(string.Format(Strings.CannotParseWithDelimiters, currentLineNumber));
}
@ -817,13 +817,13 @@ namespace Microsoft.Data.Analysis
string newLine = ReadNextDataLine();
if (newLine == null)
{
_errorLine = line.TrimEnd(newLineChars);
_errorLine = line.TrimEnd(_newLineChars);
_errorLineNumber = currentLineNumber;
throw new Exception(string.Format(Strings.CannotParseWithDelimiters, currentLineNumber));
}
if (line.Length + newLine.Length > _maxLineSize)
{
_errorLine = line.TrimEnd(newLineChars);
_errorLine = line.TrimEnd(_newLineChars);
_errorLineNumber = currentLineNumber;
throw new Exception(string.Format(Strings.LineExceedsMaxLineSize, currentLineNumber));
}
@ -832,7 +832,7 @@ namespace Microsoft.Data.Analysis
endHelper.BuildField(line, endOfLine);
if (endHelper.MalformedLine)
{
_errorLine = line.TrimEnd(newLineChars);
_errorLine = line.TrimEnd(_newLineChars);
_errorLineNumber = currentLineNumber;
throw new Exception(string.Format(Strings.CannotParseWithDelimiters, currentLineNumber));
}
@ -862,7 +862,7 @@ namespace Microsoft.Data.Analysis
index = delimiterMatch.Index + delimiterMatch.Length;
continue;
}
field = line.Substring(index).TrimEnd(newLineChars);
field = line.Substring(index).TrimEnd(_newLineChars);
if (_trimWhiteSpace)
{
field = field.Trim();
@ -881,7 +881,7 @@ namespace Microsoft.Data.Analysis
{
return null;
}
line = line.TrimEnd(newLineChars);
line = line.TrimEnd(_newLineChars);
StringInfo lineInfo = new StringInfo(line);
ValidateFixedWidthLine(lineInfo, _lineNumber - 1);
int index = 0;
@ -897,7 +897,7 @@ namespace Microsoft.Data.Analysis
private string GetFixedWidthField(StringInfo line, int index, int fieldLength)
{
string field = (fieldLength > 0) ? line.SubstringByTextElements(index, fieldLength) : ((index < line.LengthInTextElements) ? line.SubstringByTextElements(index).TrimEnd(newLineChars) : string.Empty);
string field = (fieldLength > 0) ? line.SubstringByTextElements(index, fieldLength) : ((index < line.LengthInTextElements) ? line.SubstringByTextElements(index).TrimEnd(_newLineChars) : string.Empty);
if (_trimWhiteSpace)
{
return field.Trim();
@ -1062,7 +1062,7 @@ namespace Microsoft.Data.Analysis
{
throw new Exception(Strings.EmptyDelimiters);
}
if (delimiter.IndexOfAny(newLineChars) > -1)
if (delimiter.IndexOfAny(_newLineChars) > -1)
{
throw new Exception(Strings.DelimiterCannotBeNewlineChar);
}

Просмотреть файл

@ -1,3 +1,7 @@
using System.Runtime.CompilerServices;
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("Microsoft.Extensions.ML.Tests, PublicKey=00240000048000009400000006020000002400005253413100040000010001004b86c4cb78549b34bab61a3b1800e23bfeb5b3ec390074041536a7e3cbd97f5f04cf0f857155a8928eaa29ebfd11cfbbad3ba70efea7bda3226c6a8d370a4cd303f714486b6ebc225985a638471e6ef571cc92a4613c00b8fa65d61ccee0cbe5f36330c9a01f4183559f1bef24cc2917c6d913e3a541333a1d05d9bed22b38cb")]

Просмотреть файл

@ -1,4 +1,4 @@
// Foundation under one or more agreements.
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

Просмотреть файл

@ -1,4 +1,4 @@
// Foundation under one or more agreements.
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

Просмотреть файл

@ -1,4 +1,4 @@
// Foundation under one or more agreements.
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

Просмотреть файл

@ -1,4 +1,4 @@
// Foundation under one or more agreements.
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

Просмотреть файл

@ -1,4 +1,4 @@
// Foundation under one or more agreements.
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

Просмотреть файл

@ -1,4 +1,4 @@
// Foundation under one or more agreements.
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

Просмотреть файл

@ -1,4 +1,4 @@
// Foundation under one or more agreements.
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

Просмотреть файл

@ -1,4 +1,4 @@
// Foundation under one or more agreements.
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

Просмотреть файл

@ -1,4 +1,4 @@
// Foundation under one or more agreements.
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

Просмотреть файл

@ -1,4 +1,4 @@
// Foundation under one or more agreements.
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

Просмотреть файл

@ -1,4 +1,4 @@
// Foundation under one or more agreements.
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

Просмотреть файл

@ -1,4 +1,4 @@
// Foundation under one or more agreements.
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

Просмотреть файл

@ -10,18 +10,18 @@ namespace Microsoft.ML.AutoML
{
public bool IsMaximizing { get; }
private static RegressionMetric[] _minimizingRegressionMetrics = new RegressionMetric[]
private static readonly RegressionMetric[] _minimizingRegressionMetrics = new RegressionMetric[]
{
RegressionMetric.MeanAbsoluteError,
RegressionMetric.MeanSquaredError,
RegressionMetric.RootMeanSquaredError
};
private static BinaryClassificationMetric[] _minimizingBinaryMetrics = new BinaryClassificationMetric[]
private static readonly BinaryClassificationMetric[] _minimizingBinaryMetrics = new BinaryClassificationMetric[]
{
};
private static MulticlassClassificationMetric[] _minimizingMulticlassMetrics = new MulticlassClassificationMetric[]
private static readonly MulticlassClassificationMetric[] _minimizingMulticlassMetrics = new MulticlassClassificationMetric[]
{
MulticlassClassificationMetric.LogLoss,
};

Просмотреть файл

@ -344,8 +344,8 @@ namespace Microsoft.ML.AutoML
// Todo: Remove the reflection below for TreeTreeEnsembleModelParameters methods GetLeaf and GetLeafValue.
// Long-term, replace with tree featurizer once it becomes available
// Tracking issue -- https://github.com/dotnet/machinelearning-automl/issues/342
private static MethodInfo _getLeafMethod = typeof(TreeEnsembleModelParameters).GetMethod("GetLeaf", BindingFlags.NonPublic | BindingFlags.Instance);
private static MethodInfo _getLeafValueMethod = typeof(TreeEnsembleModelParameters).GetMethod("GetLeafValue", BindingFlags.NonPublic | BindingFlags.Instance);
private static readonly MethodInfo _getLeafMethod = typeof(TreeEnsembleModelParameters).GetMethod("GetLeaf", BindingFlags.NonPublic | BindingFlags.Instance);
private static readonly MethodInfo _getLeafValueMethod = typeof(TreeEnsembleModelParameters).GetMethod("GetLeafValue", BindingFlags.NonPublic | BindingFlags.Instance);
private static int GetLeaf(TreeEnsembleModelParameters model, int treeId, VBuffer<Float> features)
{

Просмотреть файл

@ -75,7 +75,7 @@ namespace Microsoft.ML.AutoML
return options;
}
private static string[] _lightGbmBoosterParamNames = new[] { "L2Regularization", "L1Regularization" };
private static readonly string[] _lightGbmBoosterParamNames = new[] { "L2Regularization", "L1Regularization" };
private const string LightGbmBoosterPropName = "Booster";
public static TOptions CreateLightGbmOptions<TOptions, TOutput, TTransformer, TModel>(IEnumerable<SweepableParam> sweepParams, ColumnInformation columnInfo)

Просмотреть файл

@ -1,4 +1,8 @@
using System;
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
namespace Microsoft.ML.AutoML.Utils
{

Просмотреть файл

@ -76,7 +76,7 @@ namespace Microsoft.ML.CodeGenerator.CodeGenerator.CSharp
IncludeRecommenderPackage = false,
StablePackageVersion = _settings.StablePackageVersion,
UnstablePackageVersion = _settings.UnstablePackageVersion,
OnnxRuntimePackageVersion = _settings.OnnxRuntimePacakgeVersion,
OnnxRuntimePackageVersion = _settings.OnnxRuntimePackageVersion,
}.TransformText(),
Name = $"{_settings.OutputName}.ConsoleApp.csproj",
};

Просмотреть файл

@ -92,7 +92,7 @@ namespace Microsoft.ML.CodeGenerator.CodeGenerator.CSharp.AzureCodeGenerator
IncludeRecommenderPackage = false,
StablePackageVersion = _settings.StablePackageVersion,
UnstablePackageVersion = _settings.UnstablePackageVersion,
OnnxRuntimePackageVersion = _settings.OnnxRuntimePacakgeVersion,
OnnxRuntimePackageVersion = _settings.OnnxRuntimePackageVersion,
Target = _settings.Target,
}.TransformText(),
Name = $"{ _settings.OutputName }.Model.csproj",

Просмотреть файл

@ -285,7 +285,7 @@ namespace Microsoft.ML.CodeGenerator.CSharp
IncludeRecommenderPackage = includeRecommenderPackage,
StablePackageVersion = stablePackageVersion,
UnstablePackageVersion = unstablePackageVersion,
OnnxRuntimePackageVersion = _settings.OnnxRuntimePacakgeVersion,
OnnxRuntimePackageVersion = _settings.OnnxRuntimePackageVersion,
Target = target,
};
@ -336,7 +336,7 @@ namespace Microsoft.ML.CodeGenerator.CSharp
IncludeRecommenderPackage = includeRecommenderPackage,
StablePackageVersion = stablePackageVersion,
UnstablePackageVersion = unstablePackageVersion,
OnnxRuntimePackageVersion = _settings.OnnxRuntimePacakgeVersion,
OnnxRuntimePackageVersion = _settings.OnnxRuntimePackageVersion,
Target = target,
};
return predictProjectFileContent.TransformText();

Просмотреть файл

@ -54,7 +54,7 @@ namespace Microsoft.ML.CodeGenerator.CSharp
public string UnstablePackageVersion { get; set; }
public string OnnxRuntimePacakgeVersion { get; set; }
public string OnnxRuntimePackageVersion { get; set; }
public bool IsAzureAttach { get; set; }

Просмотреть файл

@ -538,7 +538,7 @@ namespace Microsoft.ML.CodeGenerator.CSharp
internal class OneVersusAll : TrainerGeneratorBase
{
private PipelineNode _node;
private readonly PipelineNode _node;
private string[] _binaryTrainerUsings;
//ClassName of the trainer

Просмотреть файл

@ -10,7 +10,7 @@ namespace Microsoft.ML.CommandLine
[BestFriend]
internal sealed class CmdLexer
{
private CharCursor _curs;
private readonly CharCursor _curs;
// Whether \ has a special significance.
private readonly bool _escapes;
@ -307,7 +307,7 @@ namespace Microsoft.ML.CommandLine
internal sealed class CmdQuoter
{
private readonly string _str;
private StringBuilder _sb;
private readonly StringBuilder _sb;
private int _ich;
private CmdQuoter(string str, StringBuilder sb)

Просмотреть файл

@ -79,7 +79,7 @@ namespace Microsoft.ML
}
// These are the open entries that may contain streams into our DirTemp.
private List<Entry> _open;
private readonly List<Entry> _open;
private bool _disposed;
@ -435,11 +435,11 @@ namespace Microsoft.ML
[BestFriend]
internal sealed class RepositoryReader : Repository
{
private ZipArchive _archive;
private readonly ZipArchive _archive;
// Maps from a normalized path to the entry in the _archive. This is needed since
// a zip might use / or \ for directory separation.
private Dictionary<string, ZipArchiveEntry> _entries;
private readonly Dictionary<string, ZipArchiveEntry> _entries;
public static RepositoryReader Open(Stream stream, IExceptionContext ectx = null, bool useFileSystem = true)
{

Просмотреть файл

@ -352,7 +352,7 @@ namespace Microsoft.ML.Runtime
private readonly MessageSensitivity _sensitivityFlags;
// This object is used to write to the test log along with the console if the host process is a test environment
private TextWriter _testWriter;
private readonly TextWriter _testWriter;
/// <summary>
/// Create an ML.NET <see cref="IHostEnvironment"/> for local execution, with console feedback.

Просмотреть файл

@ -13,7 +13,7 @@ namespace Microsoft.ML.Internal.Utilities
{
private Single[] _valuesSng; // distinct values
private Double[] _valuesDbl; // distinct values
private List<int> _counts; // counts for each value
private readonly List<int> _counts; // counts for each value
private int[] _path; // current set of pegs
protected int CountBins { get; private set; } // number of bins
@ -314,8 +314,8 @@ namespace Microsoft.ML.Internal.Utilities
}
}
private Heap<Segment> _segmentHeap; // heap used for dropping initial peg placement
private HeapNode.Heap<Peg> _pegHeap; // heap used for selecting the largest energy decrease
private readonly Heap<Segment> _segmentHeap; // heap used for dropping initial peg placement
private readonly HeapNode.Heap<Peg> _pegHeap; // heap used for selecting the largest energy decrease
private int[] _accum; // integral of counts
private int[] _path; // current set of pegs
private float _meanBinSize;

Просмотреть файл

@ -732,7 +732,7 @@ LAfterDigits:
// Map from base-10 exponent to 64-bit mantissa.
// The approximation for 10^n is _mpe10man[n-1] * 2^(_mpe10e2[n-1]-64).
private static ulong[] _mpe10Man = new ulong[] {
private static readonly ulong[] _mpe10Man = new ulong[] {
0xA000000000000000UL, 0xC800000000000000UL, 0xFA00000000000000UL, 0x9C40000000000000UL, 0xC350000000000000UL, /*005*/
0xF424000000000000UL, 0x9896800000000000UL, 0xBEBC200000000000UL, 0xEE6B280000000000UL, 0x9502F90000000000UL, /*010*/
0xBA43B74000000000UL, 0xE8D4A51000000000UL, 0x9184E72A00000000UL, 0xB5E620F480000000UL, 0xE35FA931A0000000UL, /*015*/
@ -801,7 +801,7 @@ LAfterDigits:
// Map from negative base-10 exponent to 64-bit mantissa. Note that the top bit of these is set.
// The approximation for 10^-n is _mpne10man[n-1] * 2^(-_mpne10ne2[n-1]-64).
private static ulong[] _mpne10Man = new ulong[] {
private static readonly ulong[] _mpne10Man = new ulong[] {
0xCCCCCCCCCCCCCCCDUL, 0xA3D70A3D70A3D70AUL, 0x83126E978D4FDF3BUL, 0xD1B71758E219652CUL, 0xA7C5AC471B478423UL, /*005*/
0x8637BD05AF6C69B6UL, 0xD6BF94D5E57A42BCUL, 0xABCC77118461CEFDUL, 0x89705F4136B4A597UL, 0xDBE6FECEBDEDD5BFUL, /*010*/
0xAFEBFF0BCB24AAFFUL, 0x8CBCCC096F5088CCUL, 0xE12E13424BB40E13UL, 0xB424DC35095CD80FUL, 0x901D7CF73AB0ACD9UL, /*015*/
@ -878,7 +878,7 @@ LAfterDigits:
// Map from base-10 exponent to base-2 exponent.
// The approximation for 10^n is _mpe10man[n-1] * 2^(_mpe10e2[n-1]-64).
private static short[] _mpe10e2 = new short[] {
private static readonly short[] _mpe10e2 = new short[] {
4, 7, 10, 14, 17, 20, 24, 27, 30, 34, 37, 40, 44, 47, 50, 54, 57, 60, 64, 67, /*020*/
70, 74, 77, 80, 84, 87, 90, 94, 97, 100, 103, 107, 110, 113, 117, 120, 123, 127, 130, 133, /*040*/
137, 140, 143, 147, 150, 153, 157, 160, 163, 167, 170, 173, 177, 180, 183, 187, 190, 193, 196, 200, /*060*/

Просмотреть файл

@ -42,7 +42,7 @@ namespace Microsoft.ML.Internal.Utilities
return result;
}
private static double _vScale = 2 * Math.Sqrt(2 / Math.E);
private static readonly double _vScale = 2 * Math.Sqrt(2 / Math.E);
/// <summary>
/// Returns a number sampled from a zero-mean, unit variance Gaussian
@ -242,7 +242,7 @@ namespace Microsoft.ML.Internal.Utilities
// n*p at which we switch algorithms
private const int NPThresh = 10;
private static double[] _fctab = new double[] {
private static readonly double[] _fctab = new double[] {
0.08106146679532726, 0.04134069595540929, 0.02767792568499834,
0.02079067210376509, 0.01664469118982119, 0.01387612882307075,
0.01189670994589177, 0.01041126526197209, 0.009255462182712733,

Просмотреть файл

@ -28,7 +28,7 @@ namespace Microsoft.ML.Internal.Utilities
private string _line;
private int _lineCur;
private byte[] _buff;
private readonly byte[] _buff;
private int _buffCur;
private int _buffLim;
private bool _eof;

Просмотреть файл

@ -28,7 +28,7 @@ namespace Microsoft.ML.Internal.CpuMath
private readonly int _cbAlign; // The alignment in bytes, a power of two, divisible by sizeof(Float).
private int _base; // Where the values start in Items (changes to ensure alignment).
private object _lock; // Used to make sure only one thread can re-align the values.
private readonly object _lock; // Used to make sure only one thread can re-align the values.
/// <summary>
/// Allocate an aligned vector with the given alignment (in bytes).

Просмотреть файл

@ -23,7 +23,7 @@ namespace Microsoft.ML.Data
/// For example, UInt32 and Key can be mapped to <see langword="uint"/>. This class enforces one-to-one mapping for all
/// user-registered types.
/// </summary>
private static HashSet<Type> _bannedRawTypes = new HashSet<Type>()
private static readonly HashSet<Type> _bannedRawTypes = new HashSet<Type>()
{
typeof(Boolean), typeof(SByte), typeof(Byte),
typeof(Int16), typeof(UInt16), typeof(Int32), typeof(UInt32),
@ -36,17 +36,17 @@ namespace Microsoft.ML.Data
/// <summary>
/// Mapping from a <see cref="Type"/> plus its <see cref="Attribute"/>s to a <see cref="DataViewType"/>.
/// </summary>
private static Dictionary<TypeWithAttributes, DataViewType> _rawTypeToDataViewTypeMap = new Dictionary<TypeWithAttributes, DataViewType>();
private static readonly Dictionary<TypeWithAttributes, DataViewType> _rawTypeToDataViewTypeMap = new Dictionary<TypeWithAttributes, DataViewType>();
/// <summary>
/// Mapping from a <see cref="DataViewType"/> to a <see cref="Type"/> plus its <see cref="Attribute"/>s.
/// </summary>
private static Dictionary<DataViewType, TypeWithAttributes> _dataViewTypeToRawTypeMap = new Dictionary<DataViewType, TypeWithAttributes>();
private static readonly Dictionary<DataViewType, TypeWithAttributes> _dataViewTypeToRawTypeMap = new Dictionary<DataViewType, TypeWithAttributes>();
/// <summary>
/// The lock that one should acquire if the state of <see cref="DataViewTypeManager"/> will be accessed or modified.
/// </summary>
private static object _lock = new object();
private static readonly object _lock = new object();
/// <summary>
/// Returns the <see cref="DataViewType"/> registered for <paramref name="type"/> and its <paramref name="typeAttributes"/>.
@ -228,7 +228,7 @@ namespace Microsoft.ML.Data
/// a key when using <see cref="TypeWithAttributes"/> as the key type in <see cref="Dictionary{TKey, TValue}"/>. Note that the
/// uniqueness is determined by <see cref="Equals(object)"/> and <see cref="GetHashCode"/> below.
/// </summary>
private DataViewTypeAttribute _associatedAttribute;
private readonly DataViewTypeAttribute _associatedAttribute;
public TypeWithAttributes(Type type, DataViewTypeAttribute attribute)
{

Просмотреть файл

@ -107,8 +107,8 @@ namespace Microsoft.ML.Data.IO
private sealed class WritePipe<T> : WritePipe
{
private ValueGetter<T> _getter;
private IValueCodec<T> _codec;
private readonly ValueGetter<T> _getter;
private readonly IValueCodec<T> _codec;
private IValueWriter<T> _writer;
private MemoryStream _currentStream;
private T _value;

Просмотреть файл

@ -313,7 +313,7 @@ namespace Microsoft.ML.Data.IO
private sealed class Writer : ValueWriterBase<ReadOnlyMemory<char>>
{
private StringBuilder _builder;
private List<int> _boundaries;
private readonly List<int> _boundaries;
public Writer(TextCodec codec, Stream stream)
: base(codec.Factory, stream)
@ -353,7 +353,7 @@ namespace Microsoft.ML.Data.IO
private readonly int _entries;
private readonly int[] _boundaries;
private int _index;
private string _text;
private readonly string _text;
public Reader(TextCodec codec, Stream stream, int items)
: base(codec.Factory, stream)

Просмотреть файл

@ -25,7 +25,7 @@ namespace Microsoft.ML.Internal.Internallearn
internal static class UnsafeTypeOpsFactory
{
private static Dictionary<Type, object> _type2ops;
private static readonly Dictionary<Type, object> _type2ops;
static UnsafeTypeOpsFactory()
{

Просмотреть файл

@ -249,7 +249,7 @@ namespace Microsoft.ML.Data
private readonly TryParseMapper<TResult> _conv;
// Has length Rows.Count, so indexed by irow.
private TResult[] _values;
private readonly TResult[] _values;
public override bool HasNA { get; }
@ -436,7 +436,7 @@ namespace Microsoft.ML.Data
}
// Has length Rows.Count, so indexed by irow.
private VectorValue[] _values;
private readonly VectorValue[] _values;
public VectorPipe(RowSet rows, PrimitiveDataViewType type, TryParseMapper<TItem> conv)
: base(rows)

Просмотреть файл

@ -221,7 +221,7 @@ namespace Microsoft.ML.Data.IO
{
private readonly ValueGetter<T> _getSrc;
private T _src;
private string _columnName;
private readonly string _columnName;
public ValueWriter(DataViewRowCursor cursor, PrimitiveDataViewType type, int source, char sep)
: base(type, source, sep)
@ -535,8 +535,8 @@ namespace Microsoft.ML.Data.IO
private int _dstPrev;
// Map from column to starting destination index and slot.
private int[] _mpcoldst;
private int[] _mpcolslot;
private readonly int[] _mpcoldst;
private readonly int[] _mpcolslot;
// "slot" is an index into _mpslotdst and _mpslotichLim. _mpslotdst is the sequence of
// destination indices. _mpslotichLim is the sequence of upper bounds on the characters

Просмотреть файл

@ -89,7 +89,7 @@ namespace Microsoft.ML.Data
private sealed class SubsetActive : DataViewRow
{
private readonly DataViewRow _row;
private Func<int, bool> _pred;
private readonly Func<int, bool> _pred;
public SubsetActive(DataViewRow row, Func<int, bool> pred)
{

Просмотреть файл

@ -686,7 +686,7 @@ namespace Microsoft.ML.Data
internal sealed class StreamingDataView<TRow> : DataViewBase<TRow>
where TRow : class
{
private IEnumerable<TRow> _data;
private readonly IEnumerable<TRow> _data;
public StreamingDataView(IHostEnvironment env, IEnumerable<TRow> data, InternalSchemaDefinition schemaDefn)
: base(env, "StreamingDataView", schemaDefn)
@ -936,7 +936,7 @@ namespace Microsoft.ML.Data
if (AnnotationType is VectorDataViewType annotationVectorType)
{
// VBuffer<T> -> VBuffer<T>
// REVIEW: Do we care about accomodating VBuffer<string> -> VBuffer<ReadOnlyMemory<char>>?
// REVIEW: Do we care about accommodating VBuffer<string> -> VBuffer<ReadOnlyMemory<char>>?
Contracts.Assert(typeT.IsGenericType);
Contracts.Check(typeof(TDst).IsGenericType);

Просмотреть файл

@ -58,7 +58,7 @@ namespace Microsoft.ML.Internal.Internallearn
return CmdParser.CombineSettings(settings) + " " + CmdParser.CombineSettings(SplitOnSemis(extraSettings));
}
private static char[] _dontSplitChars = new char[] { ' ', '=', '{', '}', '\t' };
private static readonly char[] _dontSplitChars = new char[] { ' ', '=', '{', '}', '\t' };
// REVIEW: Deprecate this!
public static string[] SplitOnSemis(string[] args)

Просмотреть файл

@ -427,7 +427,7 @@ namespace Microsoft.ML.EntryPoints
public TimeSpan RunTime { get; internal set; }
private static Regex _stageIdRegex = new Regex(@"[a-zA-Z0-9]*", RegexOptions.Compiled);
private static readonly Regex _stageIdRegex = new Regex(@"[a-zA-Z0-9]*", RegexOptions.Compiled);
private string _stageId;
/// <summary>
/// An alphanumeric string indicating the stage of a node.
@ -1060,7 +1060,7 @@ namespace Microsoft.ML.EntryPoints
// An EntryPoint variable can be followed with an array or dictionary specifier, which begins
// with '[', contains either an integer or alphanumeric string, optionally wrapped in single-quotes,
// followed with ']'.
private static Regex _variableRegex = new Regex(
private static readonly Regex _variableRegex = new Regex(
@"\$(?<Name>[a-zA-Z_][a-zA-Z0-9_]*)(\[(((?<NumericAccessor>[0-9]*))|(\'?(?<StringAccessor>[a-zA-Z0-9_]*)\'?))\])?",
RegexOptions.Compiled);
@ -1193,7 +1193,7 @@ namespace Microsoft.ML.EntryPoints
/// <summary>
/// Represents the l-value assignable destination of a <see cref="VariableBinding"/>.
/// Subclasses exist to express the needed bindinds for subslots
/// Subclasses exist to express the needed bindings for subslots
/// of a yet-to-be-constructed array or dictionary EntryPoint input parameter
/// (for example, "myVar": ["$var1", "$var2"] would yield two <see cref="ArrayIndexParameterBinding"/>: (myVar, 0), (myVar, 1))
/// </summary>

Просмотреть файл

@ -210,7 +210,7 @@ namespace Microsoft.ML.Data
private readonly Counters _counters;
private readonly Counters _weightedCounters;
private VBuffer<ReadOnlyMemory<char>> _slotNames;
private readonly VBuffer<ReadOnlyMemory<char>> _slotNames;
public override CountersBase UnweightedCounters { get { return _counters; } }

Просмотреть файл

@ -1069,8 +1069,8 @@ namespace Microsoft.ML.Calibrators
{
private readonly IHost _host;
private List<float> _cMargins;
private List<float> _ncMargins;
private readonly List<float> _cMargins;
private readonly List<float> _ncMargins;
public int NumBins;
public float BinSize;
@ -1846,7 +1846,7 @@ namespace Microsoft.ML.Calibrators
[BestFriend]
internal sealed class PavCalibratorTrainer : CalibratorTrainerBase
{
// a piece of the piecwise function
// a piece of the piecewise function
private readonly struct Piece
{
public readonly float MinX; // end of interval.

Просмотреть файл

@ -69,7 +69,7 @@ namespace Microsoft.ML.Calibrators
LabelColumn = TrainerUtils.MakeBoolScalarLabel(labelColumn);
else
env.CheckParam(!calibratorTrainer.NeedsTraining, nameof(labelColumn), "For trained calibrators, " + nameof(labelColumn) + " must be specified.");
ScoreColumn = TrainerUtils.MakeR4ScalarColumn(scoreColumn); // Do we fanthom this being named anything else (renaming column)? Complete metadata?
ScoreColumn = TrainerUtils.MakeR4ScalarColumn(scoreColumn); // Do we fathom this being named anything else (renaming column)? Complete metadata?
if (weightColumn != null)
WeightColumn = TrainerUtils.MakeR4ScalarWeightColumn(weightColumn);
@ -239,10 +239,10 @@ namespace Microsoft.ML.Calibrators
private sealed class Mapper<TCalibrator> : MapperBase, ISaveAsOnnx
where TCalibrator : class, ICalibrator
{
private TCalibrator _calibrator;
private readonly TCalibrator _calibrator;
private readonly int _scoreColIndex;
private CalibratorTransformer<TCalibrator> _parent;
private string _scoreColumnName;
private readonly CalibratorTransformer<TCalibrator> _parent;
private readonly string _scoreColumnName;
bool ICanSaveOnnx.CanSaveOnnx(OnnxContext ctx) => _calibrator is ICanSaveOnnx onnxMapper ? onnxMapper.CanSaveOnnx(ctx) : false;
@ -294,7 +294,7 @@ namespace Microsoft.ML.Calibrators
builder.Add(setIdCol.Name, setIdType, annotation.GetGetter<uint>(setIdCol));
// Now, this next one I'm a little less sure about. It is entirely reasonable for someone to, say,
// try to calibrate the result of a regression or ranker training, or something else. But should we
// just pass through this class just like that? Having throught through the alternatives I view this
// just pass through this class just like that? Having thought through the alternatives I view this
// as the least harmful thing we could be doing, but it is something to consider I may be wrong
// about if it proves that it ever causes problems to, say, have something identified as a probability
// column but be marked as being a regression task, or what have you.
@ -415,7 +415,7 @@ namespace Microsoft.ML.Calibrators
}
/// <summary>
/// The naive binning-based calbirator estimator.
/// The naive binning-based calibrator estimator.
/// </summary>
/// <remarks>
/// It divides the range of the outputs into equally sized bins. In each bin,

Просмотреть файл

@ -299,7 +299,7 @@ namespace Microsoft.ML.Data
private readonly BindableMapper _parent;
private readonly DataViewSchema _outputSchema;
private readonly DataViewSchema _outputGenericSchema;
private VBuffer<ReadOnlyMemory<char>> _slotNames;
private readonly VBuffer<ReadOnlyMemory<char>> _slotNames;
public RoleMappedSchema InputRoleMappedSchema { get; }

Просмотреть файл

@ -90,7 +90,7 @@ namespace Microsoft.ML.Transforms
/// <param name="keepHidden">If true will keep hidden columns and false will remove hidden columns. The argument is
/// ignored if the Estimator is in "drop mode".</param>
/// <param name="ignoreMissing">If false will check for any columns given in <paramref name="keepColumns"/>
/// or <paramref name="dropColumns"/> that are missing from the input. If a missing colums exists a
/// or <paramref name="dropColumns"/> that are missing from the input. If a missing column exists a
/// SchemaMistmatch exception is thrown. If true, the check is not made.</param>
internal ColumnSelectingEstimator(IHostEnvironment env, string[] keepColumns,
string[] dropColumns, bool keepHidden = Defaults.KeepHidden,
@ -160,7 +160,7 @@ namespace Microsoft.ML.Transforms
internal const string ChooseLoaderSignatureOld = "ChooseColumnsFunction";
private readonly IHost _host;
private string[] _selectedColumns;
private readonly string[] _selectedColumns;
bool ITransformer.IsRowToRowMapper => true;
@ -697,7 +697,7 @@ namespace Microsoft.ML.Transforms
var inputCols = ((IRowToRowMapper)this).GetDependencies(columnsNeeded);
var inputs = Source.GetRowCursorSet(inputCols, n, rand);
// Build out the acitve state for the output
// Build out the active state for the output
var active = Utils.BuildArray(_mapper.OutputSchema.Count, columnsNeeded);
_host.AssertNonEmpty(inputs);

Просмотреть файл

@ -1830,7 +1830,7 @@ namespace Microsoft.ML.Transforms
{
private readonly bool _fix;
private readonly int _numBins;
private List<TFloat> _values;
private readonly List<TFloat> _values;
private BinOneColumnFunctionBuilder(IHost host, long lim, bool fix, int numBins, ValueGetter<TFloat> getSrc)
: base(host, lim, getSrc)
@ -1874,7 +1874,7 @@ namespace Microsoft.ML.Transforms
{
private readonly bool _fix;
private readonly int _numBins;
private List<TFloat>[] _values;
private readonly List<TFloat>[] _values;
private BinVecColumnFunctionBuilder(IHost host, int cv, long lim, bool fix, int numBins,
ValueGetter<VBuffer<TFloat>> getSrc)
@ -2097,7 +2097,7 @@ namespace Microsoft.ML.Transforms
// The difference is how the min/max are used.
_minMaxAggregator = new MinMaxDblAggregator(vectorSize);
// If we aren't centering data dont need the median.
// If we aren't centering data don't need the median.
_medianAggregators = new MedianDblAggregator[vectorSize];
for (int i = 0; i < vectorSize; i++)

Просмотреть файл

@ -1993,7 +1993,7 @@ namespace Microsoft.ML.Transforms
{
private readonly bool _fix;
private readonly int _numBins;
private List<TFloat> _values;
private readonly List<TFloat> _values;
private BinOneColumnFunctionBuilder(IHost host, long lim, bool fix, int numBins, ValueGetter<TFloat> getSrc)
: base(host, lim, getSrc)
@ -2037,7 +2037,7 @@ namespace Microsoft.ML.Transforms
{
private readonly bool _fix;
private readonly int _numBins;
private List<TFloat>[] _values;
private readonly List<TFloat>[] _values;
private BinVecColumnFunctionBuilder(IHost host, int cv, long lim, bool fix, int numBins,
ValueGetter<VBuffer<TFloat>> getSrc)

Просмотреть файл

@ -714,7 +714,7 @@ namespace Microsoft.ML.Transforms
private sealed class Mapper : OneToOneMapperBase, ISaveAsOnnx, ISaveAsPfa
{
private NormalizingTransformer _parent;
private readonly NormalizingTransformer _parent;
public bool CanSaveOnnx(OnnxContext ctx) => true;
public bool CanSavePfa => true;

Просмотреть файл

@ -186,7 +186,7 @@ namespace Microsoft.ML.Transforms
internal class DataViewHelper
{
/// <summary>
/// Helper function to retrieve the Primitie type given a Type
/// Helper function to retrieve the Primitive type given a Type
/// </summary>
internal static PrimitiveDataViewType GetPrimitiveType(Type rawType, out bool isVectorType)
{
@ -359,7 +359,7 @@ namespace Microsoft.ML.Transforms
private const string DefaultMapName = "DefaultMap.idv";
internal static string DefaultKeyColumnName = "Key";
internal static string DefaultValueColumnName = "Value";
private ValueMap _valueMap;
private readonly ValueMap _valueMap;
private readonly byte[] _dataView;
internal DataViewType ValueColumnType => _valueMap.ValueColumn.Type;
@ -834,7 +834,7 @@ namespace Microsoft.ML.Transforms
private static readonly FuncStaticMethodInfo1<TValue, TValue> _getValueMethodInfo
= new FuncStaticMethodInfo1<TValue, TValue>(GetValue<int>);
private Dictionary<TKey, TValue> _mapping;
private readonly Dictionary<TKey, TValue> _mapping;
private TValue _missingValue;
private Dictionary<TKey, TValue> CreateDictionary()

Просмотреть файл

@ -46,7 +46,7 @@ namespace Microsoft.ML.Trainers.Ensemble
public IBinaryOutputCombiner CreateComponent(IHostEnvironment env) => new WeightedAverage(env, this);
}
private WeightageKind _weightageKind;
private readonly WeightageKind _weightageKind;
public string WeightageMetricName { get { return _weightageKind.ToString(); } }

Просмотреть файл

@ -19,7 +19,7 @@ namespace Microsoft.ML.Trainers.Ensemble
}
private readonly IComponentFactory<IDiversityMeasure<TOutput>> _diversityMetricType;
private ConcurrentDictionary<FeatureSubsetModel<TOutput>, TOutput[]> _predictions;
private readonly ConcurrentDictionary<FeatureSubsetModel<TOutput>, TOutput[]> _predictions;
private protected BaseDiverseSelector(IHostEnvironment env, DiverseSelectorArguments args, string name,
IComponentFactory<IDiversityMeasure<TOutput>> diversityMetricType)

Просмотреть файл

@ -420,12 +420,12 @@ namespace Microsoft.ML.Trainers.FastTree
/// </summary>
public sealed class DatasetSkeleton
{
private short[] _ratings;
private readonly short[] _ratings;
public readonly int[] Boundaries;
public readonly ulong[] QueryIds;
public readonly ulong[] DocIds;
public double[][] MaxDcg;
private int[] _docToQuery;
private readonly int[] _docToQuery;
public Dictionary<string, DatasetSkeletonQueryDocData> AuxiliaryData { get; set; }

Просмотреть файл

@ -205,7 +205,7 @@ namespace Microsoft.ML.Trainers.FastTree
{
private const int _bits = 10;
private const int _mask = (1 << _bits) - 1;
private uint[] _data;
private readonly uint[] _data;
public override IntArrayBits BitsPerItem { get { return IntArrayBits.Bits10; } }
@ -381,7 +381,7 @@ namespace Microsoft.ML.Trainers.FastTree
/// 0-bit array only represents the value -1</remarks>
internal sealed class Dense8BitIntArray : DenseDataCallbackIntArray
{
private byte[] _data;
private readonly byte[] _data;
public override IntArrayBits BitsPerItem { get { return IntArrayBits.Bits8; } }
@ -468,7 +468,7 @@ namespace Microsoft.ML.Trainers.FastTree
/// <summary>
/// For a given byte, the high 4 bits is the first value, the low 4 bits is the next value.
/// </summary>
private byte[] _data;
private readonly byte[] _data;
public override IntArrayBits BitsPerItem { get { return IntArrayBits.Bits4; } }
@ -588,7 +588,7 @@ namespace Microsoft.ML.Trainers.FastTree
/// </summary>
internal sealed class Dense16BitIntArray : DenseDataCallbackIntArray
{
private ushort[] _data;
private readonly ushort[] _data;
public override IntArrayBits BitsPerItem { get { return IntArrayBits.Bits16; } }
@ -677,7 +677,7 @@ namespace Microsoft.ML.Trainers.FastTree
/// </summary>
internal sealed class Dense32BitIntArray : DenseDataCallbackIntArray
{
private int[] _data;
private readonly int[] _data;
public override IntArrayBits BitsPerItem { get { return IntArrayBits.Bits32; } }

Просмотреть файл

@ -19,7 +19,7 @@ namespace Microsoft.ML.Trainers.FastTree
{
private readonly DenseIntArray _values;
private readonly int _length;
private byte[] _deltas;
private readonly byte[] _deltas;
private readonly int _deltasActualLength;
public DeltaRepeatIntArray(int length, IntArrayBits bitsPerItem, IEnumerable<int> values)

Просмотреть файл

@ -2616,7 +2616,7 @@ namespace Microsoft.ML.Trainers.FastTree
#if DEBUG
// Holds for each feature the row index that it was previously accessed on.
// Purely for validation purposes.
private int[] _lastRow;
private readonly int[] _lastRow;
#endif
/// <summary>

Просмотреть файл

@ -140,7 +140,7 @@ namespace Microsoft.ML.Trainers.FastTree
internal const string ShortName = "ftc";
private bool[] _trainSetLabels;
private double _sigmoidParameter;
private readonly double _sigmoidParameter;
/// <summary>
/// Initializes a new instance of <see cref="FastTreeBinaryTrainer"/>
@ -325,7 +325,7 @@ namespace Microsoft.ML.Trainers.FastTree
private readonly bool _unbalancedSets; //Should we use balanced or unbalanced loss function
private readonly long _npos;
private readonly long _nneg;
private IParallelTraining _parallelTraining;
private readonly IParallelTraining _parallelTraining;
private readonly double _sigmoidParameter; // Parameter for scaling the loss
public ObjectiveImpl(

Просмотреть файл

@ -88,7 +88,7 @@ namespace Microsoft.ML.Trainers.FastTree
/// <param name="labelColumnName">The name of the label column.</param>
/// <param name="featureColumnName">The name of the feature column.</param>
/// <param name="rowGroupColumnName">The name for the column containing the group ID. </param>
/// <param name="exampleWeightColumnName">The name for the column containing the examle weight.</param>
/// <param name="exampleWeightColumnName">The name for the column containing the example weight.</param>
/// <param name="numberOfLeaves">The max number of leaves in each regression tree.</param>
/// <param name="numberOfTrees">Total number of decision trees to create in the ensemble.</param>
/// <param name="minimumExampleCountPerLeaf">The minimal number of examples allowed in a leaf of a regression tree, out of the subsampled data.</param>
@ -266,7 +266,7 @@ namespace Microsoft.ML.Trainers.FastTree
{
if (!FastTreeTrainerOptions.UseTolerantPruning)
{
//use simple eraly stopping condition
//use simple early stopping condition
PruningTest = new TestHistory(ValidTest, 0);
}
else
@ -363,7 +363,7 @@ namespace Microsoft.ML.Trainers.FastTree
// We only print non-zero train&valid graph if earlyStoppingTruncation!=0
// In case /es is not set, we print 0 for train and valid graph NDCG
// Let's keeping this behaviour for backward compatibility with previous FR version
// Let's keeping this behavior for backward compatibility with previous FR version
// Ideally /graphtv should enforce non-zero /es in the commandline validation
if (_specialTrainSetTest != null)
{
@ -385,7 +385,7 @@ namespace Microsoft.ML.Trainers.FastTree
base.Train(ch);
// Print final last iteration.
// Note that trainNDCG printed in graph will be from copy of a value from previous iteration
// and will diffre slightly from the proper final value computed by FullTest.
// and will differ slightly from the proper final value computed by FullTest.
// We cannot compute the final NDCG here due to the fact we use FastNDCGTestForTrainSet computing NDCG based on label sort saved during gradient computation (and we don;t have gradients for n+1 iteration)
// Keeping it in sync with original FR code
PrintTestGraph(ch);
@ -507,19 +507,19 @@ namespace Microsoft.ML.Trainers.FastTree
private readonly double[] _discount;
private readonly int[] _oneTwoThree;
private int[][] _labelCounts;
private readonly int[][] _labelCounts;
// reusable memory, technical stuff
private int[][] _permutationBuffers;
private DcgPermutationComparer[] _comparers;
private readonly int[][] _permutationBuffers;
private readonly DcgPermutationComparer[] _comparers;
//gains
private double[] _gain;
private readonly double[] _gain;
private double[] _gainLabels;
// parameters
private int _maxDcgTruncationLevel;
private bool _useDcg;
private readonly int _maxDcgTruncationLevel;
private readonly bool _useDcg;
// A lookup table for the sigmoid used in the lambda calculation
// Note: Is built for a specific sigmoid parameter, so assumes this will be constant throughout computation
private double[] _sigmoidTable;
@ -533,9 +533,9 @@ namespace Microsoft.ML.Trainers.FastTree
// Secondary gains, currently not used in any way.
#pragma warning disable 0649
private double _secondaryMetricShare;
private double[] _secondaryInverseMaxDcgt;
private double[] _secondaryGains;
private readonly double _secondaryMetricShare;
private readonly double[] _secondaryInverseMaxDcgt;
private readonly double[] _secondaryGains;
#pragma warning restore 0649
// Baseline risk.
@ -546,19 +546,19 @@ namespace Microsoft.ML.Trainers.FastTree
// 1. preprocessing the scores for continuous cost function
// 2. shifted NDCG
// 3. max DCG per query
private double[] _scoresCopy;
private short[] _labelsCopy;
private short[] _groupIdToTopLabel;
private readonly double[] _scoresCopy;
private readonly short[] _labelsCopy;
private readonly short[] _groupIdToTopLabel;
// parameters
private double _sigmoidParam;
private char _costFunctionParam;
private bool _filterZeroLambdas;
private readonly double _sigmoidParam;
private readonly char _costFunctionParam;
private readonly bool _filterZeroLambdas;
private bool _distanceWeight2;
private bool _normalizeQueryLambdas;
private bool _useShiftedNdcg;
private IParallelTraining _parallelTraining;
private readonly bool _distanceWeight2;
private readonly bool _normalizeQueryLambdas;
private readonly bool _useShiftedNdcg;
private readonly IParallelTraining _parallelTraining;
// Used for training NDCG calculation
// Keeps track of labels of top 3 documents per query

Просмотреть файл

@ -204,7 +204,7 @@ namespace Microsoft.ML.Trainers.FastTree
Host.CheckParam(options.NumberOfThreads == null || options.NumberOfThreads > 0, nameof(options.NumberOfThreads), "Must be positive.");
Host.CheckParam(0 <= options.EntropyCoefficient && options.EntropyCoefficient <= 1, nameof(options.EntropyCoefficient), "Must be in [0, 1].");
Host.CheckParam(0 <= options.GainConfidenceLevel && options.GainConfidenceLevel < 1, nameof(options.GainConfidenceLevel), "Must be in [0, 1).");
Host.CheckParam(0 < options.MaximumBinCountPerFeature, nameof(options.MaximumBinCountPerFeature), "Must be posittive.");
Host.CheckParam(0 < options.MaximumBinCountPerFeature, nameof(options.MaximumBinCountPerFeature), "Must be positive.");
Host.CheckParam(0 < options.NumberOfIterations, nameof(options.NumberOfIterations), "Must be positive.");
Host.CheckParam(0 < options.MinimumExampleCountPerLeaf, nameof(options.MinimumExampleCountPerLeaf), "Must be positive.");
@ -608,7 +608,7 @@ namespace Microsoft.ML.Trainers.FastTree
private class LeafSplitHelper : ILeafSplitStatisticsCalculator
{
private bool _hasWeights;
private readonly bool _hasWeights;
public LeafSplitHelper(bool hasWeights)
{

Просмотреть файл

@ -18,15 +18,15 @@ namespace Microsoft.ML.Trainers.FastTree
private readonly int[] _oneTwoThree;
// reusable memory
private int[][] _permutationBuffers;
private double[][] _scoreBuffers;
private DcgPermutationComparer[] _comparers;
private readonly int[][] _permutationBuffers;
private readonly double[][] _scoreBuffers;
private readonly DcgPermutationComparer[] _comparers;
/// <summary>
/// Contains the instances for a second Level comparer, which gets applied after the initial rank
/// based ordering has happened. The array stores one second level comparer per thread.
/// </summary>
private DescendingStableIdealComparer[] _secondLevelcomparers;
private readonly DescendingStableIdealComparer[] _secondLevelcomparers;
private double _result;

Просмотреть файл

@ -22,7 +22,7 @@ namespace Microsoft.ML.Trainers.FastTree
private readonly int[] _leafCount;
private readonly int[] _documents;
private int[] _tempDocuments;
private int[] _initialDocuments;
private readonly int[] _initialDocuments;
/// <summary>
/// Constructor

Просмотреть файл

@ -33,7 +33,7 @@ namespace Microsoft.ML.Trainers.FastTree
// Total number of passes over data
public int NumberOfPasses;
private int _numFeatures;
private readonly int _numFeatures;
public LassoFit(int numberOfLambdas, int maxAllowedFeaturesAlongPath, int numFeatures)
{

Просмотреть файл

@ -11,14 +11,14 @@ namespace Microsoft.ML.Trainers.FastTree
{
internal class GradientDescent : OptimizationAlgorithm
{
private IGradientAdjuster _gradientWrapper;
private readonly IGradientAdjuster _gradientWrapper;
/// number of trees dropped in this iteration
private int _numberOfDroppedTrees;
// treeScores stores for every tree the predictions it makes on every training example. This is used
// to eliminate the need for computing the scores when we drop trees. However, it causes a horrifying
// memory drain.
private List<double[]> _treeScores;
private readonly List<double[]> _treeScores;
private double[] _droppedScores;
private double[] _scores;

Просмотреть файл

@ -11,8 +11,8 @@ namespace Microsoft.ML.Trainers.FastTree
/// </summary>
internal class RandomForestOptimizer : GradientDescent
{
private IGradientAdjuster _gradientWrapper;
// REVIEW: When the FastTree appliation is decoupled with tree learner and boosting logic, this class should be removed.
private readonly IGradientAdjuster _gradientWrapper;
// REVIEW: When the FastTree application is decoupled with tree learner and boosting logic, this class should be removed.
internal RandomForestOptimizer(InternalTreeEnsemble ensemble, Dataset trainData, double[] initTrainScores, IGradientAdjuster gradientWrapper)
: base(ensemble, trainData, initTrainScores, gradientWrapper)
{

Просмотреть файл

@ -135,13 +135,13 @@ namespace Microsoft.ML.Trainers.FastTree
throw Contracts.ExceptNotSupp("This code should not be reachable");
}
//Computes AGD specific mutiplier. Given that we have tree number t in ensemble (we count trees starting from 0)
//Computes AGD specific multiplier. Given that we have tree number t in ensemble (we count trees starting from 0)
//And we have total k trees in ensemble, what should be the multiplier on the tree when sum the ensemble together based on AGD formula being
//X[k+1] = Y[k] + Tree[k]
//Y[k+1] = X[k+1] + C[k] * (X[k+1] – X[k])
//C[k] = (k-1) / (k+2)
private static Dictionary<int, Dictionary<int, double>> _treeMultiplierMap = new Dictionary<int, Dictionary<int, double>>();
private static readonly Dictionary<int, Dictionary<int, double>> _treeMultiplierMap = new Dictionary<int, Dictionary<int, double>>();
public static double TreeMultiplier(int t, int k)
{
if (_treeMultiplierMap.ContainsKey(t))

Просмотреть файл

@ -16,8 +16,8 @@ namespace Microsoft.ML.Trainers.FastTree
internal sealed class LineSearch : IStepSearch, IFastTrainingScoresUpdate
{
private double _historicStepSize;
private int _numPostbracketSteps;
private double _minStepSize;
private readonly int _numPostbracketSteps;
private readonly double _minStepSize;
public LineSearch(Test lossCalculator, int lossIndex, int numPostbracketSteps, double minStepSize)
: this(lossCalculator, lossIndex) { _numPostbracketSteps = numPostbracketSteps; _minStepSize = minStepSize; }

Просмотреть файл

@ -273,11 +273,11 @@ namespace Microsoft.ML.Trainers.FastTree
private readonly int _windowSize;
private readonly double _tolerance;
// Queue for moving window
private LinkedList<double> _window;
private readonly LinkedList<double> _window;
// This queue keeps track of the iterations which are within tolerance from the best iteration
// The first element of the queue is the early stopping candidate
private LinkedList<ValueIterationPair> _toleratedQueue;
private readonly LinkedList<ValueIterationPair> _toleratedQueue;
// Average validation for the current window
private double _currentWindowSum;

Просмотреть файл

@ -9,8 +9,8 @@ namespace Microsoft.ML.Trainers.FastTree
{
internal class RandomForestLeastSquaresTreeLearner : LeastSquaresRegressionTreeLearner
{
private int _quantileSampleCount;
private bool _quantileEnabled;
private readonly int _quantileSampleCount;
private readonly bool _quantileEnabled;
public RandomForestLeastSquaresTreeLearner(Dataset trainData, int numLeaves, int minDocsInLeaf, Double entropyCoefficient, Double featureFirstUsePenalty,
Double featureReusePenalty, Double softmaxTemperature, int histogramPoolSize, int randomSeed, Double splitFraction, bool allowEmptyTrees,

Просмотреть файл

@ -39,7 +39,7 @@ namespace Microsoft.ML.Trainers.FastTree
public readonly double Bias;
// Multithread task to find best threshold.
private IThreadTask _calculateLeafSplitCandidates;
private readonly IThreadTask _calculateLeafSplitCandidates;
protected SplitInfo[] BestSplitInfoPerLeaf;
protected HashSet<int> CategoricalThresholds;
@ -74,7 +74,7 @@ namespace Microsoft.ML.Trainers.FastTree
// size of reserved memory
private readonly long _sizeOfReservedMemory;
private IParallelTraining _parallelTraining;
private readonly IParallelTraining _parallelTraining;
public int MaxCategoricalGroupsPerNode { get; }
@ -828,7 +828,7 @@ namespace Microsoft.ML.Trainers.FastTree
private double _sumWeights;
private double _sumSquaredTargets;
private int[] _docIndices;
private int[] _docIndicesCopy;
private readonly int[] _docIndicesCopy;
public readonly FloatType[] Targets;
public readonly double[] Weights;
public readonly SplitInfo[] FeatureSplitInfo;

Просмотреть файл

@ -23,12 +23,12 @@ namespace Microsoft.ML.Trainers.FastTree
internal class InternalRegressionTree
{
private double _maxOutput;
private double[] _splitGain;
private double[] _gainPValue;
private readonly double[] _splitGain;
private readonly double[] _gainPValue;
/// <summary>
/// The value of this non-leaf node, prior to split when it was a leaf.
/// </summary>
private double[] _previousLeafValue;
private readonly double[] _previousLeafValue;
// for each non-leaf, we keep the following data
public float[] DefaultValueForMissing;
@ -820,7 +820,7 @@ namespace Microsoft.ML.Trainers.FastTree
private float GetFeatureValue(float x, int node)
{
// Not need to convert missing vaules.
// Not need to convert missing values.
if (DefaultValueForMissing == null)
return x;

Просмотреть файл

@ -190,7 +190,7 @@ namespace Microsoft.ML.Trainers.FastTree
public TreeEnsembleModelParameters ModelParameters;
};
private TreeEnsembleModelParameters _modelParameters;
private readonly TreeEnsembleModelParameters _modelParameters;
internal PretrainedTreeFeaturizationEstimator(IHostEnvironment env, Options options) : base(env, options)
{

Просмотреть файл

@ -84,14 +84,14 @@ namespace Microsoft.ML.Data
/// and the i-th vector element is the prediction value predicted by the i-th tree.
/// If <see cref="_treesColumnName"/> is <see langword="null"/>, this output column may not be generated.
/// </summary>
private string _treesColumnName;
private readonly string _treesColumnName;
/// <summary>
/// The 0-1 encoding of all leaf nodes' IDs. Its type is a vector of <see cref="System.Single"/>. If the given feature
/// vector falls into the first leaf of the first tree, the first element in the 0-1 encoding would be 1.
/// If <see cref="_leavesColumnName"/> is <see langword="null"/>, this output column may not be generated.
/// </summary>
private string _leavesColumnName;
private readonly string _leavesColumnName;
/// <summary>
/// The 0-1 encoding of the paths to the leaves. If the path to the first tree's leaf is node 1 (2nd node in the first tree),
@ -99,7 +99,7 @@ namespace Microsoft.ML.Data
/// would be 1.
/// If <see cref="_pathsColumnName"/> is <see langword="null"/>, this output column may not be generated.
/// </summary>
private string _pathsColumnName;
private readonly string _pathsColumnName;
public BoundMapper(IExceptionContext ectx, TreeEnsembleFeaturizerBindableMapper owner, RoleMappedSchema schema,
string treesColumnName, string leavesColumnName, string pathsColumnName)
@ -230,7 +230,7 @@ namespace Microsoft.ML.Data
private readonly int _numLeaves;
private VBuffer<float> _src;
private ValueGetter<VBuffer<float>> _featureGetter;
private readonly ValueGetter<VBuffer<float>> _featureGetter;
private long _cachedPosition;
private readonly int[] _leafIds;
private readonly List<int>[] _pathIds;

Просмотреть файл

@ -27,7 +27,7 @@ namespace Microsoft.ML.Trainers.FastTree
/// <summary>
/// A dictionary containing all buffer pool types
/// </summary>
private static ConcurrentDictionary<Type, SortedList<int, List<Array>>> _bufferPools = new ConcurrentDictionary<Type, SortedList<int, List<Array>>>();
private static readonly ConcurrentDictionary<Type, SortedList<int, List<Array>>> _bufferPools = new ConcurrentDictionary<Type, SortedList<int, List<Array>>>();
/// <summary>
/// Gets a buffer from the pool with at least the same size as passed as input parameter

Просмотреть файл

@ -12,10 +12,10 @@ namespace Microsoft.ML.Trainers.FastTree
/// </summary>
internal class MappedObjectPool<T> where T : class
{
private T[] _pool;
private int[] _map;
private int[] _inverseMap;
private int[] _lastAccessTime;
private readonly T[] _pool;
private readonly int[] _map;
private readonly int[] _inverseMap;
private readonly int[] _lastAccessTime;
private int _time;
/// <summary>

Просмотреть файл

@ -12,7 +12,7 @@ namespace Microsoft.ML.Trainers.FastTree
{
internal static class ThreadTaskManager
{
private static object _lockObject = new object();
private static readonly object _lockObject = new object();
// REVIEW: Should this bother with number of threads? What should it do?
public static int NumThreads { get; private set; }

Просмотреть файл

@ -1,4 +1,8 @@
using System;
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Runtime.InteropServices;
@ -117,7 +121,7 @@ namespace Microsoft.ML.Featurizers
internal delegate bool DestroyTransformedDataNative(IntPtr output, out IntPtr errorHandle);
internal class TransformedDataSafeHandle : SafeHandleZeroOrMinusOneIsInvalid
{
private DestroyTransformedDataNative _destroySaveDataHandler;
private readonly DestroyTransformedDataNative _destroySaveDataHandler;
public TransformedDataSafeHandle(IntPtr handle, DestroyTransformedDataNative destroyCppTransformerEstimator) : base(true)
{
@ -137,7 +141,7 @@ namespace Microsoft.ML.Featurizers
internal delegate bool DestroyNativeTransformerEstimator(IntPtr estimator, out IntPtr errorHandle);
internal class TransformerEstimatorSafeHandle : SafeHandleZeroOrMinusOneIsInvalid
{
private DestroyNativeTransformerEstimator _destroyNativeTransformerEstimator;
private readonly DestroyNativeTransformerEstimator _destroyNativeTransformerEstimator;
public TransformerEstimatorSafeHandle(IntPtr handle, DestroyNativeTransformerEstimator destroyNativeTransformerEstimator) : base(true)
{
SetHandle(handle);
@ -169,7 +173,7 @@ namespace Microsoft.ML.Featurizers
protected override bool ReleaseHandle()
{
// Not sure what to do with error stuff here. There shoudln't ever be one though.
// Not sure what to do with error stuff here. There shouldn't ever be one though.
return DestroyTransformerSaveDataNative(handle, _dataSize, out _);
}
}

Просмотреть файл

@ -245,8 +245,8 @@ namespace Microsoft.ML.Featurizers
internal const string ShortName = "DateTimeTransform";
internal const string LoadName = "DateTimeTransform";
internal const string LoaderSignature = "DateTimeTransform";
private LongTypedColumn _column;
private DataViewSchema _schema;
private readonly LongTypedColumn _column;
private readonly DataViewSchema _schema;
#endregion
@ -554,7 +554,7 @@ namespace Microsoft.ML.Featurizers
internal abstract void CreateTransformerFromEstimator(DateTimeEstimator.HolidayList country);
private protected abstract unsafe void CreateTransformerFromSavedDataHelper(byte* rawData, IntPtr dataSize);
private protected unsafe abstract bool CreateEstimatorHelper(byte* countryName, byte* dataRootDir, out IntPtr estimator, out IntPtr errorHandle);
private protected abstract unsafe bool CreateEstimatorHelper(byte* countryName, byte* dataRootDir, out IntPtr estimator, out IntPtr errorHandle);
private protected abstract bool CreateTransformerFromEstimatorHelper(TransformerEstimatorSafeHandle estimator, out IntPtr transformer, out IntPtr errorHandle);
private protected abstract bool DestroyEstimatorHelper(IntPtr estimator, out IntPtr errorHandle);
private protected abstract bool DestroyTransformerHelper(IntPtr transformer, out IntPtr errorHandle);
@ -668,7 +668,7 @@ namespace Microsoft.ML.Featurizers
}
[DllImport("Featurizers", EntryPoint = "DateTimeFeaturizer_CreateEstimator"), SuppressUnmanagedCodeSecurity]
private static unsafe extern bool CreateEstimatorNative(byte* countryName, byte* dataRootDir, out IntPtr estimator, out IntPtr errorHandle);
private static extern unsafe bool CreateEstimatorNative(byte* countryName, byte* dataRootDir, out IntPtr estimator, out IntPtr errorHandle);
[DllImport("Featurizers", EntryPoint = "DateTimeFeaturizer_DestroyEstimator"), SuppressUnmanagedCodeSecurity]
private static extern bool DestroyEstimatorNative(IntPtr estimator, out IntPtr errorHandle); // Should ONLY be called by safe handle
@ -683,7 +683,7 @@ namespace Microsoft.ML.Featurizers
}
[DllImport("Featurizers", EntryPoint = "DateTimeFeaturizer_CreateTransformerFromSavedDataWithDataRoot"), SuppressUnmanagedCodeSecurity]
private static unsafe extern bool CreateTransformerFromSavedDataNative(byte* rawData, IntPtr bufferSize, byte* dataRootDir, out IntPtr transformer, out IntPtr errorHandle);
private static extern unsafe bool CreateTransformerFromSavedDataNative(byte* rawData, IntPtr bufferSize, byte* dataRootDir, out IntPtr transformer, out IntPtr errorHandle);
private protected override unsafe void CreateTransformerFromSavedDataHelper(byte* rawData, IntPtr dataSize)
{
byte[] dataRoot;
@ -748,7 +748,7 @@ namespace Microsoft.ML.Featurizers
_transformerHandler.Dispose();
}
private protected unsafe override bool CreateEstimatorHelper(byte* countryName, byte* dataRootDir, out IntPtr estimator, out IntPtr errorHandle) =>
private protected override unsafe bool CreateEstimatorHelper(byte* countryName, byte* dataRootDir, out IntPtr estimator, out IntPtr errorHandle) =>
CreateEstimatorNative(countryName, dataRootDir, out estimator, out errorHandle);
private protected override bool CreateTransformerFromEstimatorHelper(TransformerEstimatorSafeHandle estimator, out IntPtr transformer, out IntPtr errorHandle) =>

Просмотреть файл

@ -1,4 +1,8 @@
using System;
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
@ -84,7 +88,7 @@ namespace Microsoft.ML.Featurizers
/// boolean value representing if the row was created as a result of this operation or not.
///
/// The imputation strategies that are currently supported are ForwardFill, where the last good value is propagated forward, Backfill, where the next good value is propagated backwards,
/// and Median, where the mathmatical median is used to fill in missing values.
/// and Median, where the mathematical median is used to fill in missing values.
///
/// NOTE: It is not recommended to chain this multiple times. If a column is filtered, the default value is placed when a row is imputed, and the
/// default value is not null. Thus any other TimeSeriesImputers will not be able to replace those values anymore causing essentially a very
@ -97,7 +101,7 @@ namespace Microsoft.ML.Featurizers
/// <seealso cref="TimeSeriesImputerExtensionClass.ReplaceMissingTimeSeriesValues(TransformsCatalog, string, string[], string[], FilterMode, ImputationStrategy, bool)"/>
public sealed class TimeSeriesImputerEstimator : IEstimator<TimeSeriesImputerTransformer>
{
private Options _options;
private readonly Options _options;
internal const string IsRowImputedColumnName = "IsRowImputed";
private readonly IHost _host;
@ -158,14 +162,14 @@ namespace Microsoft.ML.Featurizers
/// <summary>
/// Method by which columns are selected for imputing values.
/// NoFilter takes all of the columns so you dont have to specify anything.
/// NoFilter takes all of the columns so you don't have to specify anything.
/// Include only does the specified ImputationStrategy on the columns you specify. The other columns will get a default value.
/// Exclude is the exact opposite of Include, and does the ImputationStrategy on all columns but the ones you specify, which will get the default value.
/// </summary>
public enum FilterMode : byte
{
/// <summary>
/// Takes all of the columns so you dont have to specify anything.
/// Takes all of the columns so you don't have to specify anything.
/// </summary>
NoFilter = 1,
@ -458,7 +462,7 @@ namespace Microsoft.ML.Featurizers
if (!success)
throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle));
// Manually dispose of the IEnumerator since we dont have a using statement;
// Manually dispose of the IEnumerator since we don't have a using statement;
cursor.Dispose();
return new TransformerEstimatorSafeHandle(transformer, DestroyTransformerNative);
@ -577,7 +581,7 @@ namespace Microsoft.ML.Featurizers
// TODO: Update entry points
[DllImport("Featurizers", EntryPoint = "TimeSeriesImputerFeaturizer_BinaryArchive_CreateEstimator", CallingConvention = CallingConvention.Cdecl), SuppressUnmanagedCodeSecurity]
private static unsafe extern bool CreateEstimatorNative(TypeId* grainTypes, IntPtr grainTypesSize, TypeId* dataTypes, IntPtr dataTypesSize, TimeSeriesImputerEstimator.ImputationStrategy strategy, bool* suppressTypeErrors, out IntPtr estimator, out IntPtr errorHandle);
private static extern unsafe bool CreateEstimatorNative(TypeId* grainTypes, IntPtr grainTypesSize, TypeId* dataTypes, IntPtr dataTypesSize, TimeSeriesImputerEstimator.ImputationStrategy strategy, bool* suppressTypeErrors, out IntPtr estimator, out IntPtr errorHandle);
[DllImport("Featurizers", EntryPoint = "TimeSeriesImputerFeaturizer_BinaryArchive_DestroyEstimator", CallingConvention = CallingConvention.Cdecl), SuppressUnmanagedCodeSecurity]
private static extern bool DestroyEstimatorNative(IntPtr estimator, out IntPtr errorHandle); // Should ONLY be called by safe handle
@ -589,7 +593,7 @@ namespace Microsoft.ML.Featurizers
private static extern bool CompleteTrainingNative(TransformerEstimatorSafeHandle estimator, out IntPtr errorHandle);
[DllImport("Featurizers", EntryPoint = "TimeSeriesImputerFeaturizer_BinaryArchive_Fit", CallingConvention = CallingConvention.Cdecl), SuppressUnmanagedCodeSecurity]
private static unsafe extern bool FitNative(TransformerEstimatorSafeHandle estimator, NativeBinaryArchiveData data, out FitResult fitResult, out IntPtr errorHandle);
private static extern unsafe bool FitNative(TransformerEstimatorSafeHandle estimator, NativeBinaryArchiveData data, out FitResult fitResult, out IntPtr errorHandle);
[DllImport("Featurizers", EntryPoint = "TimeSeriesImputerFeaturizer_BinaryArchive_CreateTransformerFromEstimator", CallingConvention = CallingConvention.Cdecl), SuppressUnmanagedCodeSecurity]
private static extern bool CreateTransformerFromEstimatorNative(TransformerEstimatorSafeHandle estimator, out IntPtr transformer, out IntPtr errorHandle);
@ -601,13 +605,13 @@ namespace Microsoft.ML.Featurizers
private static extern bool CreateOnnxSaveDataNative(TransformerEstimatorSafeHandle transformer, out IntPtr buffer, out IntPtr bufferSize, out IntPtr error);
[DllImport("Featurizers", EntryPoint = "TimeSeriesImputerFeaturizer_BinaryArchive_CreateTransformerFromSavedData"), SuppressUnmanagedCodeSecurity]
private static unsafe extern bool CreateTransformerFromSavedDataNative(byte* rawData, IntPtr bufferSize, out IntPtr transformer, out IntPtr errorHandle);
private static extern unsafe bool CreateTransformerFromSavedDataNative(byte* rawData, IntPtr bufferSize, out IntPtr transformer, out IntPtr errorHandle);
[DllImport("Featurizers", EntryPoint = "TimeSeriesImputerFeaturizer_BinaryArchive_OnDataCompleted"), SuppressUnmanagedCodeSecurity]
private static unsafe extern bool OnDataCompletedNative(TransformerEstimatorSafeHandle estimator, out IntPtr errorHandle);
private static extern unsafe bool OnDataCompletedNative(TransformerEstimatorSafeHandle estimator, out IntPtr errorHandle);
[DllImport("Featurizers", EntryPoint = "TimeSeriesImputerFeaturizer_BinaryArchive_GetState"), SuppressUnmanagedCodeSecurity]
private static unsafe extern bool GetStateNative(TransformerEstimatorSafeHandle estimator, out TrainingState trainingState, out IntPtr errorHandle);
private static extern unsafe bool GetStateNative(TransformerEstimatorSafeHandle estimator, out TrainingState trainingState, out IntPtr errorHandle);
#endregion

Просмотреть файл

@ -22,7 +22,7 @@ namespace Microsoft.ML.Transforms
internal sealed class TimeSeriesImputerDataView : IDataTransform
{
#region Typed Columns
private TimeSeriesImputerTransformer _parent;
private readonly TimeSeriesImputerTransformer _parent;
public class SharedColumnState
{
public SharedColumnState()
@ -269,7 +269,7 @@ namespace Microsoft.ML.Transforms
return typeof(T).GetNativeTypeIdFromType();
}
internal unsafe abstract T GetDataFromNativeBinaryArchiveData(byte* data, int offset);
internal abstract unsafe T GetDataFromNativeBinaryArchiveData(byte* data, int offset);
}
private abstract class NumericTypedColumn<T> : TypedColumn<T>
@ -308,7 +308,7 @@ namespace Microsoft.ML.Transforms
{
}
internal unsafe override byte GetDataFromNativeBinaryArchiveData(byte* data, int offset)
internal override unsafe byte GetDataFromNativeBinaryArchiveData(byte* data, int offset)
{
if (IsNullable)
{
@ -329,7 +329,7 @@ namespace Microsoft.ML.Transforms
{
}
internal unsafe override sbyte GetDataFromNativeBinaryArchiveData(byte* data, int offset)
internal override unsafe sbyte GetDataFromNativeBinaryArchiveData(byte* data, int offset)
{
if (IsNullable)
{
@ -350,7 +350,7 @@ namespace Microsoft.ML.Transforms
{
}
internal unsafe override short GetDataFromNativeBinaryArchiveData(byte* data, int offset)
internal override unsafe short GetDataFromNativeBinaryArchiveData(byte* data, int offset)
{
if (IsNullable)
{
@ -371,7 +371,7 @@ namespace Microsoft.ML.Transforms
{
}
internal unsafe override ushort GetDataFromNativeBinaryArchiveData(byte* data, int offset)
internal override unsafe ushort GetDataFromNativeBinaryArchiveData(byte* data, int offset)
{
if (IsNullable)
{
@ -392,7 +392,7 @@ namespace Microsoft.ML.Transforms
{
}
internal unsafe override int GetDataFromNativeBinaryArchiveData(byte* data, int offset)
internal override unsafe int GetDataFromNativeBinaryArchiveData(byte* data, int offset)
{
if (IsNullable)
{
@ -413,7 +413,7 @@ namespace Microsoft.ML.Transforms
{
}
internal unsafe override uint GetDataFromNativeBinaryArchiveData(byte* data, int offset)
internal override unsafe uint GetDataFromNativeBinaryArchiveData(byte* data, int offset)
{
if (IsNullable)
{
@ -434,7 +434,7 @@ namespace Microsoft.ML.Transforms
{
}
internal unsafe override long GetDataFromNativeBinaryArchiveData(byte* data, int offset)
internal override unsafe long GetDataFromNativeBinaryArchiveData(byte* data, int offset)
{
if (IsNullable)
{
@ -455,7 +455,7 @@ namespace Microsoft.ML.Transforms
{
}
internal unsafe override ulong GetDataFromNativeBinaryArchiveData(byte* data, int offset)
internal override unsafe ulong GetDataFromNativeBinaryArchiveData(byte* data, int offset)
{
if (IsNullable)
{
@ -476,7 +476,7 @@ namespace Microsoft.ML.Transforms
{
}
internal unsafe override float GetDataFromNativeBinaryArchiveData(byte* data, int offset)
internal override unsafe float GetDataFromNativeBinaryArchiveData(byte* data, int offset)
{
var bytes = new byte[sizeof(float)];
Marshal.Copy((IntPtr)(data + offset), bytes, 0, sizeof(float));
@ -491,7 +491,7 @@ namespace Microsoft.ML.Transforms
{
}
internal unsafe override double GetDataFromNativeBinaryArchiveData(byte* data, int offset)
internal override unsafe double GetDataFromNativeBinaryArchiveData(byte* data, int offset)
{
var bytes = new byte[sizeof(double)];
Marshal.Copy((IntPtr)(data + offset), bytes, 0, sizeof(double));
@ -506,7 +506,7 @@ namespace Microsoft.ML.Transforms
{
}
internal unsafe override bool GetDataFromNativeBinaryArchiveData(byte* data, int offset)
internal override unsafe bool GetDataFromNativeBinaryArchiveData(byte* data, int offset)
{
if (IsNullable)
{
@ -552,7 +552,7 @@ namespace Microsoft.ML.Transforms
binaryWriter.Write(stringBytes);
}
internal unsafe override ReadOnlyMemory<char> GetDataFromNativeBinaryArchiveData(byte* data, int offset)
internal override unsafe ReadOnlyMemory<char> GetDataFromNativeBinaryArchiveData(byte* data, int offset)
{
if (_isNullable)
{
@ -608,7 +608,7 @@ namespace Microsoft.ML.Transforms
binaryWriter.Write(value);
}
internal unsafe override DateTime GetDataFromNativeBinaryArchiveData(byte* data, int offset)
internal override unsafe DateTime GetDataFromNativeBinaryArchiveData(byte* data, int offset)
{
long value;
if (_isNullable)
@ -656,7 +656,7 @@ namespace Microsoft.ML.Transforms
internal class TransformedDataSafeHandle : SafeHandleZeroOrMinusOneIsInvalid
{
private IntPtr _size;
private readonly IntPtr _size;
public TransformedDataSafeHandle(IntPtr handle, IntPtr size) : base(true)
{
SetHandle(handle);
@ -728,7 +728,7 @@ namespace Microsoft.ML.Transforms
private sealed class Cursor : DataViewRowCursor
{
private readonly IChannelProvider _ch;
private DataViewRowCursor _input;
private readonly DataViewRowCursor _input;
private long _position;
private bool _isGood;
private readonly Dictionary<string, TypedColumn> _allColumns;

Просмотреть файл

@ -160,7 +160,7 @@ namespace Microsoft.ML.Transforms.Image
private sealed class Mapper : OneToOneMapperBase
{
private ImageGrayscalingTransformer _parent;
private readonly ImageGrayscalingTransformer _parent;
public Mapper(ImageGrayscalingTransformer parent, DataViewSchema inputSchema)
: base(parent.Host.Register(nameof(Mapper)), parent, inputSchema)

Просмотреть файл

@ -49,8 +49,8 @@ namespace Microsoft.ML.Transforms.Onnx
/// </summary>
public sealed class OnnxMapTypeAttribute : DataViewTypeAttribute
{
private Type _keyType;
private Type _valueType;
private readonly Type _keyType;
private readonly Type _valueType;
/// <summary>
/// Create a map (aka dictionary) type.

Просмотреть файл

@ -55,7 +55,7 @@ namespace Microsoft.ML.Transforms.Onnx
/// </summary>
public sealed class OnnxSequenceTypeAttribute : DataViewTypeAttribute
{
private Type _elemType;
private readonly Type _elemType;
// Make default constructor obsolete.
// Use default constructor will left the _elemType field empty and cause exception in methods using _elemType.

Просмотреть файл

@ -91,7 +91,7 @@ namespace Microsoft.ML.Transforms.Onnx
{
var index = OutputNames.IndexOf(name);
if (index < 0)
throw Contracts.ExceptParamValue(name, nameof(name), $"Onput tensor, {name}, does not exist in the ONNX model. " +
throw Contracts.ExceptParamValue(name, nameof(name), $"Ouput tensor, {name}, does not exist in the ONNX model. " +
$"Available output names are [{string.Join(",", OutputNames)}].");
return OutputsInfo[index];
}
@ -423,7 +423,7 @@ namespace Microsoft.ML.Transforms.Onnx
internal sealed class OnnxUtils
{
private static HashSet<Type> _onnxTypeMap =
private static readonly HashSet<Type> _onnxTypeMap =
new HashSet<Type>
{
typeof(Double),
@ -439,7 +439,7 @@ namespace Microsoft.ML.Transforms.Onnx
typeof(SByte),
typeof(Byte)
};
private static Dictionary<Type, InternalDataKind> _typeToKindMap =
private static readonly Dictionary<Type, InternalDataKind> _typeToKindMap =
new Dictionary<Type, InternalDataKind>
{
{ typeof(Single) , InternalDataKind.R4},

Просмотреть файл

@ -441,7 +441,7 @@ namespace Microsoft.ML.Data
private readonly ReaderOptions _readerOptions;
private int _curDataSetRow;
private IEnumerator<int> _dataSetEnumerator;
private IEnumerator<int> _blockEnumerator;
private readonly IEnumerator<int> _blockEnumerator;
private readonly IList[] _columnValues;
private readonly Random _rand;

Просмотреть файл

@ -370,20 +370,20 @@ namespace Microsoft.ML.Data
private static readonly FuncInstanceMethodInfo1<Cursor, int, DataViewType, Delegate> _createGetterDelegateCoreMethodInfo
= FuncInstanceMethodInfo1<Cursor, int, DataViewType, Delegate>.Create(target => target.CreateGetterDelegateCore<int>);
private PartitionedFileLoader _parent;
private readonly PartitionedFileLoader _parent;
private readonly bool[] _active;
private readonly bool[] _subActive; // Active columns of the sub-cursor.
private Delegate[] _getters;
private Delegate[] _subGetters; // Cached getters of the sub-cursor.
private readonly Delegate[] _getters;
private readonly Delegate[] _subGetters; // Cached getters of the sub-cursor.
private readonly IEnumerable<DataViewSchema.Column> _columnsNeeded;
private readonly IEnumerable<DataViewSchema.Column> _subActivecolumnsNeeded;
private ReadOnlyMemory<char>[] _colValues; // Column values cached from the file path.
private readonly ReadOnlyMemory<char>[] _colValues; // Column values cached from the file path.
private DataViewRowCursor _subCursor; // Sub cursor of the current file.
private IEnumerator<int> _fileOrder;
private readonly IEnumerator<int> _fileOrder;
public Cursor(IChannelProvider provider, PartitionedFileLoader parent, IMultiStreamSource files, IEnumerable<DataViewSchema.Column> columnsNeeded, Random rand)
: base(provider)
@ -744,7 +744,7 @@ namespace Microsoft.ML.Data
/// </summary>
/// <param name="basepath">A base path.</param>
/// <param name="files">A list of files under the base path.</param>
/// <returns>A realtive file path.</returns>
/// <returns>A relative file path.</returns>
private string GetRelativePath(string basepath, IMultiStreamSource files)
{
Contracts.CheckNonEmpty(basepath, nameof(basepath));

Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше