diff --git a/Directory.Build.props b/Directory.Build.props index 6b2845fa..489342e5 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -8,10 +8,15 @@ beforehand. We also don't need to add ourselves to MSBuildAllProjects, as that is done by the file that imports us. --> + + Debug + $(MSBuildThisFileDirectory) + + 3.4.194 all - \ No newline at end of file + diff --git a/apidocs/.editorconfig b/apidocs/.editorconfig new file mode 100644 index 00000000..5cd2d9ef --- /dev/null +++ b/apidocs/.editorconfig @@ -0,0 +1,169 @@ +# EditorConfig is awesome:http://EditorConfig.org + +# top-most EditorConfig file +root = true + +# Don't use tabs for indentation. +[*] +indent_style = space + +# (Please don't specify an indent_size here; that has too many unintended consequences.) + +[*.yml] +indent_size = 2 +indent_style = space + +# Code files +[*.{cs,csx,vb,vbx,h,cpp,idl}] +indent_size = 4 +insert_final_newline = true +trim_trailing_whitespace = true + +# Xml project files +[*.{csproj,vbproj,vcxproj,vcxproj.filters,proj,projitems,shproj,runsettings}] +indent_size = 2 + +# Xml config files +[*.{props,targets,ruleset,config,nuspec,resx,vsixmanifest,vsct}] +indent_size = 2 + +# JSON files +[*.json] +indent_size = 2 + +# Dotnet code style settings: +[*.{cs,vb}] +# Sort using and Import directives with System.* appearing first +dotnet_sort_system_directives_first = true +dotnet_style_qualification_for_field = true:warning +dotnet_style_qualification_for_property = true:warning +dotnet_style_qualification_for_method = true:warning +dotnet_style_qualification_for_event = true:warning + +# Use language keywords instead of framework type names for type references +dotnet_style_predefined_type_for_locals_parameters_members = true:suggestion +dotnet_style_predefined_type_for_member_access = true:suggestion + +# Suggest more modern language features when available +dotnet_style_object_initializer = true:suggestion +dotnet_style_collection_initializer = true:suggestion +dotnet_style_coalesce_expression = true:suggestion +dotnet_style_null_propagation = true:suggestion +dotnet_style_explicit_tuple_names = true:suggestion + +# Non-private static fields are PascalCase +dotnet_naming_rule.non_private_static_fields_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.non_private_static_fields_should_be_pascal_case.symbols = non_private_static_fields +dotnet_naming_rule.non_private_static_fields_should_be_pascal_case.style = non_private_static_field_style + +dotnet_naming_symbols.non_private_static_fields.applicable_kinds = field +dotnet_naming_symbols.non_private_static_fields.applicable_accessibilities = public, protected, internal, protected internal, private protected +dotnet_naming_symbols.non_private_static_fields.required_modifiers = static + +dotnet_naming_style.non_private_static_field_style.capitalization = pascal_case + +# Constants are PascalCase +dotnet_naming_rule.constants_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.constants_should_be_pascal_case.symbols = constants +dotnet_naming_rule.constants_should_be_pascal_case.style = constant_style + +dotnet_naming_symbols.constants.applicable_kinds = field, local +dotnet_naming_symbols.constants.required_modifiers = const + +dotnet_naming_style.constant_style.capitalization = pascal_case + +# Static fields are camelCase +dotnet_naming_rule.static_fields_should_be_camel_case.severity = suggestion +dotnet_naming_rule.static_fields_should_be_camel_case.symbols = static_fields +dotnet_naming_rule.static_fields_should_be_camel_case.style = static_field_style + +dotnet_naming_symbols.static_fields.applicable_kinds = field +dotnet_naming_symbols.static_fields.required_modifiers = static + +dotnet_naming_style.static_field_style.capitalization = camel_case + +# Instance fields are camelCase +dotnet_naming_rule.instance_fields_should_be_camel_case.severity = suggestion +dotnet_naming_rule.instance_fields_should_be_camel_case.symbols = instance_fields +dotnet_naming_rule.instance_fields_should_be_camel_case.style = instance_field_style + +dotnet_naming_symbols.instance_fields.applicable_kinds = field + +dotnet_naming_style.instance_field_style.capitalization = camel_case + +# Locals and parameters are camelCase +dotnet_naming_rule.locals_should_be_camel_case.severity = suggestion +dotnet_naming_rule.locals_should_be_camel_case.symbols = locals_and_parameters +dotnet_naming_rule.locals_should_be_camel_case.style = camel_case_style + +dotnet_naming_symbols.locals_and_parameters.applicable_kinds = parameter, local + +dotnet_naming_style.camel_case_style.capitalization = camel_case + +# Local functions are PascalCase +dotnet_naming_rule.local_functions_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.local_functions_should_be_pascal_case.symbols = local_functions +dotnet_naming_rule.local_functions_should_be_pascal_case.style = local_function_style + +dotnet_naming_symbols.local_functions.applicable_kinds = local_function + +dotnet_naming_style.local_function_style.capitalization = pascal_case + +# By default, name items with PascalCase +dotnet_naming_rule.members_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.members_should_be_pascal_case.symbols = all_members +dotnet_naming_rule.members_should_be_pascal_case.style = pascal_case_style + +dotnet_naming_symbols.all_members.applicable_kinds = * + +dotnet_naming_style.pascal_case_style.capitalization = pascal_case + +# CSharp code style settings: +[*.cs] +# Indentation preferences +csharp_indent_block_contents = true +csharp_indent_braces = false +csharp_indent_case_contents = true +csharp_indent_switch_labels = true +csharp_indent_labels = flush_left + +# Prefer "var" everywhere +csharp_style_var_for_built_in_types = true:suggestion +csharp_style_var_when_type_is_apparent = true:suggestion +csharp_style_var_elsewhere = false:suggestion + +# Prefer method-like constructs to have a block body +csharp_style_expression_bodied_methods = false:none +csharp_style_expression_bodied_constructors = false:none +csharp_style_expression_bodied_operators = false:none + +# Prefer property-like constructs to have an expression-body +csharp_style_expression_bodied_properties = true:none +csharp_style_expression_bodied_indexers = true:none +csharp_style_expression_bodied_accessors = true:none + +# Suggest more modern language features when available +csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion +csharp_style_pattern_matching_over_as_with_null_check = true:suggestion +csharp_style_inlined_variable_declaration = true:suggestion +csharp_style_throw_expression = true:suggestion +csharp_style_conditional_delegate_call = true:suggestion + +# Newline settings +csharp_new_line_before_open_brace = all +csharp_new_line_before_else = true +csharp_new_line_before_catch = true +csharp_new_line_before_finally = true +csharp_new_line_before_members_in_object_initializers = true +csharp_new_line_before_members_in_anonymous_types = true + +# Blocks are allowed +csharp_prefer_braces = true:silent + +[*.cs] + +# SA1130: Use lambda syntax +dotnet_diagnostic.SA1130.severity = silent + +[*.sln] +indent_style = tab diff --git a/apidocs/ApiDocs.sln b/apidocs/ApiDocs.sln new file mode 100644 index 00000000..909cccb7 --- /dev/null +++ b/apidocs/ApiDocs.sln @@ -0,0 +1,28 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.30114.105 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ScrapeDocs", "ScrapeDocs\ScrapeDocs.csproj", "{099C7E51-5176-4BA9-86D0-16307ED0D7D8}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Windows.SDK.Win32Docs", "Microsoft.Windows.SDK.Win32Docs\Microsoft.Windows.SDK.Win32Docs.csproj", "{E213ED4C-2A2A-40C5-A43B-4A922802B200}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {099C7E51-5176-4BA9-86D0-16307ED0D7D8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {099C7E51-5176-4BA9-86D0-16307ED0D7D8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {099C7E51-5176-4BA9-86D0-16307ED0D7D8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {099C7E51-5176-4BA9-86D0-16307ED0D7D8}.Release|Any CPU.Build.0 = Release|Any CPU + {E213ED4C-2A2A-40C5-A43B-4A922802B200}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E213ED4C-2A2A-40C5-A43B-4A922802B200}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E213ED4C-2A2A-40C5-A43B-4A922802B200}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E213ED4C-2A2A-40C5-A43B-4A922802B200}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal diff --git a/apidocs/Directory.Build.props b/apidocs/Directory.Build.props new file mode 100644 index 00000000..036cf95a --- /dev/null +++ b/apidocs/Directory.Build.props @@ -0,0 +1,22 @@ + + + + + $(RepoRootPath)bin\Packages\$(Configuration)\NuGet\ + true + 9.0 + enable + true + $(RepoRootPath)\strongname.snk + + + + + + + + + + + + diff --git a/apidocs/Microsoft.Windows.SDK.Win32Docs/ApiDetails.cs b/apidocs/Microsoft.Windows.SDK.Win32Docs/ApiDetails.cs new file mode 100644 index 00000000..228a06dc --- /dev/null +++ b/apidocs/Microsoft.Windows.SDK.Win32Docs/ApiDetails.cs @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Windows.SDK.Win32Docs +{ + using System; + using System.Collections.Generic; + using MessagePack; + + /// + /// Captures all the documentation we have available for an API. + /// + [MessagePackObject] + public class ApiDetails + { + /// + /// Gets or sets the URL that provides more complete documentation for this API. + /// + [Key(0)] + public Uri? HelpLink { get; set; } + + /// + /// Gets or sets a summary of what the API is for. + /// + [Key(1)] + public string? Description { get; set; } + + /// + /// Gets or sets the remarks section of the documentation. + /// + [Key(2)] + public string? Remarks { get; set; } + + /// + /// Gets or sets a collection of parameter docs, keyed by their names. + /// + [Key(3)] + public Dictionary Parameters { get; set; } = new(); + + /// + /// Gets or sets a collection of field docs, keyed by their names. + /// + [Key(4)] + public Dictionary Fields { get; set; } = new(); + + /// + /// Gets or sets the documentation of the return value of the API, if applicable. + /// + [Key(5)] + public string? ReturnValue { get; set; } + } +} diff --git a/apidocs/Microsoft.Windows.SDK.Win32Docs/Microsoft.Windows.SDK.Win32Docs.csproj b/apidocs/Microsoft.Windows.SDK.Win32Docs/Microsoft.Windows.SDK.Win32Docs.csproj new file mode 100644 index 00000000..975d8b95 --- /dev/null +++ b/apidocs/Microsoft.Windows.SDK.Win32Docs/Microsoft.Windows.SDK.Win32Docs.csproj @@ -0,0 +1,34 @@ + + + + netstandard2.0 + $(NoWarn);NU5128;NU5125 + + true + Microsoft Windows API documentation + Win32 Metadata + https://github.com/microsoft/win32metadata + https://aka.ms/WinSDKLicenseURL + true + images/windows.png + + + + + + + + + + + + + + + + + + + + + diff --git a/apidocs/Microsoft.Windows.SDK.Win32Docs/Microsoft.Windows.SDK.Win32Docs.targets b/apidocs/Microsoft.Windows.SDK.Win32Docs/Microsoft.Windows.SDK.Win32Docs.targets new file mode 100644 index 00000000..0ef782f5 --- /dev/null +++ b/apidocs/Microsoft.Windows.SDK.Win32Docs/Microsoft.Windows.SDK.Win32Docs.targets @@ -0,0 +1,20 @@ + + + + + https://github.com/microsoft/win32metadata/releases/tag/v$(Version) + + + + + + + + + diff --git a/apidocs/Microsoft.Windows.SDK.Win32Docs/buildTransitive/Microsoft.Windows.SDK.Win32Docs.props b/apidocs/Microsoft.Windows.SDK.Win32Docs/buildTransitive/Microsoft.Windows.SDK.Win32Docs.props new file mode 100644 index 00000000..a0562e47 --- /dev/null +++ b/apidocs/Microsoft.Windows.SDK.Win32Docs/buildTransitive/Microsoft.Windows.SDK.Win32Docs.props @@ -0,0 +1,9 @@ + + + $(MSBuildThisFileDirectory)..\apidocs.msgpack + + + + + + diff --git a/apidocs/README.md b/apidocs/README.md new file mode 100644 index 00000000..a3511360 --- /dev/null +++ b/apidocs/README.md @@ -0,0 +1,13 @@ +# This folder contains the code and package for API docs + +## Maintainence + +Periodically, run these commands from the root of the repo to update the docs we pack to the latest available: + +```cmd +cd ext\sdk-api +git fetch +git checkout origin/docs +cd .. +git add ext\sdk-api +``` diff --git a/apidocs/ScrapeDocs/.editorconfig b/apidocs/ScrapeDocs/.editorconfig new file mode 100644 index 00000000..5cd546e0 --- /dev/null +++ b/apidocs/ScrapeDocs/.editorconfig @@ -0,0 +1,7 @@ +[*.cs] + +# CA1303: Do not pass literals as localized parameters +dotnet_diagnostic.CA1303.severity = none + +# SA1600: Elements should be documented +dotnet_diagnostic.SA1600.severity = silent diff --git a/apidocs/ScrapeDocs/Directory.Build.props b/apidocs/ScrapeDocs/Directory.Build.props new file mode 100644 index 00000000..a3b22a14 --- /dev/null +++ b/apidocs/ScrapeDocs/Directory.Build.props @@ -0,0 +1,6 @@ + + + true + + + diff --git a/apidocs/ScrapeDocs/DocEnum.cs b/apidocs/ScrapeDocs/DocEnum.cs new file mode 100644 index 00000000..83946966 --- /dev/null +++ b/apidocs/ScrapeDocs/DocEnum.cs @@ -0,0 +1,136 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace ScrapeDocs +{ + using System; + using System.Collections.Generic; + using System.Linq; + + internal class DocEnum + { + internal DocEnum(bool isFlags, IReadOnlyDictionary members) + { + this.IsFlags = isFlags; + this.Members = members; + } + + internal bool IsFlags { get; } + + internal IReadOnlyDictionary Members { get; } + + public override bool Equals(object? obj) => this.Equals(obj as DocEnum); + + public override int GetHashCode() + { + unchecked + { + int hash = this.IsFlags ? 1 : 0; + foreach (KeyValuePair entry in this.Members) + { + hash += entry.Key.GetHashCode(); + hash += (int)(entry.Value.Value ?? 0u); + } + + return hash; + } + } + + public bool Equals(DocEnum? other) + { + if (other is null) + { + return false; + } + + if (this.IsFlags != other.IsFlags) + { + return false; + } + + if (this.Members.Count != other.Members.Count) + { + return false; + } + + foreach (KeyValuePair entry in this.Members) + { + if (!other.Members.TryGetValue(entry.Key, out (ulong? Value, string? Doc) value)) + { + return false; + } + + if (entry.Value.Value != value.Value) + { + return false; + } + } + + return true; + } + + internal string? GetRecommendedName(List<(string MethodName, string ParameterName, string HelpLink, bool IsMethod)> uses) + { + string? enumName = null; + if (uses.Count == 1) + { + var oneValue = uses[0]; + if (oneValue.ParameterName.Contains("flags", StringComparison.OrdinalIgnoreCase)) + { + // Only appears in one method, on a parameter named something like "flags". + enumName = $"{oneValue.MethodName}Flags"; + } + else + { + enumName = $"{oneValue.MethodName}_{oneValue.ParameterName}Flags"; + } + } + else + { + string firstName = this.Members.Keys.First(); + int commonPrefixLength = firstName.Length; + foreach (string key in this.Members.Keys) + { + commonPrefixLength = Math.Min(commonPrefixLength, GetCommonPrefixLength(key, firstName)); + } + + if (commonPrefixLength > 1) + { + int last_ = firstName.LastIndexOf('_', commonPrefixLength - 1); + if (last_ != -1 && last_ != commonPrefixLength - 1) + { + // Trim down to last underscore + commonPrefixLength = last_; + } + + if (commonPrefixLength > 1 && firstName[commonPrefixLength - 1] == '_') + { + // The enum values share a common prefix suitable to imply a name for the enum. + enumName = firstName.Substring(0, commonPrefixLength - 1); + } + } + } + + return enumName; + } + + private static int GetCommonPrefixLength(ReadOnlySpan first, ReadOnlySpan second) + { + int count = 0; + int minLength = Math.Min(first.Length, second.Length); + for (int i = 0; i < minLength; i++) + { + if (first[i] == second[i]) + { + count++; + } + else + { + break; + } + } + + return count; + } + } +} diff --git a/apidocs/ScrapeDocs/Program.cs b/apidocs/ScrapeDocs/Program.cs new file mode 100644 index 00000000..6e4251ae --- /dev/null +++ b/apidocs/ScrapeDocs/Program.cs @@ -0,0 +1,643 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace ScrapeDocs +{ + using System; + using System.Collections.Concurrent; + using System.Collections.Generic; + using System.Diagnostics; + using System.Diagnostics.CodeAnalysis; + using System.Globalization; + using System.IO; + using System.Linq; + using System.Reflection; + using System.Text; + using System.Text.Json; + using System.Text.RegularExpressions; + using System.Threading; + using MessagePack; + using Microsoft.Windows.SDK.Win32Docs; + using YamlDotNet; + using YamlDotNet.RepresentationModel; + + /// + /// Program entrypoint class. + /// + internal class Program + { + private static readonly Regex FileNamePattern = new Regex(@"^\w\w-\w+-([\w\-]+)$", RegexOptions.Compiled); + private static readonly Regex ParameterHeaderPattern = new Regex(@"^### -param (\w+)", RegexOptions.Compiled); + private static readonly Regex FieldHeaderPattern = new Regex(@"^### -field (?:\w+\.)*(\w+)", RegexOptions.Compiled); + private static readonly Regex ReturnHeaderPattern = new Regex(@"^## -returns", RegexOptions.Compiled); + private static readonly Regex RemarksHeaderPattern = new Regex(@"^## -remarks", RegexOptions.Compiled); + private static readonly Regex InlineCodeTag = new Regex(@"\(.*)\", RegexOptions.Compiled); + private static readonly Regex EnumNameCell = new Regex(@"\]*\>\([\dxa-f]+)\<\/dt\>", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private readonly string contentBasePath; + private readonly string outputPath; + + private Program(string contentBasePath, string outputPath) + { + this.contentBasePath = contentBasePath; + this.outputPath = outputPath; + } + + private bool EmitEnums { get; set; } + + private static int Main(string[] args) + { + using var cts = new CancellationTokenSource(); + Console.CancelKeyPress += (s, e) => + { + Console.WriteLine("Canceling..."); + cts.Cancel(); + e.Cancel = true; + }; + + if (args.Length < 2) + { + Console.Error.WriteLine("USAGE: {0} [enums]"); + return 1; + } + + string contentBasePath = args[0]; + string outputPath = args[1]; + bool emitEnums = args.Length > 2 ? args[2] == "enums" : false; + + try + { + new Program(contentBasePath, outputPath) { EmitEnums = true }.Worker(cts.Token); + } + catch (OperationCanceledException ex) when (ex.CancellationToken == cts.Token) + { + return 2; + } + + return 0; + } + + private static void Expect(string? expected, string? actual) + { + if (expected != actual) + { + throw new InvalidOperationException($"Expected: \"{expected}\" but read: \"{actual}\"."); + } + } + + private int AnalyzeEnums(ConcurrentDictionary results, ConcurrentDictionary<(string MethodName, string ParameterName, string HelpLink), DocEnum> parameterEnums, ConcurrentDictionary<(string MethodName, string ParameterName, string HelpLink), DocEnum> fieldEnums) + { + var uniqueEnums = new Dictionary>(); + var constantsDocs = new Dictionary>(); + + void Collect(ConcurrentDictionary<(string MethodName, string ParameterName, string HelpLink), DocEnum> enums, bool isMethod) + { + foreach (var item in enums) + { + if (!uniqueEnums.TryGetValue(item.Value, out List<(string MethodName, string ParameterName, string HelpLink, bool IsMethod)>? list)) + { + uniqueEnums.Add(item.Value, list = new()); + } + + list.Add((item.Key.MethodName, item.Key.ParameterName, item.Key.HelpLink, isMethod)); + + foreach (KeyValuePair enumValue in item.Value.Members) + { + if (enumValue.Value.Doc is object) + { + if (!constantsDocs.TryGetValue(enumValue.Key, out List<(string MethodName, string HelpLink, string Doc)>? values)) + { + constantsDocs.Add(enumValue.Key, values = new()); + } + + values.Add((item.Key.MethodName, item.Key.HelpLink, enumValue.Value.Doc)); + } + } + } + } + + Collect(parameterEnums, isMethod: true); + Collect(fieldEnums, isMethod: false); + + foreach (var item in constantsDocs) + { + var docNode = new ApiDetails(); + docNode.Description = item.Value[0].Doc; + + // If the documentation varies across methods, just link to each document. + bool differenceDetected = false; + for (int i = 1; i < item.Value.Count; i++) + { + if (item.Value[i].Doc != docNode.Description) + { + differenceDetected = true; + break; + } + } + + if (differenceDetected) + { + docNode.Description = "Documentation varies per use. Refer to each: " + string.Join(", ", item.Value.Select(v => @$"{v.MethodOrStructName}")) + "."; + } + else + { + // Just point to any arbitrary method that documents it. + docNode.HelpLink = new Uri(item.Value[0].HelpLink); + } + + results.TryAdd(item.Key, docNode); + } + + if (this.EmitEnums) + { + string enumDirectory = Path.GetDirectoryName(this.outputPath) ?? throw new InvalidOperationException("Unable to determine where to write enums."); + Directory.CreateDirectory(enumDirectory); + using var enumsJsonStream = File.OpenWrite(Path.Combine(enumDirectory, "enums.json")); + using var writer = new Utf8JsonWriter(enumsJsonStream, new JsonWriterOptions { Indented = true }); + writer.WriteStartArray(); + + foreach (KeyValuePair> item in uniqueEnums) + { + writer.WriteStartObject(); + + if (item.Key.GetRecommendedName(item.Value) is string enumName) + { + writer.WriteString("name", enumName); + } + + writer.WriteBoolean("flags", item.Key.IsFlags); + + writer.WritePropertyName("members"); + writer.WriteStartArray(); + foreach (var member in item.Key.Members) + { + writer.WriteStartObject(); + writer.WriteString("name", member.Key); + if (member.Value.Value is ulong value) + { + writer.WriteString("value", value.ToString(CultureInfo.InvariantCulture)); + } + + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + + writer.WritePropertyName("uses"); + writer.WriteStartArray(); + foreach (var uses in item.Value) + { + writer.WriteStartObject(); + + int periodIndex = uses.MethodName.IndexOf('.', StringComparison.Ordinal); + string? iface = periodIndex >= 0 ? uses.MethodName.Substring(0, periodIndex) : null; + string name = periodIndex >= 0 ? uses.MethodName.Substring(periodIndex + 1) : uses.MethodName; + + if (iface is string) + { + writer.WriteString("interface", iface); + } + + writer.WriteString(uses.IsMethod ? "method" : "struct", name); + writer.WriteString(uses.IsMethod ? "parameter" : "field", uses.ParameterName); + + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + } + + return constantsDocs.Count; + } + + private void Worker(CancellationToken cancellationToken) + { + Console.WriteLine("Enumerating documents to be parsed..."); + string[] paths = Directory.GetFiles(this.contentBasePath, "??-*-*.md", SearchOption.AllDirectories) + ////.Where(p => p.Contains(@"ns-winsock2-blob", StringComparison.OrdinalIgnoreCase)).ToArray() + ; + + Console.WriteLine("Parsing documents..."); + var timer = Stopwatch.StartNew(); + var parsedNodes = from path in paths.AsParallel() + let result = this.ParseDocFile(path) + where result is not null + select (Path: path, result.Value.ApiName, result.Value.Docs, result.Value.EnumsByParameter, result.Value.EnumsByField); + var results = new ConcurrentDictionary(); + var parameterEnums = new ConcurrentDictionary<(string MethodName, string ParameterName, string HelpLink), DocEnum>(); + var fieldEnums = new ConcurrentDictionary<(string StructName, string FieldName, string HelpLink), DocEnum>(); + if (Debugger.IsAttached) + { + parsedNodes = parsedNodes.WithDegreeOfParallelism(1); // improve debuggability + } + + parsedNodes + .WithCancellation<(string Path, string ApiName, ApiDetails Docs, IReadOnlyDictionary EnumsByParameter, IReadOnlyDictionary EnumsByField)>(cancellationToken) + .ForAll(result => + { + results.TryAdd(result.ApiName, result.Docs); + foreach (var e in result.EnumsByParameter) + { + if (result.Docs.HelpLink is object) + { + parameterEnums.TryAdd((result.ApiName, e.Key, result.Docs.HelpLink.AbsoluteUri), e.Value); + } + } + + foreach (var e in result.EnumsByField) + { + if (result.Docs.HelpLink is object) + { + fieldEnums.TryAdd((result.ApiName, e.Key, result.Docs.HelpLink.AbsoluteUri), e.Value); + } + } + }); + if (paths.Length == 0) + { + Console.Error.WriteLine("No documents found to parse."); + } + else + { + Console.WriteLine("Parsed {2} documents in {0} ({1} per document)", timer.Elapsed, timer.Elapsed / paths.Length, paths.Length); + Console.WriteLine($"Found {parameterEnums.Count + fieldEnums.Count} enums."); + } + + Console.WriteLine("Analyzing and naming enums and collecting docs on their members..."); + int constantsCount = this.AnalyzeEnums(results, parameterEnums, fieldEnums); + Console.WriteLine($"Found docs for {constantsCount} constants."); + + Console.WriteLine("Writing results to \"{0}\"", this.outputPath); + Directory.CreateDirectory(Path.GetDirectoryName(this.outputPath)!); + using var outputFileStream = File.OpenWrite(this.outputPath); + MessagePackSerializer.Serialize(outputFileStream, results.ToDictionary(kv => kv.Key, kv => kv.Value), MessagePackSerializerOptions.Standard); + } + + private (string ApiName, ApiDetails Docs, IReadOnlyDictionary EnumsByParameter, IReadOnlyDictionary EnumsByField)? ParseDocFile(string filePath) + { + try + { + var enumsByParameter = new Dictionary(); + var enumsByField = new Dictionary(); + var yaml = new YamlStream(); + using StreamReader mdFileReader = File.OpenText(filePath); + using var markdownToYamlReader = new YamlSectionReader(mdFileReader); + var yamlBuilder = new StringBuilder(); + ApiDetails docs = new(); + string? line; + while ((line = markdownToYamlReader.ReadLine()) is object) + { + yamlBuilder.AppendLine(line); + } + + try + { + yaml.Load(new StringReader(yamlBuilder.ToString())); + } + catch (YamlDotNet.Core.YamlException ex) + { + Debug.WriteLine("YAML parsing error in \"{0}\": {1}", filePath, ex.Message); + return null; + } + + YamlSequenceNode methodNames = (YamlSequenceNode)yaml.Documents[0].RootNode["api_name"]; + bool TryGetProperName(string searchFor, string? suffix, [NotNullWhen(true)] out string? match) + { + if (suffix is string) + { + if (searchFor.EndsWith(suffix, StringComparison.Ordinal)) + { + searchFor = searchFor.Substring(0, searchFor.Length - suffix.Length); + } + else + { + match = null; + return false; + } + } + + match = methodNames.Children.Cast().FirstOrDefault(c => string.Equals(c.Value?.Replace('.', '-'), searchFor, StringComparison.OrdinalIgnoreCase))?.Value; + + if (suffix is string && match is object) + { + match += suffix.ToUpper(CultureInfo.InvariantCulture); + } + + return match is object; + } + + string presumedMethodName = FileNamePattern.Match(Path.GetFileNameWithoutExtension(filePath)).Groups[1].Value; + + // Some structures have filenames that include the W or A suffix when the content doesn't. So try some fuzzy matching. + if (!TryGetProperName(presumedMethodName, null, out string? properName) && + !TryGetProperName(presumedMethodName, "a", out properName) && + !TryGetProperName(presumedMethodName, "w", out properName) && + !TryGetProperName(presumedMethodName, "32", out properName) && + !TryGetProperName(presumedMethodName, "64", out properName)) + { + Debug.WriteLine("WARNING: Could not find proper API name in: {0}", filePath); + return null; + } + + Uri helpLink = new Uri("https://docs.microsoft.com/windows/win32/api/" + filePath.Substring(this.contentBasePath.Length, filePath.Length - 3 - this.contentBasePath.Length).Replace('\\', '/')); + docs.HelpLink = helpLink; + + var description = ((YamlMappingNode)yaml.Documents[0].RootNode).Children.FirstOrDefault(n => n.Key is YamlScalarNode { Value: "description" }).Value as YamlScalarNode; + docs.Description = description?.Value; + + // Search for parameter/field docs + var parametersMap = new YamlMappingNode(); + var fieldsMap = new YamlMappingNode(); + StringBuilder docBuilder = new StringBuilder(); + line = mdFileReader.ReadLine(); + + static string FixupLine(string line) + { + line = line.Replace("href=\"/", "href=\"https://docs.microsoft.com/"); + line = InlineCodeTag.Replace(line, match => $"{match.Groups[1].Value}"); + return line; + } + + void ParseTextSection(out string text) + { + while ((line = mdFileReader.ReadLine()) is object) + { + if (line.StartsWith('#')) + { + break; + } + + line = FixupLine(line); + docBuilder.AppendLine(line); + } + + text = docBuilder.ToString(); + + docBuilder.Clear(); + } + + IReadOnlyDictionary ParseEnumTable() + { + var enums = new Dictionary(); + int state = 0; + const int StateReadingHeader = 0; + const int StateReadingName = 1; + const int StateLookingForDetail = 2; + const int StateReadingDocColumn = 3; + string? enumName = null; + ulong? enumValue = null; + var docsBuilder = new StringBuilder(); + while ((line = mdFileReader.ReadLine()) is object) + { + if (line == "") + { + break; + } + + switch (state) + { + case StateReadingHeader: + // Reading TR header + if (line == "") + { + state = StateReadingName; + } + + break; + + case StateReadingName: + // Reading an enum row's name column. + Match m = EnumNameCell.Match(line); + if (m.Success) + { + enumName = m.Groups[1].Value; + if (enumName == "0") + { + enumName = "None"; + enumValue = 0; + } + + state = StateLookingForDetail; + } + + break; + + case StateLookingForDetail: + // Looking for an enum row's doc column. + m = EnumOrdinalValue.Match(line); + if (m.Success) + { + string value = m.Groups[1].Value; + bool hex = value.StartsWith("0x", StringComparison.OrdinalIgnoreCase); + if (hex) + { + value = value.Substring(2); + } + + enumValue = ulong.Parse(value, hex ? NumberStyles.HexNumber : NumberStyles.Integer, CultureInfo.InvariantCulture); + } + else if (line.StartsWith("", StringComparison.OrdinalIgnoreCase)) + { + // The row ended before we found the doc column. + state = StateReadingName; + enums.Add(enumName!, (enumValue, null)); + enumName = null; + enumValue = null; + } + + break; + + case StateReadingDocColumn: + // Reading the enum row's doc column. + if (line.StartsWith("", StringComparison.OrdinalIgnoreCase)) + { + state = StateReadingName; + + // Some docs are invalid in documenting the same enum multiple times. + if (!enums.ContainsKey(enumName!)) + { + enums.Add(enumName!, (enumValue, docsBuilder.ToString().Trim())); + } + + enumName = null; + enumValue = null; + docsBuilder.Clear(); + break; + } + + docsBuilder.AppendLine(FixupLine(line)); + break; + } + } + + return enums; + } + + void ParseSection(Match match, IDictionary receivingMap, bool lookForParameterEnums = false, bool lookForFieldEnums = false) + { + string sectionName = match.Groups[1].Value; + bool foundEnum = false; + bool foundEnumIsFlags = false; + while ((line = mdFileReader.ReadLine()) is object) + { + if (line.StartsWith('#')) + { + break; + } + + if (lookForParameterEnums || lookForFieldEnums) + { + if (foundEnum) + { + if (line == "") + { + IReadOnlyDictionary enumNamesAndDocs = ParseEnumTable(); + if (enumNamesAndDocs.Count > 0) + { + var enums = lookForParameterEnums ? enumsByParameter : enumsByField; + if (!enums.ContainsKey(sectionName)) + { + enums.Add(sectionName, new DocEnum(foundEnumIsFlags, enumNamesAndDocs)); + } + } + + lookForParameterEnums = false; + lookForFieldEnums = false; + } + } + else + { + foundEnum = line.Contains("of the following values", StringComparison.OrdinalIgnoreCase); + foundEnumIsFlags = line.Contains("combination of", StringComparison.OrdinalIgnoreCase) + || line.Contains("zero or more of", StringComparison.OrdinalIgnoreCase) + || line.Contains("one or both of", StringComparison.OrdinalIgnoreCase) + || line.Contains("one or more of", StringComparison.OrdinalIgnoreCase); + } + } + + if (!foundEnum) + { + line = FixupLine(line); + docBuilder.AppendLine(line); + } + } + + receivingMap.TryAdd(sectionName, docBuilder.ToString().Trim()); + docBuilder.Clear(); + } + + while (line is object) + { + if (ParameterHeaderPattern.Match(line) is Match { Success: true } parameterMatch) + { + ParseSection(parameterMatch, docs.Parameters, lookForParameterEnums: true); + } + else if (FieldHeaderPattern.Match(line) is Match { Success: true } fieldMatch) + { + ParseSection(fieldMatch, docs.Fields, lookForFieldEnums: true); + } + else if (RemarksHeaderPattern.Match(line) is Match { Success: true } remarksMatch) + { + string remarks; + ParseTextSection(out remarks); + docs.Remarks = remarks; + } + else + { + // TODO: don't break out of this loop so soon... remarks sometimes follows return value docs. + if (line is object && ReturnHeaderPattern.IsMatch(line)) + { + break; + } + + line = mdFileReader.ReadLine(); + } + } + + // Search for return value documentation + while (line is object) + { + Match m = ReturnHeaderPattern.Match(line); + if (m.Success) + { + while ((line = mdFileReader.ReadLine()) is object) + { + if (line.StartsWith('#')) + { + break; + } + + docBuilder.AppendLine(line); + } + + docs.ReturnValue = docBuilder.ToString().Trim(); + docBuilder.Clear(); + break; + } + else + { + line = mdFileReader.ReadLine(); + } + } + + return (properName, docs, enumsByParameter, enumsByField); + } + catch (Exception ex) + { + throw new ApplicationException($"Failed parsing \"{filePath}\".", ex); + } + } + + private class YamlSectionReader : TextReader + { + private readonly StreamReader fileReader; + private bool firstLineRead; + private bool lastLineRead; + + internal YamlSectionReader(StreamReader fileReader) + { + this.fileReader = fileReader; + } + + public override string? ReadLine() + { + if (this.lastLineRead) + { + return null; + } + + if (!this.firstLineRead) + { + Expect("---", this.fileReader.ReadLine()); + this.firstLineRead = true; + } + + string? line = this.fileReader.ReadLine(); + if (line == "---") + { + this.lastLineRead = true; + return null; + } + + return line; + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + this.fileReader.Dispose(); + } + + base.Dispose(disposing); + } + } + } +} diff --git a/apidocs/ScrapeDocs/ScrapeDocs.csproj b/apidocs/ScrapeDocs/ScrapeDocs.csproj new file mode 100644 index 00000000..b899a058 --- /dev/null +++ b/apidocs/ScrapeDocs/ScrapeDocs.csproj @@ -0,0 +1,18 @@ + + + + Exe + net5.0 + false + + + + + + + + + + + + diff --git a/apidocs/global.json b/apidocs/global.json new file mode 100644 index 00000000..5ecd5ee6 --- /dev/null +++ b/apidocs/global.json @@ -0,0 +1,5 @@ +{ + "sdk": { + "version": "5.0.301" + } +} diff --git a/apidocs/stylecop.json b/apidocs/stylecop.json new file mode 100644 index 00000000..2453d57c --- /dev/null +++ b/apidocs/stylecop.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://raw.githubusercontent.com/DotNetAnalyzers/StyleCopAnalyzers/master/StyleCop.Analyzers/StyleCop.Analyzers/Settings/stylecop.schema.json", + "settings": { + "documentationRules": { + "companyName": "Microsoft Corporation", + "copyrightText": "Copyright (c) {companyName}. All rights reserved.\nLicensed under the {licenseName} license. See {licenseFile} file in the project root for full license information.", + "variables": { + "licenseName": "MIT", + "licenseFile": "LICENSE" + }, + "fileNamingConvention": "metadata", + "xmlHeader": false + } + } +} diff --git a/apidocs/version.json b/apidocs/version.json new file mode 100644 index 00000000..356015b5 --- /dev/null +++ b/apidocs/version.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", + "version": "0.1-alpha", + "versionHeightOffset": 0, // manually +1 each time the ext/sdk-api submodule tree is updated + "pathFilters": [ + ".", + "../ext/sdk-api" // doesn't work yet: https://github.com/dotnet/Nerdbank.GitVersioning/issues/625 + ] +} diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a267b792..7253fb80 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -2,7 +2,11 @@ trigger: branches: include: - master -pr: none +pr: +- master + +variables: + BuildConfiguration: Release jobs: - job: scrape_x64 @@ -22,7 +26,7 @@ jobs: inputs: packageType: 'sdk' version: '3.x' - + - task: PowerShell@2 displayName: Set build version inputs: @@ -41,7 +45,7 @@ jobs: arguments: '-arch x64' errorActionPreference: 'continue' pwsh: true - + - task: PowerShell@2 displayName: Scrape constants inputs: @@ -73,7 +77,7 @@ jobs: inputs: packageType: 'sdk' version: '3.x' - + - task: PowerShell@2 displayName: GenerateMetadataSource.ps1 - x86 inputs: @@ -81,7 +85,7 @@ jobs: arguments: '-arch x86' errorActionPreference: 'continue' pwsh: true - + - publish: 'generation\emitter\generated\x86' displayName: Publish x86 emitter assets artifact: 'emitter_generated_x86' @@ -102,7 +106,7 @@ jobs: inputs: packageType: 'sdk' version: '3.x' - + - task: PowerShell@2 displayName: GenerateMetadataSource.ps1 - arm64 inputs: @@ -110,7 +114,7 @@ jobs: arguments: '-arch arm64' errorActionPreference: 'continue' pwsh: true - + - publish: 'generation\emitter\generated\arm64' displayName: Publish arm64 emitter assets artifact: 'emitter_generated_arm64' @@ -139,8 +143,8 @@ jobs: - task: UseDotNet@2 displayName: Install DotNet 2.1.x for signing tasks inputs: - packageType: 'sdk' - version: '2.1.x' + packageType: runtime + version: 2.1.x - task: DownloadPipelineArtifact@2 displayName: Download x64 scraper obj assets @@ -172,7 +176,7 @@ jobs: filePath: 'scripts\BuildMetadataBin.ps1' arguments: '-arch crossarch -SkipConstants' pwsh: true - + - publish: 'bin' artifact: 'bin' @@ -223,8 +227,8 @@ jobs: SessionTimeout: '60' MaxConcurrency: '50' MaxRetryAttempts: '2' - condition: eq(variables['SignFiles'], 'true') - + condition: and(succeeded(), eq(variables['SignFiles'], 'true')) + # There's a problem on microsoft.visualstudio.com that requires the guid instead of NuGetCommand@2 - task: EsrpCodeSigning@1 @@ -268,8 +272,8 @@ jobs: SessionTimeout: '60' MaxConcurrency: '50' MaxRetryAttempts: '2' - condition: eq(variables['SignFiles'], 'true') - + condition: and(succeeded(), eq(variables['SignFiles'], 'true')) + - task: PowerShell@2 displayName: Pack metadata package inputs: @@ -317,14 +321,14 @@ jobs: SessionTimeout: '60' MaxConcurrency: '50' MaxRetryAttempts: '2' - condition: eq(variables['SignFiles'], 'true') - + condition: and(succeeded(), eq(variables['SignFiles'], 'true')) + - task: PublishPipelineArtifact@1 displayName: 'Publish NuGet packages to pipeline artifacts' inputs: targetPath: '$(OutputPackagesDir)' artifact: NuGetPackages - + # There's a problem on microsoft.visualstudio.com that requires the guid instead of NuGetCommand@2 # Don't publish if we're using pre-generated source - task: 333b11bd-d341-40d9-afcf-b32d5ce6f23b@2 @@ -334,3 +338,70 @@ jobs: packagesToPush: '$(OutputPackagesDir)/**/*.nupkg;!$(OutputPackagesDir)/**/*.symbols.nupkg' publishVstsFeed: 'c1408dcb-1833-4ae4-9af5-1a891a12cc3c' allowPackageConflicts: true + +- job: build_docs + displayName: Build API docs + pool: + vmImage: ubuntu-20.04 + steps: + - checkout: self + clean: true + submodules: recursive + - task: UseDotNet@2 + displayName: ⚙ Install .NET SDK + inputs: + packageType: sdk + useGlobalJson: true + workingDirectory: apidocs + + # ESRP Authenticode sign package DLLs + - task: UseDotNet@2 + displayName: ⚙ Install .NET Core 2.1.x + inputs: + packageType: runtime + version: 2.1.x + + - script: dotnet pack -c $(BuildConfiguration) + displayName: 📦 dotnet pack + workingDirectory: apidocs + - task: EsrpCodeSigning@1 + displayName: ✒ NuGet sign + inputs: + ConnectedServiceName: Undocked RegFree Signing Connection + FolderPath: $(System.DefaultWorkingDirectory)/bin/Packages/$(BuildConfiguration)/NuGet + Pattern: '*.nupkg' + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "KeyCode" : "CP-401405", + "OperationCode" : "NuGetSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + }, + { + "KeyCode" : "CP-401405", + "OperationCode" : "NuGetVerify", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + SessionTimeout: 60 + MaxConcurrency: 50 + MaxRetryAttempts: 5 + condition: and(succeeded(), eq(variables['SignFiles'], 'true')) + - publish: bin/Packages/$(BuildConfiguration)/NuGet + artifact: ApiDocsNuGetPackages + displayName: 📢 Publish package + # There's a problem on microsoft.visualstudio.com that requires the guid instead of NuGetCommand@2 + # Don't publish if we're using pre-generated source + - task: 333b11bd-d341-40d9-afcf-b32d5ce6f23b@2 + displayName: 📤 NuGet push + inputs: + command: push + packagesToPush: $(System.DefaultWorkingDirectory)/bin/Packages/$(BuildConfiguration)/NuGet/*.nupkg + publishVstsFeed: c1408dcb-1833-4ae4-9af5-1a891a12cc3c + allowPackageConflicts: true + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) diff --git a/ext/sdk-api b/ext/sdk-api index bd3c10cc..c2511992 160000 --- a/ext/sdk-api +++ b/ext/sdk-api @@ -1 +1 @@ -Subproject commit bd3c10cc5c89ef1209ff51c2e85217e3f1936b29 +Subproject commit c251199235b283ada4e0c5afe352077f7453a680 diff --git a/strongname.snk b/strongname.snk new file mode 100644 index 00000000..1a45d560 Binary files /dev/null and b/strongname.snk differ