Cherry-pick the api-diff changes to produce markdown on the bots (#3947)

* [tools][api-diff] Add a tool to merge several .md files into a single one (#3814)

This depends on an updated mono-api-html tool (not yet available). This
PR will test the creation (empty headers) and storage of the files on bots.

Reference https://github.com/xamarin/xamarin-macios/issues/3787

* Bump macios-binaries to get latest mono-api-html.exe to produce markdown

external/macios-binaries

@@ -1 +1 @@
-Subproject commit fd01b78da403a904b675462eb73c748f6fcaac74
+Subproject commit 7cde1e545a5b76b32f46a25dbe0762b48ad64675

tools/apidiff/.gitignore

@@ -4,3 +4,7 @@ links/
 updated-references/
 *api-diff.html
 index.html
+ios-*.md
+tvos-*.md
+watchos-*.md
+macos-*.md

tools/apidiff/Makefile

@@ -13,7 +13,7 @@ endif
 MONO_API_HTML_DIR = $(MONO_PATH)/mcs/tools/mono-api-html
 MONO_API_INFO_DIR = $(MONO_PATH)/mcs/tools/corcompare
 MONO_API_INFO = $(MONO_API_INFO_DIR)/bin/Debug/mono-api-info.exe
-MONO_API_HTML = $(MONO_API_HTML_DIR)/bin/Debug/mono-api-html.exe
+MONO_API_HTML = $(TOP)/external/macios-binaries/mono-api-html.exe
 
 # I18N are excluded - but otherwise if should be like ../../builds/Makefile + what XI adds
 # in the order to the api-diff.html merged file
@@ -203,6 +203,7 @@ verify-reference-assemblies-mac: temp/native-32/Xamarin.Mac.xml temp/native-64/X
 clean-local::
 	rm -rf temp updated-references diff *.exe* api-diff.html
 	rm -rf *.dll*
+	rm -rf ios-*.md tvos-*.md watchos-*.md macos-*.md
 
 DIRS += temp diff
 
@@ -217,6 +218,40 @@ $(DIRS):
 .SECONDARY: $(foreach file,$(WATCHOS_ASSEMBLIES),temp/xi/$(file).xml)
 .SECONDARY: $(foreach file,$(TVOS_ASSEMBLIES),temp/xi/$(file).xml)
 
+merger.exe: merger.cs
+	$(Q) $(SYSTEM_CSC) -debug $< -out:$@
+
+ifdef INCLUDE_IOS
+ios-markdown: merger.exe $(foreach file,$(IOS_ASSEMBLIES),$(APIDIFF_DIR)/diff/xi/$(file).md)
+	$(Q) $(SYSTEM_MONO) --debug merger.exe Xamarin.iOS $(CURDIR)/diff/xi/Xamarin.iOS/ ios
+else
+ios-markdown: ; @true
+endif
+
+ifdef INCLUDE_TVOS
+tvos-markdown: merger.exe $(foreach file,$(TVOS_ASSEMBLIES),$(APIDIFF_DIR)/diff/xi/$(file).md)
+	$(Q) $(SYSTEM_MONO) --debug merger.exe Xamarin.TVOS $(CURDIR)/diff/xi/Xamarin.TVOS/ tvos
+else
+tvos-markdown: ; @true
+endif
+
+ifdef INCLUDE_WATCH
+watchos-markdown: merger.exe $(foreach file,$(WATCHOS_ASSEMBLIES),$(APIDIFF_DIR)/diff/xi/$(file).md)
+	$(Q) $(SYSTEM_MONO) --debug merger.exe Xamarin.WatchOS $(CURDIR)/diff/xi/Xamarin.WatchOS/ watchos
+else
+watchos-markdown: ; @true
+endif
+
+ifdef INCLUDE_MAC
+macos-markdown: merger.exe $(foreach file,$(MAC_ASSEMBLIES),$(APIDIFF_DIR)/diff/xm/$(file).md)
+	$(Q) $(SYSTEM_MONO) --debug merger.exe Xamarin.Mac $(CURDIR)/diff/xm/Xamarin.Mac/ macos
+else
+macos-markdown: ; @true
+endif
+
+$(APIDIFF_DIR)/diff/%.md: $(APIDIFF_DIR)/temp/%.xml $(APIDIFF_DIR)/references/%.xml $(MONO_API_HTML)
+	$(QF_GEN) mono --debug $(MONO_API_HTML) $(NEW_REGEX) $(ADD_REGEX) $(APIDIFF_DIR)/references/$*.xml $(APIDIFF_DIR)/temp/$*.xml -i 'INSObjectProtocol' --md $@
+
 wrench-api-diff:
 	@echo "@MonkeyWrench: AddDirectory: $(CURDIR)/diff/xi/Xamarin.iOS"
 	@echo "@MonkeyWrench: AddDirectory: $(CURDIR)/diff/xi/Xamarin.WatchOS"
@@ -243,6 +278,7 @@ endif
 	@echo "@MonkeyWrench: AddFile: $(CURDIR)/api-diff.html"
 	@# remove empty files so they're not uploaded
 	@-find $(CURDIR)/diff -size 0 | xargs rm
+	$(MAKE) -j8 ios-markdown tvos-markdown watchos-markdown macos-markdown
 ifdef INCLUDE_IOS
 	$(MAKE) verify-reference-assemblies-ios -j8
 endif
@@ -254,4 +290,5 @@ jenkins-api-diff:
 	-$(MAKE) wrench-api-diff
 	$(Q) mkdir -p $(JENKINS_RESULTS_DIRECTORY)/api-diff
 	$(Q) cp *.html $(JENKINS_RESULTS_DIRECTORY)/api-diff
+	$(Q) cp *.md $(JENKINS_RESULTS_DIRECTORY)/api-diff
 	$(Q) cp -R diff $(JENKINS_RESULTS_DIRECTORY)/api-diff

tools/apidiff/merger.cs

@@ -0,0 +1,93 @@
+using System;
+using System.Globalization;
+using System.IO;
+
+class Merger {
+
+	static string GetVersion (string line)
+	{
+		var end = line.LastIndexOf ('"');
+		var start = line.LastIndexOf ('"', end - 1) + 1;
+		return line.Substring (start, end - start);
+	}
+
+	public static void Process (string platform, string path, string os)
+	{
+		if (!Directory.Exists (path))
+			throw new DirectoryNotFoundException (path);
+
+		var content = new StringWriter ();
+		var files = Directory.GetFileSystemEntries (path, "*.md");
+		Array.Sort (files);
+		string from = "unknown";
+		string to = "unknown";
+		bool lookForVersion = false;
+		foreach (var file in files) {
+			// skip everything before and including title (single #) from each file, we already have one
+			string foundTitle = null;
+			foreach (var line in File.ReadAllLines (file)) {
+				if (foundTitle != null) {
+					content.WriteLine (line);
+					if (line == "#### Type Changed: ObjCRuntime.Constants") {
+						lookForVersion = true;
+					} else if (line.StartsWith ("#### ")) {
+						lookForVersion = false;
+					}
+					if (lookForVersion) {
+						if (line.StartsWith ("-public const string Version = ", StringComparison.Ordinal))
+							from = GetVersion (line);
+						if (line.StartsWith ("+public const string Version = ", StringComparison.Ordinal))
+							to = GetVersion (line);
+					}
+				} else if (line.StartsWith ("## ")) {
+					// everything with ## becomes an entry in the ToC
+					foundTitle = line.Substring (3);
+					content.WriteLine ($"<a name=\"{foundTitle}\" />");
+					content.WriteLine (); // required empty line
+					content.WriteLine (line);
+				}
+			}
+		}
+
+		var headers = new StringWriter ();
+		var title = $"{platform} SDK API diff: {from} vs {to}";
+		// https://github.com/MicrosoftDocs/xamarin-docs/blob/live/contributing-guidelines/template.md#metadata
+		headers.WriteLine ("---");
+		headers.WriteLine ($"title: \"{title}\"");
+		headers.WriteLine ($"description: List of API changes between {platform} versions {from} and {to}.");
+		headers.WriteLine ($"author: spouliot");
+		headers.WriteLine ($"ms.author: sepoulio");
+		headers.WriteLine ($"ms.date: {DateTime.Now.ToString ("d", new CultureInfo ("en-US"))}");
+		headers.WriteLine ($"ms.topic: article");
+		headers.WriteLine ($"ms.assetid: {Guid.NewGuid ().ToString ().ToLowerInvariant ()}");
+		headers.WriteLine ($"ms.prod: xamarin");
+		headers.WriteLine ("---");
+		headers.WriteLine ();
+		headers.WriteLine ($"# {title}");
+		headers.WriteLine ();
+
+		// https://github.com/MicrosoftDocs/xamarin-docs/blob/live/contributing-guidelines/template.md#file-name
+		var filename = $"{os}-{from}-{to}".Replace ('.', '-').ToLowerInvariant () + ".md";
+		File.WriteAllText (filename, headers.ToString ());
+
+		var alldiffs = content.ToString ();
+		if (alldiffs.Length == 0)
+			alldiffs = "No changes were found between both versions."; // should not happen for releases (versions change)
+		File.AppendAllText (filename, alldiffs);
+		Console.WriteLine ($"@MonkeyWrench: AddFile: {Path.GetFullPath (filename)}");
+
+		if (File.Exists ("api-diff.html"))
+			File.AppendAllText ("api-diff.html", $"\n<h2><a href=\"{filename}\">{platform} API diff (markdown)</a></h2>");
+	}
+
+	public static int Main (string [] args)
+	{
+		try {
+			Process (args [0], args [1], args [2]);
+			return 0;
+		} catch (Exception e) {
+			Console.WriteLine (e);
+			return 1;
+		}
+	}
+}