using dotnetthanks; using Microsoft.Extensions.Configuration; using Octokit; using System.Diagnostics; using System.Net.Http.Json; using System.Text.Json; using System.Text.RegularExpressions; namespace dotnetthanks_loader { class Program { private static HttpClient _client; private static readonly string[] exclusions = new string[] { "dependabot[bot]", "github-actions[bot]", "msftbot[bot]", "github-actions[bot]", "dotnet bot", "dotnet-bot", "nuget team bot", "net source-build bot", "dotnet-maestro-bot", "dotnet-maestro[bot]" }; private static string _token; private static GitHubClient _ghclient; static async Task Main(string[] args) { var config = new ConfigurationBuilder() .SetBasePath(AppDomain.CurrentDomain.BaseDirectory) .AddEnvironmentVariables() .AddUserSecrets() .Build(); _token = config.GetSection("GITHUB_TOKEN").Value; _ghclient = new GitHubClient(new ProductHeaderValue("dotnet-thanks")); var basic = new Credentials(config.GetSection("GITHUB_CLIENTID").Value, config.GetSection("GITHUB_CLIENTSECRET").Value); _ghclient.Credentials = basic; var repo = "core"; var owner = "dotnet"; // load all releases for dotnet/core IEnumerable allReleases = await LoadReleasesAsync(owner, repo); // Sort releases from the yongest to the oldest by version // E.g. // 5.0.1 // 5.0.0 // GA // 5.0.0-RC2 // 5.0.0-RC1 // 5.0.0-preview9 // ... // 5.0.0-preview2 // 3.1.10 // 3.1.9 // 3.1.8 // ... // 3.1.0 // GA // ... // List sortedReleases = allReleases .OrderByDescending(o => o.Version) .ThenByDescending(o => o.Id) .ToList(); Dictionary sortedMajorReleasesDictionary = new (); List sortedMajorReleasesList = new (); // If arg 1 is "diff" calculate the diff and append it to current core.js file if (args != null && args.Length > 0 && args[0] == "diff") { // load current core.json file #if DEBUG IEnumerable corejson = LoadCurrentCoreJson(); #else IEnumerable corejson = await LoadCurrentCoreJsonAsync(); #endif // create a dictionary with preprocessed data foreach (var release in corejson) { sortedMajorReleasesDictionary.Add($"{release.Version.Major}.{release.Version.Minor}", release); } List processedReleases = new(); foreach (var o in corejson) { processedReleases.AddRange(o.ProcessedReleases); } List diff = sortedReleases.Where(o => !processedReleases.Contains(o.Tag)).ToList(); // Check if releases in diff are not in dictionary foreach (var release in diff) { if (!sortedMajorReleasesDictionary.ContainsKey($"{release.Version.Major}.{release.Version.Minor}")) { sortedMajorReleasesDictionary.Add($"{release.Version.Major}.{release.Version.Minor}", new MajorRelease { Contributions = 0, Contributors = new(), Product = release.Product, Name = $"{release.Product} {release.Version.Major}.{release.Version.Minor}", Version = release.Version, Tag = $"v{release.Version.Major}.{release.Version.Minor}", ProcessedReleases = new() }); } } if (diff.Count != 0) { Console.WriteLine($"Processing diffs in releases...\n{repo} - {diff.Count}"); // For each new release, find its prior release and add it into a new list for commit comparison List sortedNewReleases = new (); List majorReleasesList = new (); var latestGARelease = sortedReleases.ToList().Find(r => r.IsGA); foreach(var r in diff) { var idx = sortedReleases.IndexOf(r); var previousVersion = sortedReleases[idx + 1]; // Add new release sortedNewReleases.Add(r); // If previous version isn't in newReleases already - add it if (!diff.Contains(previousVersion)) { // Append the latest processed release or last GA if first release if (r.Version.Major != previousVersion.Version.Major) { sortedNewReleases.Add(latestGARelease); } else if (r.Version.Major == previousVersion.Version.Major) { sortedNewReleases.Add(previousVersion); } } else if (diff.Contains(previousVersion) && r.Version.Major != previousVersion.Version.Major) { sortedNewReleases.Add(latestGARelease); } } sortedNewReleases = sortedNewReleases .OrderByDescending(o => o.Version) .ThenByDescending(o => o.Id) .ToList(); // Process new list and trim the releases used for comparison await ProcessReleases(sortedNewReleases, sortedMajorReleasesDictionary, repo, true); sortedMajorReleasesList = sortedMajorReleasesDictionary.Values.OrderByDescending(o => o.Version).ToList(); File.WriteAllText($"./{repo}.json", JsonSerializer.Serialize(sortedMajorReleasesList)); } else { Console.WriteLine("The current releases list is up to date with core.js\nExiting..."); } } else { // create a dictionary for major versions foreach (var release in sortedReleases) { if (!sortedMajorReleasesDictionary.ContainsKey($"{release.Version.Major}.{release.Version.Minor}")) { var majorRelease = new MajorRelease { Contributions = 0, Contributors = new (), Product = release.Product, Name = $"{release.Product} {release.Version.Major}.{release.Version.Minor}", Version = release.Version, Tag = $"v{release.Version.Major}.{release.Version.Minor}", ProcessedReleases = new () }; sortedMajorReleasesDictionary.Add($"{release.Version.Major}.{release.Version.Minor}", majorRelease); } } Console.WriteLine($"Processing all releases...\n{repo} - {sortedReleases.Count}"); await ProcessReleases(sortedReleases, sortedMajorReleasesDictionary, repo); sortedMajorReleasesList = sortedMajorReleasesDictionary.Values.ToList(); File.WriteAllText($"./{repo}.json", JsonSerializer.Serialize(sortedMajorReleasesList)); } } #nullable enable private static async Task ProcessReleases(List releases, Dictionary majorReleasesDict, string repo, bool isDiff = false) { // dotnet/core dotnetthanks.Release currentRelease; dotnetthanks.Release previousRelease; for (int i = 0; i < releases.Count; i++) { currentRelease = releases[i]; majorReleasesDict.TryGetValue($"{currentRelease.Version.Major}.{currentRelease.Version.Minor}", out var majorRelease); // Add the first release to the list of processed releases so diff does not pick it up if (i == releases.Count - 1) { if (!isDiff) { majorRelease?.ProcessedReleases.Add(currentRelease.Tag); } break; } previousRelease = GetPreviousRelease(releases, currentRelease, i + 1); if (previousRelease is null) { // Is this the first release? Console.WriteLine($"[INFO]: {currentRelease.Tag} is the first release in the series."); //Debugger.Break(); continue; } majorRelease?.ProcessedReleases.Add(currentRelease.Tag); Console.WriteLine($"Processing:[{i}] {repo} {previousRelease.Tag}..{currentRelease.Tag}"); // for each child repo get commits and count contribs foreach (var repoCurrentRelease in currentRelease.ChildRepos) { var repoPrevRelease = previousRelease.ChildRepos.FirstOrDefault(r => r.Owner == repoCurrentRelease.Owner && r.Repository == repoCurrentRelease.Repository); if (repoPrevRelease is null) { // This may happen Console.WriteLine($"[ERROR]: {repoCurrentRelease.Url} doesn't exist in {previousRelease.Tag}!"); continue; } Debug.WriteLine($"{repoCurrentRelease.Tag} : {repoCurrentRelease.Name}"); try { Console.WriteLine($"\tProcessing: {repoCurrentRelease.Name}: {repoPrevRelease.Tag}..{repoCurrentRelease.Tag}"); if (repoPrevRelease.Tag != repoCurrentRelease.Tag) { var releaseDiff = await LoadCommitsForReleasesAsync(repoPrevRelease.Tag, repoCurrentRelease.Tag, repoCurrentRelease.Owner, repoCurrentRelease.Repository); if (releaseDiff is null || releaseDiff.Count < 1) { //Debugger.Break(); continue; } if (majorRelease is not null) { majorRelease.Contributions += releaseDiff.Count; TallyCommits(majorRelease, repoCurrentRelease.Repository, releaseDiff); } } } catch (Exception ex) { Console.WriteLine(ex.Message); } } if (Environment.GetEnvironmentVariable("TEST") == "1") break; } } #nullable disable private static async Task CreatePullRequestFromFork(string forkname, string branch) { var basic = new Credentials(_token); var client = new GitHubClient(new ProductHeaderValue("dotnet-thanks")) { Credentials = basic }; NewPullRequest newPr = new("Update thanks data file", $"spboyer:{branch}", "master"); try { var pullRequest = await client.PullRequest.Create("dotnet", "website-resources", newPr); return pullRequest; } catch (Exception ex) { Console.WriteLine(ex.Message); } return null; } /// /// Find the previous release for the current release in the sorted collection of all releases. /// Take the immediate previous release it it has the same major.minor version (e.g. 5.0.0-RC1 for 5.0.0-RC2), /// or take the previous GA release (e.g. 3.0.0 for 5.0.0-preview2 not 3.1.10). /// /// The index of the in the list. /// The previous release, if found; otherwise , if the current release if the first release. private static dotnetthanks.Release GetPreviousRelease(List sortedReleases, dotnetthanks.Release currentRelease, int index) { if (currentRelease.Version.Major == sortedReleases[index].Version.Major && currentRelease.Version.Minor == sortedReleases[index].Version.Minor) return sortedReleases[index]; return sortedReleases.Skip(index).FirstOrDefault(r => currentRelease.Version > r.Version && r.IsGA); } private static void TallyCommits(dotnetthanks.MajorRelease majorRelease, string repoName, List commits) { // these the commits within the release foreach (var item in commits) { if (item.author != null) { var author = item.author; author.name = item.commit.author.name; if (string.IsNullOrEmpty(author.name)) author.name = "Unknown"; if (!exclusions.Contains(author.name.ToLower())) { // find if the author has been counted var person = majorRelease.Contributors.Find(p => p.Link == author.html_url); if (person == null) { person = new dotnetthanks.Contributor() { Name = author.name, Link = author.html_url, Avatar = author.avatar_url, Count = 1 }; person.Repos.Add(new RepoItem() { Name = repoName, Count = 1 }); majorRelease.Contributors.Add(person); } else { // found the author, does the repo exist as well? person.Count += 1; var repoItem = person.Repos.Find(r => r.Name == repoName); if (repoItem == null) { person.Repos.Add(new RepoItem() { Name = repoName, Count = 1 }); } else { repoItem.Count += 1; } } } } } } private static async Task> LoadReleasesAsync(string owner, string repo) { var results = await _ghclient.Repository.Release.GetAll(owner, repo); return results.Select(release => new dotnetthanks.Release { Name = release.Name, Tag = release.TagName, Id = release.Id, ChildRepos = ParseReleaseBody(release.Body), Contributors = new List() }); } private static List ParseReleaseBody(string body) { var results = new List(); var pattern = "\\[(.+)\\]\\(([^ ]+?)( \"(.+)\")?\\)"; var rg = new Regex(pattern, RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.Multiline); var match = rg.Match(body); while (match.Success) { var name = match.Groups[1]?.Value.Trim(); var url = match.Groups[2]?.Value.Trim(); if (url.Contains("/tag/")) { results.Add(new ChildRepo() { Name = name, Url = url }); } match = match.NextMatch(); } return results; } private static async Task> LoadCommitsForReleasesAsync(string fromRelease, string toRelease, string owner, string repo) { _client = new HttpClient(); _client.DefaultRequestHeaders.UserAgent.Add(new System.Net.Http.Headers.ProductInfoHeaderValue("dotnet-thanks", "1.0")); _client.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Token", _token); try { string url = $"https://api.github.com/repos/{owner}/{repo}/compare/{fromRelease}...{toRelease}"; var compare = await ExecuteWithRateLimitHandling(() => _client.GetStringAsync(url)); var compareDetails = JsonSerializer.Deserialize(compare); var remainingCommits = compareDetails.ahead_by; var page = 0; var releaseCommits = new List(remainingCommits); while (remainingCommits > 0) { url = $"https://api.github.com/repos/{owner}/{repo}/commits?sha={toRelease}&page={page}"; var commits = await ExecuteWithRateLimitHandling(() => _client.GetStringAsync(url)); var pageDetails = JsonSerializer.Deserialize>(commits); releaseCommits.AddRange(remainingCommits >= pageDetails.Count ? pageDetails : pageDetails.Take(remainingCommits)); remainingCommits -= pageDetails.Count; page++; } return releaseCommits; } catch (Exception ex) { Console.WriteLine(ex.Message); } return null; } private static List LoadCurrentCoreJson() { try { string fileName = "core.json"; string jsonString = File.ReadAllText(fileName); var corejson = JsonSerializer.Deserialize>(jsonString); return corejson; } catch (Exception ex) { Console.WriteLine(ex.Message); } return null; } private static async Task> LoadCurrentCoreJsonAsync() { _client = new HttpClient(); var url = "https://dotnetwebsitestorage.blob.core.windows.net/blob-assets/json/thanks/core.json"; try { var response = await _client.GetFromJsonAsync>(url); return response; } catch (Exception ex) { Console.WriteLine(ex.Message); } return null; } private static async Task ExecuteWithRateLimitHandling(Func> operation) { var remainingRetries = 3; Retry: try { return await operation(); } catch (Exception ex) when (remainingRetries > 0) { if (ex.Message.Contains("403")) { string url = "https://api.github.com/rate_limit"; var limit = await _client.GetStringAsync(url); var response = JsonSerializer.Deserialize(limit); var delay = new TimeSpan(response.rate.reset) .Add(TimeSpan.FromMinutes(10)); // Add some buffer var until = DateTime.Now.Add(delay); Console.WriteLine($"Rate limit exceeded. Waiting for {delay.TotalMinutes:N1} mins until {until}."); await Task.Delay(delay); remainingRetries--; goto Retry; } else { return await Task.FromException(ex); } } } } }