Apiview revision upgradability to new parser version test (#8882)

* Verify revision upgradability as a dry run task

src/dotnet/APIView/APIViewWeb/HostedServices/ReviewBackgroundHostedService.cs

@@ -21,6 +21,8 @@ namespace APIViewWeb.HostedServices
         private readonly HashSet<string> _upgradeDisabledLangs = new HashSet<string>();
         private readonly int _backgroundBatchProcessCount;
         private readonly TelemetryClient _telemetryClient;
+        private readonly bool _isUpgradeTestEnabled;
+        private readonly string _packageNameFilterForUpgrade;
 
         public ReviewBackgroundHostedService(
             IReviewManager reviewManager, IAPIRevisionsManager apiRevisionManager,
@@ -36,6 +38,17 @@ namespace APIViewWeb.HostedServices
                 _isDisabled = taskDisabled;
             }
 
+            if (bool.TryParse(configuration["ReviewUpgradabilityTestEnabled"], out bool upgradeTestEnabled))
+            {
+                _isUpgradeTestEnabled = upgradeTestEnabled;
+            }
+
+            var packageNameFilterForUpgrade = configuration["PackageNameFilterForReviewUpgrade"];
+            if (!string.IsNullOrEmpty(packageNameFilterForUpgrade))
+            {
+                _packageNameFilterForUpgrade = packageNameFilterForUpgrade;
+            }
+
             var gracePeriod = configuration["ArchiveReviewGracePeriodInMonths"];
             if (String.IsNullOrEmpty(gracePeriod) || !int.TryParse(gracePeriod, out _autoArchiveInactiveGracePeriodMonths))
             {
@@ -61,7 +74,7 @@ namespace APIViewWeb.HostedServices
             {
                 try
                 {
-                    await _reviewManager.UpdateReviewsInBackground(_upgradeDisabledLangs, _backgroundBatchProcessCount);
+                    await _reviewManager.UpdateReviewsInBackground(_upgradeDisabledLangs, _backgroundBatchProcessCount, _isUpgradeTestEnabled, _packageNameFilterForUpgrade);
                     await ArchiveInactiveAPIReviews(stoppingToken, _autoArchiveInactiveGracePeriodMonths);
                 }
                 catch (Exception ex)

src/dotnet/APIView/APIViewWeb/Managers/APIRevisionsManager.cs

@@ -711,8 +711,9 @@ namespace APIViewWeb.Managers
         /// </summary>
         /// <param name="revision"></param>
         /// <param name="languageService"></param>
+        /// <param name="verifyUpgradabilityOnly"> </param>
         /// <returns></returns>
-        public async Task UpdateAPIRevisionAsync(APIRevisionListItemModel revision, LanguageService languageService)
+        public async Task UpdateAPIRevisionAsync(APIRevisionListItemModel revision, LanguageService languageService, bool verifyUpgradabilityOnly)
         {
             foreach (var file in revision.Files)
             {
@@ -728,20 +729,31 @@ namespace APIViewWeb.Managers
                     // This is causing issue when updating review using latest parser since it expects Name field as file name
                     // We have added a new property FileName which is only set for new reviews
                     // All older reviews needs to be handled by checking review name field
-                    var fileName = file.FileName ?? file.Name;
+                    var fileName = file.FileName ?? file.FileId;
                     var codeFile = await languageService.GetCodeFileAsync(fileName, fileOriginal, false);
-                    await _codeFileRepository.UpsertCodeFileAsync(revision.Id, file.FileId, codeFile);
-                    // update only version string
-                    file.VersionString = codeFile.VersionString;
-                    if (codeFile.ReviewLines.Count > 0) {
-                        file.ParserStyle = ParserStyle.Tree;
+                    if (!verifyUpgradabilityOnly)
+                    {
+                        await _codeFileRepository.UpsertCodeFileAsync(revision.Id, file.FileId, codeFile);
+                        // update only version string
+                        file.VersionString = codeFile.VersionString;
+                        if (codeFile.ReviewLines.Count > 0)
+                        {
+                            file.ParserStyle = ParserStyle.Tree;
+                        }
+                        await _apiRevisionsRepository.UpsertAPIRevisionAsync(revision);
+                        _telemetryClient.TrackTrace($"Successfully Updated {revision.Language} revision with id {revision.Id}");
+                    }
+                    else
+                    {
+                        _telemetryClient.TrackTrace($"Revision with id {revision.Id} for package {codeFile.PackageName} can be upgraded using new parser version.");
                     }
-                    await _apiRevisionsRepository.UpsertAPIRevisionAsync(revision);
-                    _telemetryClient.TrackTrace($"Successfully Updated {revision.Language} revision with id {revision.Id}");
                 }
                 catch (Exception ex)
                 {
-                    _telemetryClient.TrackTrace($"Failed to update {revision.Language} revision with id {revision.Id}");
+                    if (!verifyUpgradabilityOnly)
+                        _telemetryClient.TrackTrace($"Failed to update {revision.Language} revision with id {revision.Id}");
+                    else
+                        _telemetryClient.TrackTrace($"Revision with id {revision.Id} for package {file.PackageName} cannot be upgraded using new parser version.");
                     _telemetryClient.TrackException(ex);
                 }
             }

src/dotnet/APIView/APIViewWeb/Managers/Interfaces/IAPIRevisionsManager.cs

@@ -38,7 +38,7 @@ namespace APIViewWeb.Managers.Interfaces
         public TreeNode<InlineDiffLine<CodeLine>> ComputeSectionDiff(TreeNode<CodeLine> before, TreeNode<CodeLine> after, RenderedCodeFile beforeFile, RenderedCodeFile afterFile);
         public Task<APIRevisionListItemModel> CreateAPIRevisionAsync(string userName, string reviewId, APIRevisionType apiRevisionType, string label,
             MemoryStream memoryStream, CodeFile codeFile, string originalName = null, int? prNumber = null);
-        public Task UpdateAPIRevisionAsync(APIRevisionListItemModel revision, LanguageService languageService);
+        public Task UpdateAPIRevisionAsync(APIRevisionListItemModel revision, LanguageService languageService, bool verifyUpgradabilityOnly);
         public Task UpdateAPIRevisionAsync(APIRevisionListItemModel revision);
         public Task AutoArchiveAPIRevisions(int archiveAfterMonths);
         public Task AssignReviewersToAPIRevisionAsync(ClaimsPrincipal User, string apiRevisionId, HashSet<string> reviewers);

src/dotnet/APIView/APIViewWeb/Managers/Interfaces/IReviewManager.cs

@@ -25,6 +25,6 @@ namespace APIViewWeb.Managers
         public Task<ReviewListItemModel> ToggleReviewApprovalAsync(ClaimsPrincipal user, string id, string revisionId, string notes="");
         public Task ApproveReviewAsync(ClaimsPrincipal user, string reviewId, string notes = "");
         public Task<int> GenerateAIReview(string reviewId, string revisionId);
-        public Task UpdateReviewsInBackground(HashSet<string> updateDisabledLanguages, int backgroundBatchProcessCount);
+        public Task UpdateReviewsInBackground(HashSet<string> updateDisabledLanguages, int backgroundBatchProcessCount, bool verifyUpgradabilityOnly, string packageNameFilterForUpgrade);
     }
 }

src/dotnet/APIView/APIViewWeb/Managers/ReviewManager.cs

@@ -397,14 +397,24 @@ namespace APIViewWeb.Managers
         /// </summary>
         /// <param name="updateDisabledLanguages"></param>
         /// <param name="backgroundBatchProcessCount"></param>
+        /// <param name="verifyUpgradabilityOnly"></param>
+        /// <param name="packageNameFilterForUpgrade"></param>
         /// <returns></returns>
-        public async Task UpdateReviewsInBackground(HashSet<string> updateDisabledLanguages, int backgroundBatchProcessCount)
+        public async Task UpdateReviewsInBackground(HashSet<string> updateDisabledLanguages, int backgroundBatchProcessCount, bool verifyUpgradabilityOnly, string packageNameFilterForUpgrade = "")
         {
+            // verifyUpgradabilityOnly is set when we need to run the upgrade in read only mode to recreate code files
+            // But review code file or metadata in the DB will not be updated
+            // This flag is set only to make sure revisions are upgradable to the latest version of the parser
+            if(verifyUpgradabilityOnly)
+            {
+                _telemetryClient.TrackTrace("Running background task to verify review upgradability only.");
+            }
+
             foreach (var language in LanguageService.SupportedLanguages)
             {
                 if (updateDisabledLanguages.Contains(language))
                 {
-                    _telemetryClient.TrackTrace("Background task to update API review at startup is disabled for langauge " + language);
+                    _telemetryClient.TrackTrace("Background task to update API review at startup is disabled for language " + language);
                     continue;
                 }
                 var languageService = LanguageServiceHelpers.GetLanguageService(language, _languageServices);
@@ -414,16 +424,24 @@ namespace APIViewWeb.Managers
                 // If review is updated using devops pipeline then batch process update review requests
                 if (languageService.IsReviewGenByPipeline)
                 {
-                    await UpdateReviewsUsingPipeline(language, languageService, backgroundBatchProcessCount);
+                    _telemetryClient.TrackTrace($"{language} uses sandboxing pipeline to upgrade API revisions. Upgrade eligibility test is not yet supported for {language}.");
+                    // Do not run sandboxing based upgrade during verify upgradability only mode
+                    // This requires some changes in the pipeline to support this mode
+                    if (!verifyUpgradabilityOnly)
+                    {
+                        await UpdateReviewsUsingPipeline(language, languageService, backgroundBatchProcessCount);
+                    }                    
                 }
                 else
                 {
                     var reviews = await _reviewsRepository.GetReviewsAsync(language: language, isClosed: false);
-
+                    if (!string.IsNullOrEmpty(packageNameFilterForUpgrade))
+                    {
+                        reviews = reviews.Where(r => r.PackageName == packageNameFilterForUpgrade);
+                    }
                     foreach (var review in reviews)
                     {
                         var revisions = await _apiRevisionsManager.GetAPIRevisionsAsync(review.Id);
-
                         foreach (var revision in revisions)
                         {
                             if (
@@ -434,8 +452,8 @@ namespace APIViewWeb.Managers
                                 var operation = _telemetryClient.StartOperation(requestTelemetry);
                                 try
                                 {
-                                    await Task.Delay(500);
-                                    await _apiRevisionsManager.UpdateAPIRevisionAsync(revision, languageService);
+                                    await Task.Delay(100);
+                                    await _apiRevisionsManager.UpdateAPIRevisionAsync(revision, languageService, verifyUpgradabilityOnly);
                                 }
                                 catch (Exception e)
                                 {