Rekha/detector deploy (#730)

src/Diagnostics.Logger/ETWProvider/DiagnosticsETWProvider.cs

@@ -834,5 +834,21 @@ namespace Diagnostics.Logger
         }
 
         #endregion
+
+        #region Detector Deployment Events (ID: 5700 - 5709)
+
+        [Event(5700, Level = EventLevel.Informational, Channel = EventChannel.Admin, Message = ETWMessageTemplates.LogDeploymentOperationMessage)]
+        public void LogDeploymentOperationMessage(string RequestId, string DeploymentId, string Message, string DiagEnvironment = null, string DiagWebsiteHostName = null)
+        {
+            WriteDiagnosticsEvent(
+                5700,
+                RequestId,
+                DeploymentId,
+                Message,
+                DiagEnvironment,
+                DiagWebsiteHostName);
+        }
+
+        #endregion
     }
 }

src/Diagnostics.Logger/ETWProvider/ETWMessageTemplates.cs

@@ -36,6 +36,7 @@ namespace Diagnostics.Logger
         public const string LogRuntimeHostSupportTopicAscInsight = "ASC Insight Detail for Detector";
         public const string LogRuntimeMessage = "Runtime Log from Detector";
         public const string LogDevOpsApiException = "DevOps API Exception";
+        public const string LogDeploymentOperationMessage = "Detector deployment Message : {0}";
 
         #endregion Runtime Host Event Message Templates
 

src/Diagnostics.ModelsAndUtils/Models/Storage/DeploymentParameters.cs

@@ -0,0 +1,66 @@
+using System.Collections.Generic;
+
+namespace Diagnostics.ModelsAndUtils.Models.Storage
+{
+    /// <summary>
+    /// Deployment Parameters used by client/service caller to deploy detectors.
+    /// </summary>
+    public class DeploymentParameters
+    {
+        /// <summary>
+        /// Single commit id to deploy. Not applicable if <see cref="FromCommitId"/> and <see cref="ToCommitId"/> is given.
+        /// </summary>
+        public string CommitId;
+
+        /// <summary>
+        /// Start commit to deploy. Not applicable if <see cref="CommitId"/> is given.
+        /// </summary>
+        public string FromCommitId;
+
+        /// <summary>
+        /// End commit to deploy. Not applicable if <see cref="CommitId"/> is given.
+        /// </summary>
+        public string ToCommitId;
+
+        /// <summary>
+        /// If provided, includes detectors modified after this date. Cannot be combined with <see cref="FromCommitId"/> and <see cref="ToCommitId"/>.
+        /// </summary>
+        public string StartDate;
+
+        /// <summary>
+        /// If provided, includes detectors modified before this date. Cannot be combined with <see cref="FromCommitId"/> and <see cref="ToCommitId"/>.
+        /// </summary>
+        public string EndDate;
+
+        /// <summary>
+        /// Resource type of the caller. eg. Microsoft.Web/sites.
+        /// </summary>
+        public string ResourceType;
+    }
+
+    /// <summary>
+    /// Deployment response sent back to the caller;
+    /// </summary>
+    public class DeploymentResponse
+    {
+        /// <summary>
+        /// List of detectors that got updated/added/edited;
+        /// </summary>
+        public List<string> DeployedDetectors;
+
+        /// <summary>
+        /// List of detectors that failed deployment along with the reason of failure;
+        /// </summary>
+        public Dictionary<string, string> FailedDetectors;
+
+        /// <summary>
+        /// List of detectors that were marked for deletion;
+        /// </summary>
+        public List<string> DeletedDetectors;
+
+        /// <summary>
+        /// Unique Guid to track the deployment;
+        /// </summary>
+        public string DeploymentGuid;
+    }
+}

src/Diagnostics.ModelsAndUtils/Models/Storage/DevopsFileChange.cs

@@ -0,0 +1,40 @@
+namespace Diagnostics.ModelsAndUtils.Models.Storage
+{
+    public class DevopsFileChange
+    {
+        /// <summary>
+        /// Commit id 
+        /// </summary>
+        public string CommitId { get; set; }
+
+        /// <summary>
+        /// Detector/gist id.
+        /// </summary>
+        public string Id { get; set; }
+
+        /// <summary>
+        /// Content of .csx file
+        /// </summary>
+        public string Content { get; set; }
+
+        /// <summary>
+        /// Path of the detector csx file
+        /// </summary>
+        public string Path { get; set; }
+
+        /// <summary>
+        /// Content of package.json file
+        /// </summary>
+        public string PackageConfig { get; set; }
+
+        /// <summary>
+        /// Content of metadata.json file
+        /// </summary>
+        public string Metadata { get; set; }
+
+        /// <summary>
+        /// Checks if the detector should be marked as disabled.
+        /// </summary>
+        public bool MarkAsDisabled { get; set; }
+    }
+}

src/Diagnostics.RuntimeHost/Controllers/ApiManagement/ApiManagementController.cs

@@ -8,6 +8,7 @@ using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.Authorization;
 using Newtonsoft.Json;
 using Microsoft.CSharp.RuntimeBinder;
+using Microsoft.Extensions.Configuration;
 
 namespace Diagnostics.RuntimeHost.Controllers
 {
@@ -16,8 +17,8 @@ namespace Diagnostics.RuntimeHost.Controllers
     [Route(UriElements.ApiManagementServiceResource)]
     public sealed class ApiManagementController : DiagnosticControllerBase<ApiManagementService>
     {
-        public ApiManagementController(IServiceProvider services, IRuntimeContext<ApiManagementService> runtimeContext)
-            : base(services, runtimeContext)
+        public ApiManagementController(IServiceProvider services, IRuntimeContext<ApiManagementService> runtimeContext, IConfiguration config)
+            : base(services, runtimeContext, config)
         {
         }
 

src/Diagnostics.RuntimeHost/Controllers/AppServiceCertificate/AppServiceCertificateController.cs

@@ -8,6 +8,7 @@ using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.Authorization;
 using Newtonsoft.Json;
 using Microsoft.CSharp.RuntimeBinder;
+using Microsoft.Extensions.Configuration;
 
 namespace Diagnostics.RuntimeHost.Controllers
 {
@@ -16,8 +17,8 @@ namespace Diagnostics.RuntimeHost.Controllers
     [Route(UriElements.AppServiceCertResource)]
     public sealed class AppServiceCertificateController : DiagnosticControllerBase<AppServiceCertificate>
     {
-        public AppServiceCertificateController(IServiceProvider services, IRuntimeContext<AppServiceCertificate> runtimeContext)
-            : base(services, runtimeContext)
+        public AppServiceCertificateController(IServiceProvider services, IRuntimeContext<AppServiceCertificate> runtimeContext, IConfiguration config)
+            : base(services, runtimeContext, config)
         {
         }
 

src/Diagnostics.RuntimeHost/Controllers/AppServiceDomain/AppServiceDomainController.cs

@@ -8,6 +8,7 @@ using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.Authorization;
 using Newtonsoft.Json;
 using Microsoft.CSharp.RuntimeBinder;
+using Microsoft.Extensions.Configuration;
 
 namespace Diagnostics.RuntimeHost.Controllers
 {
@@ -16,8 +17,8 @@ namespace Diagnostics.RuntimeHost.Controllers
     [Route(UriElements.AppServiceDomainResource)]
     public sealed class AppServiceDomainController : DiagnosticControllerBase<AppServiceDomain>
     {
-        public AppServiceDomainController(IServiceProvider services, IRuntimeContext<AppServiceDomain> runtimeContext)
-            : base(services, runtimeContext)
+        public AppServiceDomainController(IServiceProvider services, IRuntimeContext<AppServiceDomain> runtimeContext, IConfiguration config)
+            : base(services, runtimeContext, config)
         {
         }
 

src/Diagnostics.RuntimeHost/Controllers/ArmResource/ArmResourceController.cs

@@ -9,6 +9,7 @@ using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.Authorization;
 using Newtonsoft.Json;
 using Microsoft.CSharp.RuntimeBinder;
+using Microsoft.Extensions.Configuration;
 
 namespace Diagnostics.RuntimeHost.Controllers
 {
@@ -20,8 +21,8 @@ namespace Diagnostics.RuntimeHost.Controllers
     [Route(UriElements.ArmResource)]
     public class ArmResourceController : DiagnosticControllerBase<ArmResource>
     {
-        public ArmResourceController(IServiceProvider services, IRuntimeContext<ArmResource> runtimeContext)
-            : base(services, runtimeContext)
+        public ArmResourceController(IServiceProvider services, IRuntimeContext<ArmResource> runtimeContext, IConfiguration config)
+            : base(services, runtimeContext, config)
         {
         }
 

src/Diagnostics.RuntimeHost/Controllers/AzureKubernetesService/AzureKubernetesServiceController.cs

@@ -8,6 +8,7 @@ using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.Authorization;
 using Newtonsoft.Json;
 using Microsoft.CSharp.RuntimeBinder;
+using Microsoft.Extensions.Configuration;
 
 namespace Diagnostics.RuntimeHost.Controllers
 {
@@ -16,8 +17,8 @@ namespace Diagnostics.RuntimeHost.Controllers
     [Route(UriElements.AzureKubernetesServiceResource)]
     public class AzureKubernetesServiceController : DiagnosticControllerBase<AzureKubernetesService>
     {
-        public AzureKubernetesServiceController(IServiceProvider services, IRuntimeContext<AzureKubernetesService> runtimeContext)
-            : base(services, runtimeContext)
+        public AzureKubernetesServiceController(IServiceProvider services, IRuntimeContext<AzureKubernetesService> runtimeContext, IConfiguration config)
+            : base(services, runtimeContext, config)
         {
         }
 

src/Diagnostics.RuntimeHost/Controllers/DeploymentController.cs

@@ -0,0 +1,226 @@
+using Diagnostics.Logger;
+using Diagnostics.ModelsAndUtils.Models;
+using Diagnostics.ModelsAndUtils.Models.Storage;
+using Diagnostics.RuntimeHost.Services;
+using Diagnostics.RuntimeHost.Services.CacheService;
+using Diagnostics.RuntimeHost.Services.StorageService;
+using Diagnostics.RuntimeHost.Services.DevOpsClient;
+using Diagnostics.RuntimeHost.Utilities;
+using Diagnostics.Scripts;
+using Diagnostics.Scripts.CompilationService.Utilities;
+using Diagnostics.Scripts.Models;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Configuration;
+using Newtonsoft.Json;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Reflection;
+using System.Threading.Tasks;
+
+namespace Diagnostics.RuntimeHost.Controllers
+{
+    [Authorize]
+    [Produces("application/json")]
+    [Route("api/deploy")]
+    public class DeploymentController : Controller
+    {
+
+        private IRepoClient devopsClient;
+        protected ICompilerHostClient _compilerHostClient;
+        protected IStorageService storageService;
+        private IInvokerCacheService detectorCache;
+
+        public DeploymentController(IStorageService storage, IRepoClient repo, ICompilerHostClient compilerHost, IInvokerCacheService invokerCache)
+        {
+            this.storageService = storage;
+            this.devopsClient = repo;
+            this._compilerHostClient = compilerHost;
+            this.detectorCache = invokerCache;
+        }
+
+        [HttpPost]
+        public async Task<IActionResult> Post([FromBody] DeploymentParameters deploymentParameters)
+        {
+           string validationError = ValidateDeploymentParameters(deploymentParameters);
+           if(!string.IsNullOrEmpty(validationError))
+            {
+                return BadRequest(validationError);
+            }
+                     
+            DeploymentResponse response = new DeploymentResponse();
+            response.DeploymentGuid = Guid.NewGuid().ToString();
+            response.DeployedDetectors = new List<string>();
+            response.FailedDetectors = new Dictionary<string, string>();
+           
+            var commitId = deploymentParameters.CommitId;
+            var timeTakenStopWatch = new Stopwatch();
+            timeTakenStopWatch.Start();
+
+            string requestId = HttpContext.Request.Headers[HeaderConstants.RequestIdHeaderName].FirstOrDefault();
+            DiagnosticsETWProvider.Instance.LogDeploymentOperationMessage(requestId, response.DeploymentGuid, $"Starting deployment operation");
+            //  Get files to compile 
+            var filesToCompile = string.IsNullOrWhiteSpace(commitId) ? 
+                await this.devopsClient.GetFilesBetweenCommits(deploymentParameters)
+              : await this.devopsClient.GetFilesInCommit(commitId);
+
+            QueryResponse<DiagnosticApiResponse> queryRes = new QueryResponse<DiagnosticApiResponse>
+            {
+                InvocationOutput = new DiagnosticApiResponse()
+            };
+
+            IDictionary<string, string> references = new Dictionary<string, string>();
+
+
+            DiagnosticsETWProvider.Instance.LogDeploymentOperationMessage(requestId, response.DeploymentGuid, $"{filesToCompile.Count} to compile");
+          
+            // Batch update files to be deleted.
+
+            var filesTobeDeleted = filesToCompile.Where(file => file.MarkAsDisabled);
+            List<DiagEntity> batchDetectors = new List<DiagEntity>();
+            List<DiagEntity> batchGists = new List<DiagEntity>();
+            foreach (var file in filesTobeDeleted)
+            {
+                var diagEntity = JsonConvert.DeserializeObject<DiagEntity>(file.PackageConfig);
+                diagEntity.GithubLastModified = DateTime.UtcNow;
+                diagEntity.PartitionKey = diagEntity.EntityType;
+                diagEntity.RowKey = diagEntity.DetectorId;
+                var detectorId = diagEntity.DetectorId;
+                DiagnosticsETWProvider.Instance.LogDeploymentOperationMessage(requestId, response.DeploymentGuid, $"Making {detectorId} as disabled");
+                diagEntity.IsDisabled = true;
+                if (response.DeletedDetectors == null)
+                {
+                    response.DeletedDetectors = new List<string>();
+                }
+                response.DeletedDetectors.Add(diagEntity.RowKey);
+                if(diagEntity.PartitionKey.Equals("Detector"))
+                {
+                    batchDetectors.Add(diagEntity);
+                } else if (diagEntity.PartitionKey.Equals("Gist"))
+                {
+                    batchGists.Add(diagEntity);
+                }
+            }
+            // Batch update must share the same partition key, so two separate tasks.
+            Task batchDeleteDetectors = batchDetectors.Count > 0 ? storageService.LoadBatchDataToTable(batchDetectors) : Task.CompletedTask;
+            Task batchDeleteGists = batchGists.Count > 0 ? storageService.LoadBatchDataToTable(batchGists) : Task.CompletedTask;
+            await Task.WhenAll(new Task[] { batchDeleteDetectors, batchDeleteGists });
+
+            foreach ( var file in filesToCompile.Where(file => !file.MarkAsDisabled))
+            {
+                // For each of the files to compile:
+                // 1. Create the diag entity object.
+                var diagEntity = JsonConvert.DeserializeObject<DiagEntity>(file.PackageConfig);
+                diagEntity.GithubLastModified = DateTime.UtcNow;
+                diagEntity.PartitionKey = diagEntity.EntityType;
+                diagEntity.RowKey = diagEntity.DetectorId;
+                var detectorId = diagEntity.DetectorId;
+          
+                List<string> gistReferences = DetectorParser.GetLoadDirectiveNames(file.Content);
+                
+                // Get the latest version of gist from the repo.
+                foreach(string gist in gistReferences)
+                {                   
+                    var gistContent = await devopsClient.GetFileContentAsync($"{gist}/{gist}.csx", deploymentParameters.ResourceType, HttpContext.Request.Headers[HeaderConstants.RequestIdHeaderName]);
+                    references.Add(gist, gistContent.ToString());                                                  
+                }
+
+                // Otherwise, compile the detector to generate dll.
+                queryRes.CompilationOutput = await _compilerHostClient.GetCompilationResponse(file.Content, diagEntity.EntityType, references);
+
+                //  If compilation success, save dll to storage container.
+                if (queryRes.CompilationOutput.CompilationSucceeded)
+                {
+                    var blobName = $"{detectorId.ToLower()}/{detectorId.ToLower()}.dll";
+                    DiagnosticsETWProvider.Instance.LogDeploymentOperationMessage(requestId, response.DeploymentGuid, $"Saving {blobName} to storage container");
+                    //  Save blob to storage account
+                    var etag = await storageService.LoadBlobToContainer(blobName, queryRes.CompilationOutput.AssemblyBytes);
+                    if (string.IsNullOrWhiteSpace(etag))
+                    {
+                        throw new Exception($"Could not save changes {blobName} to storage");
+                    }
+                    response.DeployedDetectors.Add(detectorId);
+
+                    // Save entity to table
+                    diagEntity.Metadata = file.Metadata;
+                    diagEntity.GitHubSha = file.CommitId;
+                    byte[] asmData = Convert.FromBase64String(queryRes.CompilationOutput.AssemblyBytes);
+                    byte[] pdbData = Convert.FromBase64String(queryRes.CompilationOutput.PdbBytes);
+                    diagEntity = DiagEntityHelper.PrepareEntityForLoad(asmData, file.Content, diagEntity);
+
+                    DiagnosticsETWProvider.Instance.LogDeploymentOperationMessage(requestId, response.DeploymentGuid, $"Saving {diagEntity.RowKey} to storage table");
+                    await storageService.LoadDataToTable(diagEntity);
+
+                    DiagnosticsETWProvider.Instance.LogDeploymentOperationMessage(requestId, response.DeploymentGuid, $"Updating invoker cache for {diagEntity.RowKey}");
+                    // update invoker cache for detector. For gists, we dont need to update invoker cache as we pull latest code each time.
+                    if (diagEntity.PartitionKey.Equals("Detector"))
+                    {
+                        Assembly tempAsm = Assembly.Load(asmData, pdbData);
+                        EntityType entityType = EntityType.Detector;
+                        EntityMetadata metaData = new EntityMetadata(file.Content, entityType, null);
+                        var newInvoker = new EntityInvoker(metaData);
+                        newInvoker.InitializeEntryPoint(tempAsm);
+                        detectorCache.AddOrUpdate(detectorId, newInvoker);
+                    }
+
+                } else
+                {
+                    DiagnosticsETWProvider.Instance.LogDeploymentOperationMessage(requestId, response.DeploymentGuid, $"Compilation failed for {detectorId}, Reason: {queryRes.CompilationOutput.CompilationTraces.FirstOrDefault()} ");
+                    // If compilation fails, add failure reason to the response
+                    response.FailedDetectors.Add(detectorId, queryRes.CompilationOutput.CompilationTraces.FirstOrDefault());
+                }
+            }
+
+            timeTakenStopWatch.Stop();
+            DiagnosticsETWProvider.Instance.LogDeploymentOperationMessage(requestId, response.DeploymentGuid, $"Deployment completed for {response.DeploymentGuid}, time elapsed {timeTakenStopWatch.ElapsedMilliseconds}");
+            return Ok(response);
+        }     
+   
+        private string ValidateDeploymentParameters(DeploymentParameters deploymentParameters)
+        {
+            string errorMessage = string.Empty;
+
+            // If all required parameters are empty, reject the request.
+            if (string.IsNullOrWhiteSpace(deploymentParameters.CommitId) && string.IsNullOrWhiteSpace(deploymentParameters.FromCommitId)
+                && string.IsNullOrWhiteSpace(deploymentParameters.ToCommitId) && string.IsNullOrWhiteSpace(deploymentParameters.StartDate)
+                && string.IsNullOrWhiteSpace(deploymentParameters.EndDate))
+            {
+                errorMessage =  "The given deployment parameters are invalid. Please provide commit id/commit range/date range";
+                return errorMessage;
+            }
+
+            if(string.IsNullOrWhiteSpace(deploymentParameters.CommitId))
+            {
+                // validate other parameters
+                // Caller has not provided any of the parameters, throw validation error.
+                if (string.IsNullOrWhiteSpace(deploymentParameters.StartDate) && string.IsNullOrWhiteSpace(deploymentParameters.EndDate)
+                   && string.IsNullOrWhiteSpace(deploymentParameters.FromCommitId) && string.IsNullOrWhiteSpace(deploymentParameters.ToCommitId))
+                {
+                    errorMessage = "The given deployment parameters are invalid. Please provide both StartDate & EndDate or FromCommit & ToCommit";
+                }
+                // If both start date & End date are not given, throw validation error.
+                if ((string.IsNullOrWhiteSpace(deploymentParameters.StartDate) && !string.IsNullOrWhiteSpace(deploymentParameters.EndDate))
+                    || (string.IsNullOrWhiteSpace(deploymentParameters.EndDate) && !string.IsNullOrWhiteSpace(deploymentParameters.StartDate)))
+                {
+                    errorMessage = "The given deployment parameters are invalid. Please provide both StartDate & EndDate";
+                }
+                // If both start and end date are present but start date is greater than end date, throw validation error.
+                if (!string.IsNullOrWhiteSpace(deploymentParameters.StartDate) && !string.IsNullOrWhiteSpace(deploymentParameters.EndDate)
+                    && ((DateTime.Parse(deploymentParameters.StartDate) > DateTime.Parse(deploymentParameters.EndDate))))
+                {
+                    errorMessage = "Start date cannot be greater than end date";
+                }
+
+                // If both FromCommit & ToCommit are not given, throw validation error.
+                if ((string.IsNullOrWhiteSpace(deploymentParameters.FromCommitId) && !string.IsNullOrWhiteSpace(deploymentParameters.ToCommitId))
+                    || (string.IsNullOrWhiteSpace(deploymentParameters.ToCommitId) && !string.IsNullOrWhiteSpace(deploymentParameters.FromCommitId)))
+                {
+                    errorMessage = "The given deployment parameters are invalid. Please provide both FromCommitId & ToCommitId";
+                } 
+            }
+            return errorMessage;
+        }
+    }
+}

src/Diagnostics.RuntimeHost/Controllers/DiagnosticControllerBase.cs

@@ -9,6 +9,7 @@ using System.Web;
 using Diagnostics.DataProviders;
 using Diagnostics.DataProviders.Exceptions;
 using Diagnostics.DataProviders.Interfaces;
+using Microsoft.Extensions.Configuration;
 using Diagnostics.Logger;
 using Diagnostics.ModelsAndUtils.Attributes;
 using Diagnostics.ModelsAndUtils.Models;
@@ -19,6 +20,7 @@ using Diagnostics.ModelsAndUtils.Utilities;
 using Diagnostics.RuntimeHost.Models;
 using Diagnostics.RuntimeHost.Models.Exceptions;
 using Diagnostics.RuntimeHost.Services;
+using Diagnostics.RuntimeHost.Services.DevOpsClient;
 using Diagnostics.RuntimeHost.Services.CacheService;
 using Diagnostics.RuntimeHost.Services.CacheService.Interfaces;
 using Diagnostics.RuntimeHost.Services.DiagnosticsTranslator;
@@ -29,11 +31,13 @@ using Diagnostics.RuntimeHost.Utilities;
 using Diagnostics.Scripts;
 using Diagnostics.Scripts.Models;
 using Diagnostics.Scripts.Utilities;
+using Diagnostics.Scripts.CompilationService.Utilities;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Primitives;
 using Newtonsoft.Json;
+using Diagnostics.Logger;
 
 namespace Diagnostics.RuntimeHost.Controllers
 {
@@ -52,12 +56,15 @@ namespace Diagnostics.RuntimeHost.Controllers
         protected ISupportTopicService _supportTopicService;
         protected IKustoMappingsCacheService _kustoMappingCacheService;
         protected IRuntimeLoggerProvider _loggerProvider;
+        protected IRepoClient devopsClient;
         private InternalAPIHelper _internalApiHelper;
         private IDiagEntityTableCacheService tableCacheService;
         private ISourceWatcher storageWatcher;
         private IDiagnosticTranslatorService _diagnosticTranslator;
+        private bool loadGistFromRepo;
 
-        public DiagnosticControllerBase(IServiceProvider services, IRuntimeContext<TResource> runtimeContext)
+
+        public DiagnosticControllerBase(IServiceProvider services, IRuntimeContext<TResource> runtimeContext, IConfiguration config)
         {
             this._compilerHostClient = (ICompilerHostClient)services.GetService(typeof(ICompilerHostClient));
             this._sourceWatcherService = (ISourceWatcherService)services.GetService(typeof(ISourceWatcherService));
@@ -79,6 +86,14 @@ namespace Diagnostics.RuntimeHost.Controllers
             }
             this._internalApiHelper = new InternalAPIHelper();
             _runtimeContext = runtimeContext;
+            if(bool.TryParse(config["LoadGistFromRepo"], out bool retVal))
+            {
+                loadGistFromRepo = retVal;
+            } else
+            {
+                loadGistFromRepo = false;
+            }
+            devopsClient = (IRepoClient)services.GetService(typeof(IRepoClient));
         }
 
         #region API Response Methods
@@ -273,14 +288,29 @@ namespace Diagnostics.RuntimeHost.Controllers
 
             var dataProviders = new DataProviders.DataProviders((DataProviderContext)HttpContext.Items[HostConstants.DataProviderContextKey]);
 
-            foreach (var p in _gistCache.GetAllReferences(runtimeContext))
+            if(loadGistFromRepo)
             {
-                if (!jsonBody.References.ContainsKey(p.Key))
+                List<string> gistReferences = DetectorParser.GetLoadDirectiveNames(jsonBody.Script);
+                foreach (string gist in gistReferences)
                 {
-                    // Add latest version to references
-                    jsonBody.References.Add(p);
+                    if(!jsonBody.References.ContainsKey(gist))
+                    {
+                        object gistContent = await devopsClient.GetFileContentAsync($"{gist}/{gist}.csx", resource.ResourceUri, HttpContext.Request.Headers[HeaderConstants.RequestIdHeaderName]);
+                        jsonBody.References.Add(gist, gistContent.ToString());
+                    }                   
+                }
+            } else
+            {
+                foreach (var p in _gistCache.GetAllReferences(runtimeContext))
+                {
+                    if (!jsonBody.References.ContainsKey(p.Key))
+                    {
+                        // Add latest version to references
+                        jsonBody.References.Add(p);
+                    }
                 }
             }
+           
 
             if (!Enum.TryParse(jsonBody.EntityType, true, out EntityType entityType))
             {
@@ -1025,6 +1055,7 @@ namespace Diagnostics.RuntimeHost.Controllers
 
             if (tableCacheService.IsStorageAsSourceEnabled())
             {
+                
                 var allDetectorsFromStorage = await tableCacheService.GetEntityListByType<TResource>(context);
                 if (allDetectorsFromStorage.Count == 0)
                 {

src/Diagnostics.RuntimeHost/Controllers/HostingEnvironment/HostingEnvironmentController.cs

@@ -11,6 +11,7 @@ using Microsoft.AspNetCore.Authorization;
 using Newtonsoft.Json;
 using Microsoft.CSharp.RuntimeBinder;
 using Diagnostics.ModelsAndUtils.Utilities;
+using Microsoft.Extensions.Configuration;
 
 namespace Diagnostics.RuntimeHost.Controllers
 {
@@ -19,8 +20,8 @@ namespace Diagnostics.RuntimeHost.Controllers
     [Route(UriElements.HostingEnvironmentResource)]
     public sealed class HostingEnvironmentController : DiagnosticControllerBase<HostingEnvironment>
     {
-        public HostingEnvironmentController(IServiceProvider services, IRuntimeContext<HostingEnvironment> runtimeContext)
-            : base(services, runtimeContext)
+        public HostingEnvironmentController(IServiceProvider services, IRuntimeContext<HostingEnvironment> runtimeContext, IConfiguration config)
+            : base(services, runtimeContext, config)
         {
         }
 

src/Diagnostics.RuntimeHost/Controllers/LogicApp/LogicAppController.cs

@@ -8,6 +8,7 @@ using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.Authorization;
 using Newtonsoft.Json;
 using Microsoft.CSharp.RuntimeBinder;
+using Microsoft.Extensions.Configuration;
 
 namespace Diagnostics.RuntimeHost.Controllers
 {
@@ -16,8 +17,8 @@ namespace Diagnostics.RuntimeHost.Controllers
     [Route(UriElements.LogicAppResource)]
     public sealed class LogicAppController : DiagnosticControllerBase<LogicApp>
     {
-        public LogicAppController(IServiceProvider services, IRuntimeContext<LogicApp> runtimeContext)
-            : base(services, runtimeContext)
+        public LogicAppController(IServiceProvider services, IRuntimeContext<LogicApp> runtimeContext, IConfiguration config)
+            : base(services, runtimeContext, config)
         {
         }
 

src/Diagnostics.RuntimeHost/Controllers/Site/SiteControllerBase.cs

@@ -9,6 +9,7 @@ using Diagnostics.ModelsAndUtils.Models;
 using Diagnostics.RuntimeHost.Models;
 using Diagnostics.RuntimeHost.Services;
 using Diagnostics.RuntimeHost.Utilities;
+using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Primitives;
 
 namespace Diagnostics.RuntimeHost.Controllers
@@ -17,8 +18,8 @@ namespace Diagnostics.RuntimeHost.Controllers
     {
         protected ISiteService _siteService;
 
-        public SiteControllerBase(IServiceProvider services, IRuntimeContext<App> runtimeContext)
-            : base(services, runtimeContext)
+        public SiteControllerBase(IServiceProvider services, IRuntimeContext<App> runtimeContext, IConfiguration config)
+            : base(services, runtimeContext, config)
         {
             this._siteService = (ISiteService)services.GetService(typeof(ISiteService));
         }

src/Diagnostics.RuntimeHost/Controllers/Site/SitesController.cs

@@ -14,6 +14,7 @@ using Diagnostics.Logger;
 using Microsoft.Extensions.Primitives;
 using System.Linq;
 using Diagnostics.ModelsAndUtils.Utilities;
+using Microsoft.Extensions.Configuration;
 
 namespace Diagnostics.RuntimeHost.Controllers
 {
@@ -25,8 +26,8 @@ namespace Diagnostics.RuntimeHost.Controllers
     [Route(UriElements.SitesResource)]
     public sealed class SitesController : SiteControllerBase
     {
-        public SitesController(IServiceProvider services, IRuntimeContext<App> runtimeContext)
-            : base(services, runtimeContext)
+        public SitesController(IServiceProvider services, IRuntimeContext<App> runtimeContext, IConfiguration config)
+            : base(services, runtimeContext, config)
         {
         }
 

src/Diagnostics.RuntimeHost/Controllers/WorkerApp/WorkerAppController.cs

@@ -8,6 +8,7 @@ using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.Authorization;
 using Newtonsoft.Json;
 using Microsoft.CSharp.RuntimeBinder;
+using Microsoft.Extensions.Configuration;
 using Diagnostics.DataProviders;
 using Newtonsoft.Json.Linq;
 using System.Collections.Generic;
@@ -19,8 +20,8 @@ namespace Diagnostics.RuntimeHost.Controllers
     [Route(UriElements.WorkerAppResource)]
     public class WorkerAppController : DiagnosticControllerBase<WorkerApp>
     {
-        public WorkerAppController(IServiceProvider services, IRuntimeContext<WorkerApp> runtimeContext)
-            : base(services, runtimeContext)
+        public WorkerAppController(IServiceProvider services, IRuntimeContext<WorkerApp> runtimeContext, IConfiguration config)
+            : base(services, runtimeContext, config)
         {
         }
 

src/Diagnostics.RuntimeHost/Diagnostics.RuntimeHost.csproj

@@ -25,6 +25,7 @@
     <PackageReference Include="Microsoft.IdentityModel.Protocols.OpenIdConnect" Version="5.5.0" />
     <PackageReference Include="Microsoft.IdentityModel.Tokens" Version="5.5.0" />
     <PackageReference Include="Microsoft.TeamFoundationServer.Client" Version="16.170.0" />
+    <PackageReference Include="Microsoft.VisualStudio.Services.Client" Version="16.170.0" />
     <PackageReference Include="Octokit" Version="0.47.0" />
     <PackageReference Include="WindowsAzure.Storage" Version="9.3.3" />
     <PackageReference Include="Microsoft.Extensions.Logging.AzureAppServices" Version="3.1.9" />

src/Diagnostics.RuntimeHost/Services/CacheService/DiagEntityTableCacheService.cs

@@ -134,7 +134,7 @@ namespace Diagnostics.RuntimeHost.Services.CacheService
                     existingEntities.RemoveAt(index);
                 }
             });
-            existingEntities.AddRange(latestentities);
+            existingEntities.AddRange(latestentities.Where(s => !s.IsDisabled));
             AddOrUpdate(key, existingEntities);
         }
     }

src/Diagnostics.RuntimeHost/Services/DevOpsClient/DevOpsClient.cs

@@ -1,22 +1,17 @@
-using Diagnostics.Logger;
-using Diagnostics.RuntimeHost.Utilities;
-using Microsoft.AspNetCore.Hosting;
-using Microsoft.AspNetCore.Mvc;
+using Diagnostics.DataProviders;
+using Diagnostics.Logger;
+using Diagnostics.ModelsAndUtils.Models.Storage;
 using Microsoft.Extensions.Configuration;
 using Microsoft.TeamFoundation.SourceControl.WebApi;
 using Microsoft.VisualStudio.Services.Common;
 using Microsoft.VisualStudio.Services.WebApi;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Linq;
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
-using System.Net;
-using System.Net.Http;
-using System.Net.Http.Headers;
-using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading;
 using System.Threading.Tasks;
-using System.Web;
 
 namespace Diagnostics.RuntimeHost.Services.DevOpsClient
 {
@@ -112,7 +107,7 @@ namespace Diagnostics.RuntimeHost.Services.DevOpsClient
 
             try
             {
-                if (!string.IsNullOrWhiteSpace(branch))
+               if (!string.IsNullOrWhiteSpace(branch))
                 {
                     version = new GitVersionDescriptor()
                     {
@@ -221,6 +216,149 @@ namespace Diagnostics.RuntimeHost.Services.DevOpsClient
             return result;
         }
 
+        /// <summary>
+        /// Gets file change in the given commit.
+        /// </summary>
+        /// <param name="commitId">Commit id to process.</param>
+        public async Task<List<DevopsFileChange>> GetFilesInCommit(string commitId)
+        {
+            if (string.IsNullOrWhiteSpace(commitId))
+                throw new ArgumentNullException("commit id cannot be null");
+            CancellationTokenSource tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(DataProviderConstants.DefaultTimeoutInSeconds));
+            GitRepository repositoryAsync = await gitClient.GetRepositoryAsync(_project, _repoID, (object)null, tokenSource.Token);
+            GitCommitChanges changesAsync = await gitClient.GetChangesAsync(commitId, repositoryAsync.Id, null, null, null, tokenSource.Token);
+            List<DevopsFileChange> stringList = new List<DevopsFileChange>();
+            foreach (GitChange change in changesAsync.Changes)
+            {
+                var gitversion = new GitVersionDescriptor
+                {
+                    Version = commitId,
+                    VersionType = GitVersionType.Commit,
+                    VersionOptions = GitVersionOptions.None
+                };
+                if (change.Item.Path.EndsWith(".csx") && (change.ChangeType == VersionControlChangeType.Add || change.ChangeType == VersionControlChangeType.Edit))
+                {
+                    // hack right now, ideally get from config
+                    var detectorId = String.Join(";", Regex.Matches(change.Item.Path, @"\/(.+?)\/")
+                                        .Cast<Match>()
+                                        .Select(m => m.Groups[1].Value));
+
+
+                    Task<string> detectorScriptTask =  GetFileContentInCommit(repositoryAsync.Id, change.Item.Path, gitversion);
+                    Task<string> packageContentTask =  GetFileContentInCommit(repositoryAsync.Id, $"/{detectorId}/package.json", gitversion);
+                    Task<string> metadataContentTask =  GetFileContentInCommit(repositoryAsync.Id, $"/{detectorId}/metadata.json", gitversion);            
+                    await Task.WhenAll( new Task[] { detectorScriptTask, packageContentTask, metadataContentTask });
+                    string detectorScriptContent = await detectorScriptTask;
+                    string packageContent = await packageContentTask;
+                    string metadataContent = await metadataContentTask;
+                    stringList.Add(new DevopsFileChange
+                    {
+                        CommitId = commitId,
+                        Content = detectorScriptContent,
+                        Path = change.Item.Path,
+                        PackageConfig = packageContent,
+                        Metadata = metadataContent
+                    });
+                }
+                else if (change.Item.Path.EndsWith(".csx") && (change.ChangeType == VersionControlChangeType.Delete))
+                {
+                    var detectorId = String.Join(";", Regex.Matches(change.Item.Path, @"\/(.+?)\/")
+                                        .Cast<Match>()
+                                        .Select(m => m.Groups[1].Value));
+
+                    GitCommit gitCommitDetails = await gitClient.GetCommitAsync(commitId, repositoryAsync.Id, null, null, tokenSource.Token);
+                    // Get the package.json from the parent commit since at this commit, the file doesn't exist.
+                    var packageContent = await GetFileContentInCommit(repositoryAsync.Id, $"/{detectorId}/package.json", new GitVersionDescriptor
+                    {
+                        Version = gitCommitDetails.Parents.FirstOrDefault(),
+                        VersionType = GitVersionType.Commit,
+                        VersionOptions = GitVersionOptions.None
+                    });
+                    // Mark this detector as disabled. 
+                    stringList.Add(new DevopsFileChange
+                    {
+                        CommitId = commitId,
+                        Content = "",
+                        PackageConfig = packageContent,
+                        Path = change.Item.Path,
+                        Metadata = "",
+                        MarkAsDisabled = true
+                    });
+                }
+            }
+            return stringList;
+        }
+
+        /// <summary>
+        /// Gets the file content as string for the given path and repo at a specific commit.
+        /// </summary>
+        /// <param name="repoId">Repo guid</param>
+        /// <param name="ItemPath">Path of the item</param>
+        /// <param name="gitVersionDescriptor">Git version descriptior</param>
+        private async Task<string> GetFileContentInCommit(Guid repoId, string ItemPath, GitVersionDescriptor gitVersionDescriptor)
+        {
+            string content = string.Empty;
+            var streamResult = await gitClient.GetItemContentAsync(repoId, ItemPath, null, VersionControlRecursionType.None, null,
+                       null, null, gitVersionDescriptor);
+            using (var reader = new StreamReader(streamResult))
+            {
+                content = reader.ReadToEnd();
+            }
+            return content;
+        }
+
+        /// <summary>
+        /// Gets file changes to deploy between commits based on date range or commit range.
+        /// </summary>
+        /// <param name="parameters">The given deployment parameters</param>
+        public async Task<List<DevopsFileChange>> GetFilesBetweenCommits(DeploymentParameters parameters)
+        {
+            var result = new List<DevopsFileChange>();
+
+            string gitFromdate;
+            string gitToDate;
+            CancellationTokenSource tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(DataProviderConstants.DefaultTimeoutInSeconds));
+            GitRepository repositoryAsync = await gitClient.GetRepositoryAsync(_project, _repoID, (object)null, tokenSource.Token);
+            if (!string.IsNullOrWhiteSpace(parameters.FromCommitId) && !string.IsNullOrWhiteSpace(parameters.ToCommitId))
+            {
+                GitCommit fromCommitDetails = await gitClient.GetCommitAsync(parameters.FromCommitId, repositoryAsync.Id);
+                GitCommit toCommitDetails = await gitClient.GetCommitAsync(parameters.ToCommitId, repositoryAsync.Id);
+                gitFromdate = fromCommitDetails.Committer.Date.ToString();
+                gitToDate = toCommitDetails.Committer.Date.ToString();
+            }
+            else
+            {
+                gitFromdate = parameters.StartDate;
+                gitToDate = parameters.EndDate;
+            }
+            
+            var defaultBranch = repositoryAsync.DefaultBranch.Replace("refs/heads/", "");
+            GitVersionDescriptor gitVersionDescriptor = new GitVersionDescriptor
+            {
+                VersionType = GitVersionType.Branch,
+                Version = defaultBranch,
+                VersionOptions = GitVersionOptions.None
+            };
+            List<GitCommitRef> commitsToProcess = await gitClient.GetCommitsAsync(repositoryAsync.Id, new GitQueryCommitsCriteria
+            {
+                FromDate = gitFromdate,
+                ToDate = gitToDate,
+                ItemVersion = gitVersionDescriptor
+            }, null, null, null, tokenSource.Token);
+            foreach (var commit in commitsToProcess)
+            {
+                var filesinCommit = await GetFilesInCommit(commit.CommitId);
+                // Process only the latest file update. 
+                if (!result.Select(s => s.Path).Contains(filesinCommit.FirstOrDefault().Path))
+                {
+                    result.AddRange(filesinCommit);
+                }
+
+            }
+            return result;
+        }
+
+
         public void Dispose()
         {
             throw new NotImplementedException();

src/Diagnostics.RuntimeHost/Services/DevOpsClient/IRepoClient.cs

@@ -4,6 +4,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Net.Http;
 using System.Threading.Tasks;
+using Diagnostics.ModelsAndUtils.Models.Storage;
 
 namespace Diagnostics.RuntimeHost.Services.DevOpsClient
 {
@@ -43,5 +44,17 @@ namespace Diagnostics.RuntimeHost.Services.DevOpsClient
         /// 
         /// <returns></returns>
         Task<object> GetBranchesAsync(string resourceUri, string requestId);
+
+        /// <summary>
+        /// Gets file changed in a given commit id
+        /// </summary>
+        /// <param name="commitId">Commit id</param>
+        Task<List<DevopsFileChange>> GetFilesInCommit(string commitId);
+
+        /// <summary>
+        /// Gets file changed between two commits or date range
+        /// </summary>
+        /// <param name="parameters">Deployment parameters provided by caller</param>
+        Task<List<DevopsFileChange>> GetFilesBetweenCommits(DeploymentParameters parameters);
     }
 }

src/Diagnostics.RuntimeHost/Services/GithubClient/GithubClient.cs

@@ -4,14 +4,12 @@ using System.IO;
 using System.Linq;
 using System.Net.Http;
 using System.Net.Http.Headers;
-using System.Security.Policy;
 using System.Threading.Tasks;
 using System.Web;
 using Diagnostics.RuntimeHost.Models;
 using Diagnostics.RuntimeHost.Utilities;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.Extensions.Configuration;
-using Microsoft.Win32;
 using Octokit;
 using static Diagnostics.Logger.HeaderConstants;
 

src/Diagnostics.RuntimeHost/Services/SourceWatcher/Watchers/StorageWatcher.cs

@@ -41,6 +41,21 @@ namespace Diagnostics.RuntimeHost.Services.SourceWatcher.Watchers
         private Task kustoConfigDownloadTask;
         private IDiagEntityTableCacheService diagEntityTableCacheService;
 
+        /// <summary>
+        /// Using a flag incase anything goes wrong.
+        /// </summary>
+        private bool LoadGistFromRepo
+        {
+            get
+            {
+                if (bool.TryParse(configuration["LoadGistFromRepo"], out bool retval))
+                {
+                    return retval;
+                }
+                return false;
+            }
+        }
+
         private bool LoadOnlyPublicDetectors
         {
             get
@@ -296,9 +311,9 @@ namespace Diagnostics.RuntimeHost.Services.SourceWatcher.Watchers
             }
 
             var script = string.Empty;
-            if (partitionkey.Equals("Gist"))
-            {
-                script = await gitHubClient.GetFileContent($"{rowkey.ToLower()}/{rowkey.ToLower()}.csx");
+            if (partitionkey.Equals("Gist") && !LoadGistFromRepo)
+            {           
+               script = await gitHubClient.GetFileContent($"{rowkey.ToLower()}/{rowkey.ToLower()}.csx"); 
             }
             EntityMetadata metaData = new EntityMetadata(script, entityType, metadata);
             var newInvoker = new EntityInvoker(metaData);

src/Diagnostics.RuntimeHost/Services/StorageService/NullableStorageService.cs

@@ -47,5 +47,10 @@ namespace Diagnostics.RuntimeHost.Services.StorageService
         {
             return Task.FromResult(new List<DetectorRuntimeConfiguration>());
         }
+
+        public Task LoadBatchDataToTable(List<DiagEntity> diagEntities)
+        {
+            return Task.CompletedTask;
+        }
     }
 }

src/Diagnostics.RuntimeHost/Services/StorageService/StorageService.cs

@@ -25,23 +25,23 @@ namespace Diagnostics.RuntimeHost.Services.StorageService
         Task<DiagEntity> LoadDataToTable(DiagEntity detectorEntity);
         Task<string> LoadBlobToContainer(string blobname, string contents);
         Task<byte[]> GetBlobByName(string name);
-
         Task<int> ListBlobsInContainer();
         Task<DetectorRuntimeConfiguration> LoadConfiguration(DetectorRuntimeConfiguration configuration);
         Task<List<DetectorRuntimeConfiguration>> GetKustoConfiguration();
+
+        Task LoadBatchDataToTable(List<DiagEntity> diagEntities);
     }
     public class StorageService : IStorageService
     {
         public static readonly string PartitionKey = "PartitionKey";
         public static readonly string RowKey = "RowKey";
-        
+
         private static CloudTableClient tableClient;
-        private static CloudBlobContainer containerClient;
+        private static CloudBlobClient cloudBlobClient;
         private string tableName;
         private string container;
         private bool loadOnlyPublicDetectors;
         private bool isStorageEnabled;
-        private CloudTable cloudTable;
         private string detectorRuntimeConfigTable;
 
         public StorageService(IConfiguration configuration, IHostingEnvironment hostingEnvironment)
@@ -49,11 +49,14 @@ namespace Diagnostics.RuntimeHost.Services.StorageService
             tableName = configuration["SourceWatcher:TableName"];
             container = configuration["SourceWatcher:BlobContainerName"];
             detectorRuntimeConfigTable = configuration["SourceWatcher:DetectorRuntimeConfigTable"];
-            if(hostingEnvironment != null && hostingEnvironment.EnvironmentName.Equals("UnitTest", StringComparison.CurrentCultureIgnoreCase))
+            
+           
+            if (hostingEnvironment != null && hostingEnvironment.EnvironmentName.Equals("UnitTest", StringComparison.CurrentCultureIgnoreCase))
             {
                 tableClient = CloudStorageAccount.DevelopmentStorageAccount.CreateCloudTableClient();
-                containerClient = CloudStorageAccount.DevelopmentStorageAccount.CreateCloudBlobClient().GetContainerReference(container);
-            } else
+                cloudBlobClient = CloudStorageAccount.DevelopmentStorageAccount.CreateCloudBlobClient();             
+            }
+            else
             {
                 var accountname = configuration["SourceWatcher:DiagStorageAccount"];
                 var key = configuration["SourceWatcher:DiagStorageKey"];
@@ -64,9 +67,9 @@ namespace Diagnostics.RuntimeHost.Services.StorageService
                 }
                 var storageAccount = new CloudStorageAccount(new StorageCredentials(accountname, key), accountname, dnsSuffix, true);
                 tableClient = storageAccount.CreateCloudTableClient();
-                containerClient = storageAccount.CreateCloudBlobClient().GetContainerReference(container);
+                cloudBlobClient = storageAccount.CreateCloudBlobClient();             
             }
-         
+
             if (!bool.TryParse((configuration[$"SourceWatcher:{RegistryConstants.LoadOnlyPublicDetectorsKey}"]), out loadOnlyPublicDetectors))
             {
                 loadOnlyPublicDetectors = false;
@@ -76,7 +79,7 @@ namespace Diagnostics.RuntimeHost.Services.StorageService
             {
                 isStorageEnabled = true;
             }
-            cloudTable = tableClient.GetTableReference(tableName);
+
         }
 
         public async Task<List<DiagEntity>> GetEntitiesByPartitionkey(string partitionKey = null, DateTime? startTime = null)
@@ -91,7 +94,7 @@ namespace Diagnostics.RuntimeHost.Services.StorageService
                 {
                     CloudTable table = tableClient.GetTableReference(tableName);
                     var timeTakenStopWatch = new Stopwatch();
-                    if(string.IsNullOrWhiteSpace(partitionKey))
+                    if (string.IsNullOrWhiteSpace(partitionKey))
                     {
                         partitionKey = "Detector";
                     }
@@ -126,11 +129,12 @@ namespace Diagnostics.RuntimeHost.Services.StorageService
                     } while (tableContinuationToken != null);
                     timeTakenStopWatch.Stop();
                     DiagnosticsETWProvider.Instance.LogAzureStorageMessage(nameof(StorageService), $"GetEntities by Partition key {partitionKey} took {timeTakenStopWatch.ElapsedMilliseconds}, Total rows = {detectorsResult.Count}, ClientRequestId = {clientRequestId} ");
-                    return detectorsResult.Where(result => !result.IsDisabled).ToList();
+                     return startTime == DateTime.MinValue ? detectorsResult.Where(result => !result.IsDisabled).ToList() :
+                        detectorsResult.ToList();
                 }
                 catch (Exception ex)
                 {
-                    DiagnosticsETWProvider.Instance.LogAzureStorageException(nameof(StorageService), $"ClientRequestId : {clientRequestId} {ex.Message}", ex.GetType().ToString(), ex.ToString());               
+                    DiagnosticsETWProvider.Instance.LogAzureStorageException(nameof(StorageService), $"ClientRequestId : {clientRequestId} {ex.Message}", ex.GetType().ToString(), ex.ToString());
                 }
                 finally
                 {
@@ -151,7 +155,7 @@ namespace Diagnostics.RuntimeHost.Services.StorageService
             try
             {
                 // Create a table client for interacting with the table service 
-                CloudTable table = tableClient.GetTableReference(tableName);
+                CloudTable table =  tableClient.GetTableReference(tableName);
                 if (detectorEntity == null || detectorEntity.PartitionKey == null || detectorEntity.RowKey == null)
                 {
                     throw new ArgumentNullException(nameof(detectorEntity));
@@ -159,10 +163,10 @@ namespace Diagnostics.RuntimeHost.Services.StorageService
 
                 var timeTakenStopWatch = new Stopwatch();
                 timeTakenStopWatch.Start();
-                    
+
                 // Create the InsertOrReplace table operation
                 TableOperation insertOrReplaceOperation = TableOperation.InsertOrReplace(detectorEntity);
-
+             
                 TableRequestOptions tableRequestOptions = new TableRequestOptions();
                 tableRequestOptions.LocationMode = LocationMode.PrimaryOnly;
                 tableRequestOptions.MaximumExecutionTime = TimeSpan.FromSeconds(60);
@@ -189,9 +193,10 @@ namespace Diagnostics.RuntimeHost.Services.StorageService
             var clientRequestId = Guid.NewGuid().ToString();
             try
             {
-                var timeTakenStopWatch = new Stopwatch();    
+                var timeTakenStopWatch = new Stopwatch();
                 timeTakenStopWatch.Start();
-                var cloudBlob = containerClient.GetBlockBlobReference(blobname);
+                var containerReference = cloudBlobClient.GetContainerReference(container);
+                var cloudBlob = containerReference.GetBlockBlobReference(blobname);
                 BlobRequestOptions blobRequestOptions = new BlobRequestOptions();
                 blobRequestOptions.LocationMode = LocationMode.PrimaryOnly;
                 blobRequestOptions.MaximumExecutionTime = TimeSpan.FromSeconds(60);
@@ -200,14 +205,15 @@ namespace Diagnostics.RuntimeHost.Services.StorageService
                 DiagnosticsETWProvider.Instance.LogAzureStorageMessage(nameof(StorageService), $"Loading {blobname} with ClientRequestId {clientRequestId}");
                 using (var uploadStream = new MemoryStream(Convert.FromBase64String(contents)))
                 {
-                    await cloudBlob.UploadFromStreamAsync(uploadStream, null, blobRequestOptions, oc);               
+                    await cloudBlob.UploadFromStreamAsync(uploadStream, null, blobRequestOptions, oc);
                 }
                 await cloudBlob.FetchAttributesAsync();
                 timeTakenStopWatch.Stop();
-                var uploadResult = cloudBlob.Properties;  
+                var uploadResult = cloudBlob.Properties;
                 DiagnosticsETWProvider.Instance.LogAzureStorageMessage(nameof(StorageService), $"Loaded {blobname}, etag {uploadResult.ETag}, time taken {timeTakenStopWatch.ElapsedMilliseconds} ClientRequestId {clientRequestId}");
                 return uploadResult.ETag;
-            } catch (Exception ex)
+            }
+            catch (Exception ex)
             {
                 DiagnosticsETWProvider.Instance.LogAzureStorageException(nameof(StorageService), $" ClientRequestId {clientRequestId} {ex.Message}", ex.GetType().ToString(), ex.ToString());
                 return null;
@@ -234,10 +240,11 @@ namespace Diagnostics.RuntimeHost.Services.StorageService
                     }
                     var timeTakenStopWatch = new Stopwatch();
                     timeTakenStopWatch.Start();
-                    var cloudBlob = containerClient.GetBlockBlobReference(name);
                     OperationContext oc = new OperationContext();
                     oc.ClientRequestID = clientRequestId;
                     DiagnosticsETWProvider.Instance.LogAzureStorageMessage(nameof(StorageService), $"Fetching blob {name} with ClientRequestid {clientRequestId}");
+                    var containerReference = cloudBlobClient.GetContainerReference(container);
+                    var cloudBlob = containerReference.GetBlockBlobReference(name);
                     using (MemoryStream ms = new MemoryStream())
                     {
                         await cloudBlob.DownloadToStreamAsync(ms, null, options, oc);
@@ -245,6 +252,7 @@ namespace Diagnostics.RuntimeHost.Services.StorageService
                         DiagnosticsETWProvider.Instance.LogAzureStorageMessage(nameof(StorageService), $"Downloaded {name} to memory stream, time taken {timeTakenStopWatch.ElapsedMilliseconds} ClientRequestid {clientRequestId}");
                         return ms.ToArray();
                     }
+                                      
                 }
                 catch (Exception ex)
                 {
@@ -257,7 +265,7 @@ namespace Diagnostics.RuntimeHost.Services.StorageService
             } while (attempt <= retryThreshold);
             return null;
         }
-    
+
         public async Task<int> ListBlobsInContainer()
         {
             var clientRequestId = Guid.NewGuid().ToString();
@@ -270,17 +278,18 @@ namespace Diagnostics.RuntimeHost.Services.StorageService
                 options.MaximumExecutionTime = TimeSpan.FromSeconds(60);
                 OperationContext oc = new OperationContext();
                 oc.ClientRequestID = clientRequestId;
-                var blobsResult =  await containerClient.ListBlobsSegmentedAsync(null, true, BlobListingDetails.All, 1000, null, options, oc );
+                var blobsResult = await cloudBlobClient.ListBlobsSegmentedAsync(null, true, BlobListingDetails.All, 1000, null, options, oc);
                 timeTakenStopWatch.Stop();
                 DiagnosticsETWProvider.Instance.LogAzureStorageMessage(nameof(StorageService), $"Number of blobs stored in container {container} is {blobsResult.Results.Count()}, time taken {timeTakenStopWatch.ElapsedMilliseconds} milliseconds, ClientRequestId {clientRequestId}");
-                return blobsResult.Results != null ? blobsResult.Results.Count() : 0 ;
-            } catch (Exception ex)
+                return blobsResult.Results != null ? blobsResult.Results.Count() : 0;
+            }
+            catch (Exception ex)
             {
                 DiagnosticsETWProvider.Instance.LogAzureStorageException(nameof(StorageService), $"ClientRequestId {clientRequestId} {ex.Message}", ex.GetType().ToString(), ex.ToString());
                 throw ex;
             }
         }
-    
+
         public async Task<DetectorRuntimeConfiguration> LoadConfiguration(DetectorRuntimeConfiguration configuration)
         {
             var clientRequestId = Guid.NewGuid().ToString();
@@ -288,7 +297,7 @@ namespace Diagnostics.RuntimeHost.Services.StorageService
             {
                 // Create a table client for interacting with the table service 
                 CloudTable table = tableClient.GetTableReference(detectorRuntimeConfigTable);
-                if(configuration == null || configuration.PartitionKey == null || configuration.RowKey == null )
+                if (configuration == null || configuration.PartitionKey == null || configuration.RowKey == null)
                 {
                     throw new ArgumentNullException(nameof(configuration));
                 }
@@ -309,14 +318,15 @@ namespace Diagnostics.RuntimeHost.Services.StorageService
                 DiagnosticsETWProvider.Instance.LogAzureStorageMessage(nameof(StorageService), $"InsertOrReplace result : {result.HttpStatusCode}, time taken {timeTakenStopWatch.ElapsedMilliseconds} ClientRequestId {clientRequestId}");
                 DetectorRuntimeConfiguration insertedRow = result.Result as DetectorRuntimeConfiguration;
                 return insertedRow;
-            } catch (Exception ex)
+            }
+            catch (Exception ex)
             {
                 DiagnosticsETWProvider.Instance.LogAzureStorageException(nameof(StorageService), $"ClientRequestId {clientRequestId} {ex.Message}", ex.GetType().ToString(), ex.ToString());
                 return null;
             }
 
         }
-   
+
         public async Task<List<DetectorRuntimeConfiguration>> GetKustoConfiguration()
         {
             var clientRequestId = Guid.NewGuid().ToString();
@@ -350,11 +360,55 @@ namespace Diagnostics.RuntimeHost.Services.StorageService
                 timeTakenStopWatch.Stop();
                 DiagnosticsETWProvider.Instance.LogAzureStorageMessage(nameof(StorageService), $"GetConfiguration by Partition key {partitionkey} took {timeTakenStopWatch.ElapsedMilliseconds}, Total rows = {diagConfigurationsResult.Count} ClientRequestId {clientRequestId}");
                 return diagConfigurationsResult.Where(row => !row.IsDisabled).ToList();
-            } catch (Exception ex)
+            }
+            catch (Exception ex)
             {
                 DiagnosticsETWProvider.Instance.LogAzureStorageException(nameof(StorageService), $"ClientRequestId {clientRequestId} {ex.Message}", ex.GetType().ToString(), ex.ToString());
                 return null;
             }
         }
+
+        public async Task LoadBatchDataToTable(List<DiagEntity> diagEntities)
+        {
+            var clientRequestId = Guid.NewGuid().ToString();
+            try
+            {
+                // Create a table reference for interacting with the table service 
+                CloudTable table = tableClient.GetTableReference(tableName);
+                if (diagEntities == null || diagEntities.Count == 0)
+                {
+                    throw new ArgumentNullException("List is empty");
+                }
+
+                var timeTakenStopWatch = new Stopwatch();
+                timeTakenStopWatch.Start();
+
+                TableBatchOperation batchOperation = new TableBatchOperation();
+                
+                foreach (var entity in diagEntities)
+                {
+                    batchOperation.InsertOrReplace(entity);
+                }
+                
+                TableRequestOptions tableRequestOptions = new TableRequestOptions();
+                tableRequestOptions.LocationMode = LocationMode.PrimaryOnly;
+                tableRequestOptions.MaximumExecutionTime = TimeSpan.FromSeconds(60);
+                OperationContext oc = new OperationContext();
+                oc.ClientRequestID = clientRequestId;
+                DiagnosticsETWProvider.Instance.LogAzureStorageMessage(nameof(StorageService), $"Insert or Replace batch diag entities into {tableName} ClientRequestId {clientRequestId}");
+                //Execute batch operation
+                IList<TableResult> result = await table.ExecuteBatchAsync(batchOperation, tableRequestOptions, oc);            
+                timeTakenStopWatch.Stop();
+                DiagnosticsETWProvider.Instance.LogAzureStorageMessage(nameof(StorageService), $"InsertOrReplace batch time taken {timeTakenStopWatch.ElapsedMilliseconds}, ClientRequestId {clientRequestId}");
+                
+            }
+            catch (Exception ex)
+            {
+                DiagnosticsETWProvider.Instance.LogAzureStorageException(nameof(StorageService), $"ClientRequestId {clientRequestId} {ex.Message}", ex.GetType().ToString(), ex.ToString());
+                throw;
+            }
+          
+        }
     }
 }
+

src/Diagnostics.RuntimeHost/Startup.cs

@@ -242,7 +242,6 @@ namespace Diagnostics.RuntimeHost
             services.AddDiagEntitiesTableCacheService(Configuration);
 
             InjectSourceWatcher(services);
-
             services.AddLogging(loggingConfig =>
             {
                 loggingConfig.ClearProviders();

src/Diagnostics.RuntimeHost/appsettings.json

@@ -166,6 +166,7 @@
     "ApiVersion": "3.0",
     "TranslatorSubscriptionKey": ""
   },
+  "LoadGistFromRepo": false,
   "DevOps": {
     "PersonalAccessToken": "",
     "Organization": "",

src/Diagnostics.Scripts/CompilationService/Utilities/DetectorParser.cs

@@ -0,0 +1,35 @@
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.CodeAnalysis.CSharp.Syntax;
+using System.Collections.Generic;
+
+namespace Diagnostics.Scripts.CompilationService.Utilities
+{
+    /// <summary>
+    /// Parser for detector script.
+    /// </summary>
+    public static class DetectorParser
+    {
+        /// <summary>
+        /// Gets the names of gists referenced in detector code.
+        /// </summary>
+        /// <param name="detectorCode">Detector script.</param>
+        /// <returns>List of gist referenced.</returns>
+        public static List<string> GetLoadDirectiveNames(string detectorCode)
+        {
+            var result = new List<string>();
+            SyntaxTree tree = CSharpSyntaxTree.ParseText(detectorCode);
+            CompilationUnitSyntax root = tree.GetCompilationUnitRoot();
+
+            if (root == null) return result;
+
+            foreach(var loadRef in root.GetLoadDirectives())
+            {
+                result.Add(loadRef.File.ValueText);
+            }
+
+            return result;
+        }
+    
+    }
+}

tests/Diagnostics.Tests/Diagnostics.Tests.csproj

@@ -36,6 +36,9 @@
     <None Update="TestData\TestDetectorWithSupportTopic.csx">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
+    <None Update="TestData\TestDetectorWithGist.csx">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
   </ItemGroup>
 
   <PropertyGroup>

tests/Diagnostics.Tests/ScriptsTests/DetectorParserTests.cs

@@ -0,0 +1,36 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Xunit;
+using System.IO;
+using Diagnostics.Scripts.CompilationService.Utilities;
+
+namespace Diagnostics.Tests.ScriptsTests
+{
+    public class DetectorParserTests
+    {
+        [Fact]
+
+        public async void TestParserForGistClass()
+        {
+            string code = await File.ReadAllTextAsync(@"TestData/TestDetectorWithGist.csx");
+            var gistClass = DetectorParser.GetLoadDirectiveNames(code);
+            Assert.NotEmpty(gistClass);
+            Assert.Equal(2, gistClass.Count);
+
+            string programText = @"
+                                namespace HelloWorld
+                                {
+                                    class Program
+                                    {
+                                        static void Main(string[] args)
+                                        {
+                                            Console.WriteLine(""Hello, World!"");
+                                        }
+                                    }
+                                }";
+            var gistClassEmpty = DetectorParser.GetLoadDirectiveNames(programText);
+            Assert.Equal(0, gistClassEmpty.Count);
+        }
+    }
+}

tests/Diagnostics.Tests/TestData/TestDetectorWithGist.csx

@@ -0,0 +1,10 @@
+#load "DetectorUtils"
+#load "CredentialTrapper"
+
+[AppFilter(AppType = AppType.WebApp, PlatformType = PlatformType.Windows, StackType = StackType.All)]
+[Definition(Id = "testdetectorwithgist", Name = "Test Detector with Gist", Author = "dev", Description = "")]
+public async static Task<Response> Run(DataProviders dp, OperationContext<App> cxt, Response res)
+{
+    res.AddMarkdownView("Hello World!");
+    return res;
+}