Scheduled func app to back fill data in storage (#471)

* Scheduled func app to back fill data in storage

.gitignore

@@ -319,3 +319,7 @@ aot/
 appsettings.*.json
 
 [Dd]etectorsCache
+
+# Azure Functions localsettings file
+local.settings.json
+configuration.json

Diagnostics.sln

@@ -56,6 +56,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "metadata", "metadata", "{83
 		src\Diagnostics.AIProjects\TrainingFactory\metadata\trainingConfig.json = src\Diagnostics.AIProjects\TrainingFactory\metadata\trainingConfig.json
 	EndProjectSection
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SourceWatcherFuncApp", "src\SourceWatcherFuncApp\SourceWatcherFuncApp.csproj", "{29B746B2-AB68-428F-8613-2BDCB49DD8D6}"
+EndProject
 Global
 	GlobalSection(SharedMSBuildProjectFiles) = preSolution
 		data\samples\Diagnostics.Samples\Diagnostics.Samples.projitems*{99368f78-83b7-46d4-b49b-f26fe165f53c}*SharedItemsImports = 13
@@ -153,6 +155,18 @@ Global
 		{9EBCA533-692C-4A6E-9831-AE84C0AFCEF2}.Release|x64.Build.0 = Release|Any CPU
 		{9EBCA533-692C-4A6E-9831-AE84C0AFCEF2}.Release|x86.ActiveCfg = Release|Any CPU
 		{9EBCA533-692C-4A6E-9831-AE84C0AFCEF2}.Release|x86.Build.0 = Release|Any CPU
+		{29B746B2-AB68-428F-8613-2BDCB49DD8D6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{29B746B2-AB68-428F-8613-2BDCB49DD8D6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{29B746B2-AB68-428F-8613-2BDCB49DD8D6}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{29B746B2-AB68-428F-8613-2BDCB49DD8D6}.Debug|x64.Build.0 = Debug|Any CPU
+		{29B746B2-AB68-428F-8613-2BDCB49DD8D6}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{29B746B2-AB68-428F-8613-2BDCB49DD8D6}.Debug|x86.Build.0 = Debug|Any CPU
+		{29B746B2-AB68-428F-8613-2BDCB49DD8D6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{29B746B2-AB68-428F-8613-2BDCB49DD8D6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{29B746B2-AB68-428F-8613-2BDCB49DD8D6}.Release|x64.ActiveCfg = Release|Any CPU
+		{29B746B2-AB68-428F-8613-2BDCB49DD8D6}.Release|x64.Build.0 = Release|Any CPU
+		{29B746B2-AB68-428F-8613-2BDCB49DD8D6}.Release|x86.ActiveCfg = Release|Any CPU
+		{29B746B2-AB68-428F-8613-2BDCB49DD8D6}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -170,6 +184,7 @@ Global
 		{4991C2E8-75B2-46AE-9F0C-D9491028F6BE} = {3A34997D-ECF8-4BA5-8BC7-6BA9A6DBDB3A}
 		{F542FAE9-5CBA-4B7F-AD2A-54596E0004CA} = {3A34997D-ECF8-4BA5-8BC7-6BA9A6DBDB3A}
 		{837CE97F-30D7-4923-B0E8-1F9ABA325221} = {F542FAE9-5CBA-4B7F-AD2A-54596E0004CA}
+		{29B746B2-AB68-428F-8613-2BDCB49DD8D6} = {0F59F43E-D06C-4A0F-8D38-9538BA58FC6A}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {B4A19BFB-9A33-49E2-98AD-955E9472EFAD}

src/SourceWatcherFuncApp/DiagSourceWatcher.cs

@@ -0,0 +1,85 @@
+using System.Threading.Tasks;
+using Microsoft.Azure.WebJobs;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Configuration;
+using System;
+using System.Linq;
+using SourceWatcherFuncApp.Services;
+using SourceWatcherFuncApp.Utilities;
+using SourceWatcherFuncApp.Entities;
+namespace Diag.SourceWatcher
+{
+    public class DiagSourceWatcher
+    {        
+        private IGithubService githubService;
+
+        private ITableStorageService storageService;
+
+        private IConfigurationRoot config;
+
+        private IBlobService blobService;
+
+        public DiagSourceWatcher(IConfigurationRoot configurationRoot, IGithubService GithubService, 
+                                 ITableStorageService StorageService, IBlobService BlobService)
+        {
+            githubService = GithubService;
+            config = configurationRoot;
+            storageService = StorageService;
+            blobService = BlobService;
+        }
+
+        [FunctionName("DiagSourceWatcher")]
+        public async Task Run([TimerTrigger("0 0 0 * * *")]TimerInfo timerInfo, ILogger log)
+        {
+            log.LogInformation($"C# Timer trigger function executed at: {DateTime.Now}");
+
+            log.LogInformation($"Loading directories from github branch  {config["Github:Branch"]}");
+
+            var githubDirectories = await githubService.DownloadGithubDirectories(config["Github:Branch"]);
+
+            foreach (var githubdir in githubDirectories)
+            {
+                if (!githubdir.Type.Equals("dir", StringComparison.OrdinalIgnoreCase)) continue;
+                
+                var contentList = await githubService.DownloadGithubDirectories(branchdownloadUrl: githubdir.Url);
+                
+                var assemblyFile = contentList.Where(githubFile => githubFile.Name.EndsWith("dll")).FirstOrDefault();
+                var scriptFile = contentList.Where(githubfile => githubfile.Name.EndsWith(".csx")).FirstOrDefault();
+                var configFile = contentList.Where(githubFile => githubFile.Name.Equals("package.json", StringComparison.CurrentCultureIgnoreCase)).FirstOrDefault();
+               
+                if (assemblyFile != null && scriptFile != null && configFile != null)
+                {
+
+                    log.LogInformation($"Getting content for Assembly file : {assemblyFile.Path}");
+                    var assemblyData = await githubService.GetFileContentStream(assemblyFile.Download_url);
+
+                    //log.LogInformation("Reading detector metadata");
+                    var configFileData = await githubService.GetFileContentByType<DetectorEntity>(configFile.Download_url);
+
+                    configFileData = EntityHelper.PrepareEntityForLoad(assemblyData, string.Empty, configFileData);
+                    configFileData.GitHubSha = githubdir.Sha;
+                    configFileData.GithubLastModified = await githubService.GetCommitDate(scriptFile.Path);
+
+                    //First check if table already has the entry
+                    var existingDetectorEntity = await storageService.GetEntityFromTable(configFileData.PartitionKey, configFileData.RowKey);
+
+                     //If there is no entry in table or github last modifed date has been changed, upload to blob
+                      if (existingDetectorEntity == null || existingDetectorEntity.GithubLastModified != configFileData.GithubLastModified)
+                    {
+                        var assemblyLastModified = await githubService.GetCommitDate(assemblyFile.Path);
+                        blobService.LoadBlobToContainer(assemblyFile.Path, assemblyData);
+                    }
+
+                    var result = await storageService.LoadDataToTable(configFileData);
+
+                }
+                else
+                {
+                    log.LogInformation($"One or more files were not found in {githubdir.Name}, skipped storage entry");
+                }
+            }
+
+            log.LogInformation($"C# Timer trigger function executed at: {DateTime.Now}");
+        }
+    }
+}

src/SourceWatcherFuncApp/Entities/DetectorEntity.cs

@@ -0,0 +1,69 @@
+using Diagnostics.ModelsAndUtils.Attributes;
+using Newtonsoft.Json;
+using System;
+using System.Collections.Generic;
+using Microsoft.WindowsAzure.Storage.Table;
+
+namespace SourceWatcherFuncApp.Entities
+{
+    // Entity that gets stored in the table
+    public class DetectorEntity: TableEntity
+    {
+        public DetectorEntity()
+        {
+
+        }
+        public DetectorEntity(string entityType, string detectorid)
+        {
+            PartitionKey = entityType;
+            RowKey = detectorid;
+        }
+        [JsonProperty("id")]
+        public string DetectorId { get; set; }
+        [JsonProperty("name")]
+        public string DetectorName { get; set; }
+        [JsonProperty("type")]
+        public string EntityType { get; set; }
+
+        public string Author { get; set; }
+        public string Description { get; set; }
+        public bool IsInternal { get; set; }
+        public string Category { get; set; }
+        public string ResourceType { get; set; }
+
+        public DateTime GithubLastModified { get; set; }
+
+        public string GitHubSha { get; set; }
+
+        public string SupportTopicListRaw { get; set; }
+
+        public string AnalysisTypesRaw { get; set; }
+
+        [IgnoreProperty]
+
+        public  IEnumerable<SupportTopic> SupportTopicList
+        {
+            get
+            {
+                return JsonConvert.DeserializeObject<IEnumerable<SupportTopic>>(SupportTopicListRaw);
+            }
+            set
+            {
+                SupportTopicListRaw = JsonConvert.SerializeObject(value);
+            }
+        }
+
+        [IgnoreProperty]
+        public List<string> AnalysisTypes
+        {
+            get
+            {
+                return JsonConvert.DeserializeObject<List<string>>(AnalysisTypesRaw);
+            }
+            set
+            {
+                AnalysisTypesRaw = JsonConvert.SerializeObject(value);
+            }
+        }
+    }
+}

src/SourceWatcherFuncApp/GithubEntry.cs

@@ -0,0 +1,35 @@
+namespace SourceWatcherFuncApp
+{
+    public class GithubEntry
+    {
+        /// <summary>
+        /// Gets or sets the name.
+        /// </summary>
+        public string Name { get; set; }
+
+        /// <summary>
+        /// Gets or sets the path.
+        /// </summary>
+        public string Path { get; set; }
+
+        /// <summary>
+        /// Gets or sets the SHA.
+        /// </summary>
+        public string Sha { get; set; }
+
+        /// <summary>
+        /// Gets or sets the URL.
+        /// </summary>
+        public string Url { get; set; }
+
+        /// <summary>
+        /// Gets or sets the download URL.
+        /// </summary>
+        public string Download_url { get; set; }
+
+        /// <summary>
+        /// Gets or sets the type.
+        /// </summary>
+        public string Type { get; set; }
+    }
+}

src/SourceWatcherFuncApp/Services/IBlobService.cs

@@ -0,0 +1,71 @@
+using System;
+using System.IO;
+using Azure.Storage;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
+using Azure.Storage.Blobs;
+using Azure;
+using System.Net.Http;
+using Azure.Core.Pipeline;
+
+namespace SourceWatcherFuncApp.Services
+{
+    public interface IBlobService
+    {
+        void LoadBlobToContainer(string name, Stream uploadStream);
+    }
+
+    public class BlobService : IBlobService
+    {
+        private static BlobContainerClient cloudBobClient;
+
+        private ILogger<BlobService> blobServiceLogger;
+
+        private string containerName;
+
+        private string storageUri;
+
+        private Uri containerUri;
+
+        private static HttpClient httpClient = new HttpClient();
+
+        public BlobService(IConfigurationRoot configuration, ILogger<BlobService> logger)
+        {
+            var accountname = configuration["AzureStorageAccount"];
+            var key = configuration["AzureStorageKey"];
+            containerName = configuration["BlobContainerName"];
+            storageUri = $"https://{accountname}.blob.core.windows.net";
+            containerUri = new Uri($"{storageUri}/{containerName}");
+            StorageSharedKeyCredential credential = new StorageSharedKeyCredential(accountname, key);
+            httpClient.Timeout = TimeSpan.FromSeconds(102);
+            cloudBobClient = new BlobContainerClient(containerUri, credential, new BlobClientOptions
+            {
+               Transport = new HttpClientTransport(httpClient)
+            });
+            this.blobServiceLogger = logger ?? logger;
+        }
+
+      
+        public async void LoadBlobToContainer(string name, Stream uploadStream)
+        {
+            try
+            {
+                blobServiceLogger.LogInformation($"Uploading {name} to blob");
+                var uploadResponse = await cloudBobClient.UploadBlobAsync(name, uploadStream);
+                blobServiceLogger.LogInformation($"Upload response for {name} : {uploadResponse.GetRawResponse().Status}, {uploadResponse.GetRawResponse().ReasonPhrase}");
+            }
+            catch (RequestFailedException e) when (e.Status == 409)
+            {
+                // handle existing blob;
+                blobServiceLogger.LogInformation($"Conflict occured for {name}, trying to override upload");
+                var blobClient = cloudBobClient.GetBlobClient(name);
+                var uploadExistingBlob = await blobClient.UploadAsync(uploadStream, true);
+                blobServiceLogger.LogInformation($"Updated existing blob {name}, response: {uploadExistingBlob.GetRawResponse().Status}, {uploadExistingBlob.GetRawResponse().ReasonPhrase}");
+            }
+            catch (Exception ex)
+            {
+                blobServiceLogger.LogError(ex.ToString());
+            }
+        }
+    }
+}

src/SourceWatcherFuncApp/Services/IGithubService.cs

@@ -0,0 +1,147 @@
+using System;
+using Octokit;
+using Microsoft.Extensions.Configuration;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
+using System.IO;
+using SourceWatcherFuncApp.Entities;
+
+namespace SourceWatcherFuncApp.Services
+{
+    public interface IGithubService
+    {
+        Task<Reference> CreateBranchIfNotExists();
+
+        Task<GithubEntry[]> DownloadGithubDirectories(string branchName = "", string branchdownloadUrl = "");
+        Task<Stream> GetFileContentStream(string url);
+        Task<DetectorEntity> GetFileContentJson(string url);
+        Task<T> GetFileContentByType<T>(string url);
+
+        Task<DateTime> GetCommitDate(string path);
+    }
+
+    public class GithubService: IGithubService
+    {
+        private static HttpClient httpClient = new HttpClient();
+
+        private GitHubClient octokitClient;
+
+        private string baseBranch;
+
+        private string targetBranch;
+
+        private string githubUserName;
+
+        private string repoName;
+        public GithubService(IConfigurationRoot config)
+        {
+
+            githubUserName = config["Github:UserName"];
+            repoName = config["Github:RepoName"];
+            baseBranch = config["Github:Branch"];
+            var AccessToken = config["Github:AccessToken"];
+            targetBranch = config["Github:TargetBranch"];
+
+            octokitClient = new GitHubClient(new Octokit.ProductHeaderValue(githubUserName))
+            {
+                Credentials = new Credentials(AccessToken)
+            };
+
+            // Initialize HttpClient
+
+            httpClient.DefaultRequestHeaders.Add("Authorization", $@"token {AccessToken}");
+            httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
+            httpClient.DefaultRequestHeaders.Add("User-Agent", "sourcewatcherfuncapp");
+        } 
+
+        public async Task<Reference> CreateBranchIfNotExists()
+        {
+
+            // Check if branch exists
+
+            var gitHubbasebranch = await octokitClient.Git.Reference.Get(githubUserName, repoName, $"heads/{this.baseBranch}");
+            Reference targetGithubBranch = null;
+
+            try
+            {
+                targetGithubBranch = await octokitClient.Git.Reference.Get(githubUserName, repoName, $"heads/{this.targetBranch}");
+            }
+            catch (Exception ex)
+            {
+                if (ex.ToString().Contains("Not found"))
+                {
+                    targetGithubBranch = await Octokit.Helpers.ReferenceExtensions.CreateBranch(octokitClient.Git.Reference, githubUserName, repoName, targetBranch, gitHubbasebranch);
+                }
+            }
+            return targetGithubBranch;
+        }
+
+        public async Task<GithubEntry[]> DownloadGithubDirectories(string branchName = "", string branchdownloadUrl = "")
+        {
+            var downloadUrl = !string.IsNullOrWhiteSpace(branchName) ? $"https://api.github.com/repos/{githubUserName}/{repoName}/contents?ref={branchName}" : branchdownloadUrl;
+            HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, downloadUrl);
+
+            var response = await httpClient.SendAsync(request);
+            var jsonString = await response.Content.ReadAsStringAsync();
+            var githubDirectories = JsonConvert.DeserializeObject<GithubEntry[]>(jsonString);
+
+            return githubDirectories;
+           
+        }
+
+        public async Task<DetectorEntity> GetFileContentJson(string url)
+        {
+            var response = await httpClient.SendAsync(new HttpRequestMessage(HttpMethod.Get, url));
+            var responsestring = await response.Content.ReadAsStringAsync();
+            return JsonConvert.DeserializeObject<DetectorEntity>(responsestring);
+        }
+
+        public async Task<Stream> GetFileContentStream(string url)
+        {
+            var response = await httpClient.SendAsync(new HttpRequestMessage(HttpMethod.Get, url));
+            var stream = await response.Content.ReadAsStreamAsync();
+            return stream;
+        }
+
+        public async Task<T> GetFileContentByType<T>(string url)
+        {
+            var response = await httpClient.SendAsync(new HttpRequestMessage(HttpMethod.Get, url));
+            var responseContent = await response.Content.ReadAsStringAsync();
+
+            if (typeof(T).Equals(typeof(string)))
+            {
+                return (T)(object)responseContent;
+            }
+            else
+            {
+
+                T value;
+                try
+                {
+                    value = JsonConvert.DeserializeObject<T>(responseContent);
+                }
+                catch (JsonSerializationException serializeException)
+                {
+                    throw new JsonSerializationException($"Failed to serialize response {responseContent}", serializeException);
+                }
+
+                return value;
+            }
+        }
+
+        public async Task<DateTime> GetCommitDate(string path)
+        {
+            if(path == null)
+            {
+                throw new ArgumentNullException(nameof(path));
+            }
+            var commitsForFile = await octokitClient.Repository.Commit.GetAll(githubUserName, repoName, new CommitRequest { Path = path, Sha = baseBranch });
+            var mostRecentCommit = commitsForFile[0];
+            var authorDate = mostRecentCommit.Commit.Author.Date;
+            return authorDate.DateTime.ToUniversalTime();
+        }
+        
+    }
+}

src/SourceWatcherFuncApp/Services/ITableStorageService.cs

@@ -0,0 +1,95 @@
+using System;
+using Microsoft.Extensions.Configuration;
+using SourceWatcherFuncApp.Entities;
+using System.Threading.Tasks;
+using Microsoft.WindowsAzure.Storage.Auth;
+using Microsoft.WindowsAzure.Storage.Table;
+using Microsoft.Extensions.Logging;
+using CloudStorageAccount = Microsoft.WindowsAzure.Storage.CloudStorageAccount;
+
+namespace SourceWatcherFuncApp.Services
+{
+    public interface ITableStorageService
+    {
+        Task<DetectorEntity> LoadDataToTable(DetectorEntity detectorEntity);
+        Task<DetectorEntity> GetEntityFromTable(string partitionKey, string rowKey);
+    }
+
+    public class TableStorageService : ITableStorageService
+    {
+        private static CloudTableClient tableClient;
+        
+        private string blobContainerName;
+
+        private string tableName;
+
+        private ILogger<TableStorageService> storageServiceLogger;
+
+        public TableStorageService(IConfigurationRoot configuration, ILogger<TableStorageService> logger)
+        {
+            blobContainerName = configuration["BlobContainerName"];
+            tableName = configuration["TableName"];
+            var accountname = configuration["AzureStorageAccount"];
+            var key = configuration["AzureStorageKey"];
+            var containerUri = new Uri($"https://{accountname}.blob.core.windows.net/{blobContainerName}");
+            var tableUri = new Uri($"https://{accountname}.table.core.windows.net/{tableName}");
+            var storageAccount = new CloudStorageAccount(new StorageCredentials(accountname, key), accountname, "core.windows.net", true);
+            tableClient = storageAccount.CreateCloudTableClient();
+            storageServiceLogger = logger ?? logger;
+        }
+    
+        public async Task<DetectorEntity> LoadDataToTable(DetectorEntity detectorEntity)
+        {
+            try { 
+            // Create a table client for interacting with the table service 
+            CloudTable table = tableClient.GetTableReference(tableName);
+
+            if(detectorEntity == null || detectorEntity.PartitionKey == null || detectorEntity.RowKey == null)
+            {
+                throw new ArgumentNullException(nameof(detectorEntity));
+            }
+
+                storageServiceLogger.LogInformation($"Insert or Replace {detectorEntity.RowKey} into {tableName}");
+                // Create the InsertOrReplace table operation
+                TableOperation insertOrReplaceOperation = TableOperation.InsertOrReplace(detectorEntity);
+
+                // Execute the operation.
+                TableResult result = await table.ExecuteAsync(insertOrReplaceOperation);
+
+                storageServiceLogger.LogInformation($"InsertOrReplace result : {result.HttpStatusCode}");
+                DetectorEntity insertedCustomer = result.Result as DetectorEntity;          
+                return detectorEntity;
+            }
+            catch (Exception ex)
+            {
+                storageServiceLogger.LogError(ex.ToString());
+                return null;
+            }
+        }
+
+        public async Task<DetectorEntity> GetEntityFromTable(string partitionKey, string rowKey)
+        {
+            try
+            {
+                CloudTable table = tableClient.GetTableReference(tableName);
+
+                if(string.IsNullOrWhiteSpace(partitionKey) || string.IsNullOrWhiteSpace(rowKey))
+                {
+                    throw new ArgumentNullException($"{nameof(partitionKey)} or {nameof(rowKey)} is either null or empty");
+                }
+
+                storageServiceLogger.LogInformation($"Retrieving info from table for {rowKey}, {partitionKey}");
+                TableOperation retrieveOperation = TableOperation.Retrieve<DetectorEntity>(partitionKey, rowKey);
+                // Execute the operation.
+                TableResult result = await table.ExecuteAsync(retrieveOperation);
+                DetectorEntity existingEntity = result.Result as DetectorEntity;
+                return existingEntity;
+            } catch (Exception ex)
+            {
+                storageServiceLogger.LogError(ex.ToString());
+                return null;
+            }
+        }
+
+    }
+}

src/SourceWatcherFuncApp/SourceWatcherFuncApp.csproj

@@ -0,0 +1,33 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <AzureFunctionsVersion>v2</AzureFunctionsVersion>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Azure.Storage.Blobs" Version="12.4.0" />
+    <PackageReference Include="Microsoft.Azure.Functions.Extensions" Version="1.0.0" />
+    <PackageReference Include="Microsoft.Azure.Storage.Blob" Version="11.1.3" />
+    <PackageReference Include="Microsoft.Azure.Storage.Common" Version="11.1.3" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.AzureAppConfiguration" Version="3.0.0" />
+    <PackageReference Include="Microsoft.NET.Sdk.Functions" Version="1.0.31" />
+    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
+    <PackageReference Include="Octokit" Version="0.46.0" />
+    <PackageReference Include="WindowsAzure.Storage" Version="9.3.3" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\Diagnostics.DataProviders\Diagnostics.DataProviders.csproj" />
+    <ProjectReference Include="..\Diagnostics.Scripts\Diagnostics.Scripts.csproj" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Update="configuration.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="host.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="local.settings.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      <CopyToPublishDirectory>Never</CopyToPublishDirectory>
+    </None>
+  </ItemGroup>
+</Project>

src/SourceWatcherFuncApp/Startup.cs

@@ -0,0 +1,30 @@
+using Microsoft.Azure.Functions.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Azure.WebJobs.Host.Bindings;
+using Microsoft.Extensions.Options;
+using SourceWatcherFuncApp.Services;
+
+[assembly: FunctionsStartup(typeof(ContentModeratorFunction.Startup))]
+namespace ContentModeratorFunction
+{
+    public class Startup : FunctionsStartup
+    {
+        public IConfigurationRoot Configuration;
+
+        public override void Configure(IFunctionsHostBuilder builder)
+        {
+            var executionContextOptions = builder.Services.BuildServiceProvider().GetService<IOptions<ExecutionContextOptions>>().Value;
+            var appDirectory = executionContextOptions.AppDirectory;
+            Configuration = new ConfigurationBuilder()
+                           .SetBasePath(appDirectory)
+                           .AddJsonFile("configuration.json", optional: true, reloadOnChange: true)
+                           .AddEnvironmentVariables().Build();
+            builder.Services.AddLogging();
+            builder.Services.AddSingleton(Configuration);
+            builder.Services.AddSingleton<IGithubService, GithubService>();
+            builder.Services.AddSingleton<ITableStorageService, TableStorageService>();
+            builder.Services.AddSingleton<IBlobService, BlobService>();
+        }
+    }
+}

src/SourceWatcherFuncApp/Utilities/EntityHelper.cs

@@ -0,0 +1,49 @@
+using System;
+using System.Reflection;
+using System.IO;
+using Diagnostics.Scripts;
+using SourceWatcherFuncApp.Entities;
+using Diagnostics.Scripts.Models;
+using Diagnostics.ModelsAndUtils.ScriptUtilities;
+using System.Collections.Generic;
+using Diagnostics.ModelsAndUtils.Attributes;
+
+namespace SourceWatcherFuncApp.Utilities
+{
+    public static class EntityHelper
+    {
+        public static DetectorEntity PrepareEntityForLoad(Stream streamAssemblyData, string detectorScript, DetectorEntity detectorPackage)
+        {
+            byte[] assemblyData = GetByteFromStream(streamAssemblyData);
+            Assembly temp = Assembly.Load(assemblyData);
+
+            if (!Enum.TryParse(detectorPackage.EntityType, true, out EntityType entityType))
+            {
+                entityType = EntityType.Signal;
+            }
+            EntityMetadata metaData = new EntityMetadata(detectorScript, entityType);
+            using (var invoker = new EntityInvoker(metaData, ScriptHelper.GetFrameworkReferences(), ScriptHelper.GetFrameworkImports()))
+            {
+                invoker.InitializeEntryPoint(temp);
+                var resourceFilter = invoker.ResourceFilter;
+                detectorPackage.IsInternal = resourceFilter != null? resourceFilter.InternalOnly : false;
+                detectorPackage.ResourceType = resourceFilter != null ? resourceFilter.ResourceType.ToString() : "";
+                detectorPackage.SupportTopicList = invoker.EntryPointDefinitionAttribute != null ? invoker.EntryPointDefinitionAttribute.SupportTopicList : new List<SupportTopic>() ;
+                detectorPackage.AnalysisTypes = invoker.EntryPointDefinitionAttribute != null ? invoker.EntryPointDefinitionAttribute.AnalysisTypes : new List<string>();
+            }
+            detectorPackage.PartitionKey = detectorPackage.EntityType;
+            detectorPackage.RowKey = detectorPackage.DetectorId;
+            return detectorPackage;
+
+        }
+
+        public static byte[] GetByteFromStream(Stream input)
+        {
+            using (MemoryStream ms = new MemoryStream())
+            {
+                input.CopyTo(ms);
+                return ms.ToArray();
+            }
+        }
+    }
+}

src/SourceWatcherFuncApp/host.json

@@ -0,0 +1,16 @@
+{
+    "version": "2.0",
+  "logging": {
+      "fileLoggingMode": "debugOnly",
+        "logLevel": {
+          "Function": "Information",
+          "default": "Information"
+        },
+    "applicationInsights": {
+      "samplingExcludedTypes": "Request",
+      "samplingSettings": {
+        "isEnabled": true
+      }
+    }
+  }
+}