Azure
/
Azure-AppServices-Diagnostics
зеркало из https://github.com/Azure/Azure-AppServices-Diagnostics.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Add ETW Events for Source Watcher

This commit is contained in:
ShekharGupta1988 2018-03-19 23:25:12 -07:00
Родитель c4cd9b8d77
Коммит d6e444292b
4 изменённых файлов: 200 добавлений и 83 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

26
src/Diagnostics.Logger/ETWProvider/DiagnosticsETWProvider.cs
Просмотреть файл

@ -13,7 +13,7 @@ namespace Diagnostics.Logger
#region Compile Host Events (ID Range : 1000 - 1999)
[Event(1000, Level = EventLevel.Informational, Channel = EventChannel.Admin)]
public void LogCompilerHostStartup(string Message)
public void LogCompilerHostMessage(string Message)
{
WriteDiagnosticsEvent(1000, Message);
}
@ -46,7 +46,7 @@ namespace Diagnostics.Logger
#region Runtime Host Events (ID Range : 2000 - 2999)
[Event(2000, Level = EventLevel.Informational, Channel = EventChannel.Admin)]
public void LogRuntimeHostStartup(string Message)
public void LogRuntimeHostMessage(string Message)
{
WriteDiagnosticsEvent(2000, Message);
}
@ -81,6 +81,28 @@ namespace Diagnostics.Logger
EndTime);
}
#region SourceWatcher Events (ID Range : 2500 - 2599)
[Event(2500, Level = EventLevel.Informational, Channel = EventChannel.Admin)]
public void LogSourceWatcherMessage(string Source, string Message)
{
WriteDiagnosticsEvent(2500, Source, Message);
}
[Event(2501, Level = EventLevel.Warning, Channel = EventChannel.Admin)]
public void LogSourceWatcherWarning(string Source, string Message)
{
WriteDiagnosticsEvent(2501, Source, Message);
}
[Event(2502, Level = EventLevel.Error, Channel = EventChannel.Admin)]
public void LogSourceWatcherException(string Source, string Message, string ExceptionType, string ExceptionDetails)
{
WriteDiagnosticsEvent(2502, Source, Message, ExceptionType, ExceptionDetails);
}
#endregion
#endregion
#region Data Provider Events (ID Range : 3000 - 3999)

121
src/Diagnostics.RuntimeHost/Services/SourceWatcher/Watchers/GitHubWatcher.cs
Просмотреть файл

@ -16,26 +16,23 @@ using System.Threading.Tasks;
namespace Diagnostics.RuntimeHost.Services.SourceWatcher
{
public class GitHubWatcher : ISourceWatcher
public class GitHubWatcher : SourceWatcherBase
{
private Task _firstTimeCompletionTask;
private IHostingEnvironment _env;
private IConfiguration _config;
private ICache<string, EntityInvoker> _invokerCache;
private IGithubClient _githubClient;
private string _rootContentApiPath;
private string _lastModifiedMarkerName;
private string _deleteMarkerName;
private string _cacheIdFileName;
private readonly string _lastModifiedMarkerName;
private readonly string _deleteMarkerName;
private readonly string _cacheIdFileName;
private string _destinationCsxPath;
private int _pollingIntervalInSeconds;
protected override Task FirstTimeCompletionTask => _firstTimeCompletionTask;
public GitHubWatcher(IHostingEnvironment env, IConfiguration configuration, ICache<string, EntityInvoker> invokerCache, IGithubClient githubClient)
: base(env, configuration, invokerCache, "GithubWatcher")
{
_env = env;
_config = configuration;
_invokerCache = invokerCache;
_githubClient = githubClient;
_lastModifiedMarkerName = "_lastModified.marker";
@ -45,25 +42,32 @@ namespace Diagnostics.RuntimeHost.Services.SourceWatcher
Start();
}
public void Start()
public override void Start()
{
_firstTimeCompletionTask = StartWatcherInternal();
StartPollingForChanges();
}
public Task WaitForFirstCompletion() => _firstTimeCompletionTask;
private async Task StartWatcherInternal()
{
try
{
LogMessage("SourceWatcher : Start");
DirectoryInfo destDirInfo = new DirectoryInfo(_destinationCsxPath);
string destLastModifiedMarker = await FileHelper.GetFileContentAsync(destDirInfo.FullName, _lastModifiedMarkerName);
HttpResponseMessage response = await _githubClient.Get(_rootContentApiPath, etag: destLastModifiedMarker);
LogMessage($"Http call to repository root path completed. Status Code : {response.StatusCode.ToString()}");
if (response.StatusCode >= HttpStatusCode.NotFound)
{
// TODO: log fatal error
string errorContent = string.Empty;
try
{
errorContent = await response.Content.ReadAsStringAsync();
}
catch (Exception) { }
LogException($"Unexpected resonse from repository root path. Response : {errorContent}", null);
return;
}
@ -74,7 +78,7 @@ namespace Diagnostics.RuntimeHost.Services.SourceWatcher
* Make Sure this entity is loaded in Invoker cache for runtime.
* This codepath will be mostly used when the process restarts or machine reboot (and no changes are done in scripts source).
*/
LogMessage($"Checking if any invoker present locally needs to be added in cache");
foreach (DirectoryInfo subDir in destDirInfo.EnumerateDirectories())
{
await AddInvokerToCacheIfNeeded(subDir);
@ -83,6 +87,7 @@ namespace Diagnostics.RuntimeHost.Services.SourceWatcher
return;
}
LogMessage("Syncing local directories with github changes");
string githubRootContentETag = GetHeaderValue(response, HeaderConstants.EtagHeaderName).Replace("W/", string.Empty);
GithubEntry[] githubDirectories = await response.Content.ReadAsAsyncCustom<GithubEntry[]>();
@ -93,6 +98,7 @@ namespace Diagnostics.RuntimeHost.Services.SourceWatcher
DirectoryInfo subDir = new DirectoryInfo(Path.Combine(destDirInfo.FullName, gitHubDir.Name));
if (!subDir.Exists)
{
LogMessage($"Folder : {subDir.Name} present in github but not on local disk. Creating it...");
subDir.Create();
}
@ -105,6 +111,8 @@ namespace Diagnostics.RuntimeHost.Services.SourceWatcher
continue;
}
LogMessage($"Deteced changes in Github Folder : {gitHubDir.Name}. Syncing it locally ...");
await DownloadContentAndUpdateInvokerCache(gitHubDir, subDir);
await FileHelper.WriteToFileAsync(subDir.FullName, _lastModifiedMarkerName, gitHubDir.Sha);
}
@ -113,12 +121,16 @@ namespace Diagnostics.RuntimeHost.Services.SourceWatcher
await FileHelper.WriteToFileAsync(destDirInfo.FullName, _lastModifiedMarkerName, githubRootContentETag);
}
catch (Exception)
catch (Exception ex)
{
// TODO : Log and consume the exception
LogException(ex.Message, ex);
}
finally
{
LogMessage("SourceWatcher : End");
}
}
private async void StartPollingForChanges()
{
await _firstTimeCompletionTask;
@ -130,28 +142,44 @@ namespace Diagnostics.RuntimeHost.Services.SourceWatcher
} while (true);
}
private async Task AddInvokerToCacheIfNeeded(DirectoryInfo subDir)
{
string cacheId = await FileHelper.GetFileContentAsync(subDir.FullName, _cacheIdFileName);
if (string.IsNullOrWhiteSpace(cacheId) || !_invokerCache.TryGetValue(cacheId, out EntityInvoker invoker))
{
LogMessage($"Folder : {subDir.FullName} missing in invoker cache.");
FileInfo mostRecentAssembly = GetMostRecentFileByExtension(subDir, ".dll");
FileInfo csxScriptFile = GetMostRecentFileByExtension(subDir, ".csx");
FileInfo deleteMarkerFile = new FileInfo(Path.Combine(subDir.FullName, _deleteMarkerName));
if (mostRecentAssembly != default(FileInfo) && csxScriptFile != default(FileInfo) && !deleteMarkerFile.Exists)
if (mostRecentAssembly == default(FileInfo) || csxScriptFile == default(FileInfo))
{
string scriptText = await FileHelper.GetFileContentAsync(csxScriptFile.FullName);
Assembly asm = Assembly.LoadFrom(mostRecentAssembly.FullName);
invoker = new EntityInvoker(new EntityMetadata(scriptText));
invoker.InitializeEntryPoint(asm);
LogWarning($"No Assembly file (.dll) or Csx File found (.csx). Skipping cache update");
return;
}
if (invoker.EntryPointDefinitionAttribute != null)
{
_invokerCache.AddOrUpdate(invoker.EntryPointDefinitionAttribute.Id, invoker);
await FileHelper.WriteToFileAsync(subDir.FullName, _cacheIdFileName, invoker.EntryPointDefinitionAttribute.Id);
}
if (deleteMarkerFile.Exists)
{
LogMessage("Folder marked for deletion. Skipping cache update");
return;
}
string scriptText = await FileHelper.GetFileContentAsync(csxScriptFile.FullName);
LogMessage($"Loading assembly : {mostRecentAssembly.FullName}");
Assembly asm = Assembly.LoadFrom(mostRecentAssembly.FullName);
invoker = new EntityInvoker(new EntityMetadata(scriptText));
invoker.InitializeEntryPoint(asm);
if (invoker.EntryPointDefinitionAttribute != null)
{
LogMessage($"Updating cache with new invoker with id : {invoker.EntryPointDefinitionAttribute.Id}");
_invokerCache.AddOrUpdate(invoker.EntryPointDefinitionAttribute.Id, invoker);
await FileHelper.WriteToFileAsync(subDir.FullName, _cacheIdFileName, invoker.EntryPointDefinitionAttribute.Id);
}
else
{
LogWarning("Missing Entry Point Definition attribute. skipping cache update");
}
}
}
@ -167,11 +195,12 @@ namespace Diagnostics.RuntimeHost.Services.SourceWatcher
HttpResponseMessage response = await _githubClient.Get(parentGithubEntry.Url);
if (!response.IsSuccessStatusCode)
{
LogException($"GET Failed. Url : {parentGithubEntry.Url}, StatusCode : {response.StatusCode.ToString()}", null);
return;
}
GithubEntry[] githubFiles = await response.Content.ReadAsAsyncCustom<GithubEntry[]>();
foreach (GithubEntry githubFile in githubFiles)
{
string fileExtension = githubFile.Name.Split(new char[] { '.' }).LastOrDefault();
@ -191,43 +220,55 @@ namespace Diagnostics.RuntimeHost.Services.SourceWatcher
// Use Guids for Assembly and PDB Names to ensure uniqueness.
downloadFilePath = Path.Combine(destDir.FullName, $"{assemblyName}.{fileExtension.ToLower()}");
}
LogMessage($"Begin downloading File : {githubFile.Name} and saving it as : {downloadFilePath}");
await _githubClient.DownloadFile(githubFile.Download_url, downloadFilePath);
}
string scriptText = await FileHelper.GetFileContentAsync(csxFilePath);
asm = Assembly.LoadFrom(Path.Combine(destDir.FullName, $"{assemblyName}.dll"));
string assemblyPath = Path.Combine(destDir.FullName, $"{assemblyName}.dll");
LogMessage($"Loading assembly : {assemblyPath}");
asm = Assembly.LoadFrom(assemblyPath);
EntityInvoker newInvoker = new EntityInvoker(new EntityMetadata(scriptText));
newInvoker.InitializeEntryPoint(asm);
// Remove the Old Invoker from Cache
lastCacheId = await FileHelper.GetFileContentAsync(cacheIdFilePath);
if(!string.IsNullOrWhiteSpace(lastCacheId) && _invokerCache.TryRemoveValue(lastCacheId, out EntityInvoker oldInvoker))
if (!string.IsNullOrWhiteSpace(lastCacheId) && _invokerCache.TryRemoveValue(lastCacheId, out EntityInvoker oldInvoker))
{
LogMessage($"Removing old invoker with id : {oldInvoker.EntryPointDefinitionAttribute.Id} from Cache");
oldInvoker.Dispose();
}
// Add new invoker to Cache and update Cache Id File
if (newInvoker.EntryPointDefinitionAttribute != null)
{
LogMessage($"Updating cache with new invoker with id : {newInvoker.EntryPointDefinitionAttribute.Id}");
_invokerCache.AddOrUpdate(newInvoker.EntryPointDefinitionAttribute.Id, newInvoker);
await FileHelper.WriteToFileAsync(cacheIdFilePath, newInvoker.EntryPointDefinitionAttribute.Id);
}
else
{
LogWarning("Missing Entry Point Definition attribute. skipping cache update");
}
}
private async Task SyncLocalDirForDeletedEntriesInGitHub(GithubEntry[] githubDirectories, DirectoryInfo destDirInfo)
{
if (!destDirInfo.Exists) return;
foreach(DirectoryInfo subDir in destDirInfo.EnumerateDirectories())
LogMessage("Checking for deleted folders in github");
foreach (DirectoryInfo subDir in destDirInfo.EnumerateDirectories())
{
bool dirExistsInGithub = githubDirectories.Any(p => p.Name.Equals(subDir.Name));
if (!dirExistsInGithub)
{
LogMessage($"Folder : {subDir.Name} not present in github. Marking for deletion");
// remove the entry from cache and mark the folder for deletion.s
string cacheId = await FileHelper.GetFileContentAsync(subDir.FullName, _cacheIdFileName);
if(!string.IsNullOrWhiteSpace(cacheId) && _invokerCache.TryRemoveValue(cacheId, out EntityInvoker invoker))
if (!string.IsNullOrWhiteSpace(cacheId) && _invokerCache.TryRemoveValue(cacheId, out EntityInvoker invoker))
{
invoker.Dispose();
}
@ -237,7 +278,7 @@ namespace Diagnostics.RuntimeHost.Services.SourceWatcher
}
}
private string GetHeaderValue(HttpResponseMessage responseMsg, string headerName)
private string GetHeaderValue(HttpResponseMessage responseMsg, string headerName)
{
if (responseMsg.Headers.TryGetValues(headerName, out IEnumerable<string> values))
{
@ -250,9 +291,9 @@ namespace Diagnostics.RuntimeHost.Services.SourceWatcher
private FileInfo GetMostRecentFileByExtension(DirectoryInfo dir, string extension)
{
return dir.GetFiles().Where(p => (!string.IsNullOrWhiteSpace(p.Extension) && p.Extension.Equals(extension, StringComparison.OrdinalIgnoreCase)))
.OrderByDescending(f=>f.LastWriteTimeUtc).FirstOrDefault();
.OrderByDescending(f => f.LastWriteTimeUtc).FirstOrDefault();
}
private void LoadConfigurations()
{
_rootContentApiPath = $@"https://api.github.com/repos/{_githubClient.UserName}/{_githubClient.RepoName}/contents?ref={_githubClient.Branch}";
@ -268,7 +309,7 @@ namespace Diagnostics.RuntimeHost.Services.SourceWatcher
pollingIntervalvalue = (_config[$"SourceWatcher:{RegistryConstants.PollingIntervalInSecondsKey}"]).ToString();
}
if(!int.TryParse(pollingIntervalvalue, out _pollingIntervalInSeconds))
if (!int.TryParse(pollingIntervalvalue, out _pollingIntervalInSeconds))
{
_pollingIntervalInSeconds = HostConstants.WatcherDefaultPollingIntervalInSeconds;
}

85
src/Diagnostics.RuntimeHost/Services/SourceWatcher/Watchers/LocalFileSystemWatcher.cs
Просмотреть файл

@ -12,65 +12,68 @@ using System.Threading.Tasks;
namespace Diagnostics.RuntimeHost.Services.SourceWatcher
{
public class LocalFileSystemWatcher : ISourceWatcher
public class LocalFileSystemWatcher : SourceWatcherBase
{
private Task _firstTimeCompletionTask;
private IHostingEnvironment _env;
private IConfiguration _config;
private ICache<string, EntityInvoker> _invokerCache;
private string _localScriptsPath;
private Task _firstTimeCompletionTask;
protected override Task FirstTimeCompletionTask => _firstTimeCompletionTask;
public LocalFileSystemWatcher(IHostingEnvironment env, IConfiguration configuration, ICache<string, EntityInvoker> invokerCache)
: base(env, configuration, invokerCache, "LocalFileSystemWatcher")
{
_env = env;
_config = configuration;
_invokerCache = invokerCache;
LoadConfigurations();
Start();
}
public LocalFileSystemWatcher(string localScriptsSourcePath, ICache<string, EntityInvoker> invokerCache)
{
_localScriptsPath = localScriptsSourcePath;
_invokerCache = invokerCache;
Start();
}
public void Start()
public override void Start()
{
_firstTimeCompletionTask = StartWatcherInternal();
}
public Task WaitForFirstCompletion() => _firstTimeCompletionTask;
private async Task StartWatcherInternal()
{
DirectoryInfo srcDirectoryInfo = new DirectoryInfo(_localScriptsPath);
foreach (DirectoryInfo srcSubDirInfo in srcDirectoryInfo.GetDirectories())
try
{
var files = srcSubDirInfo.GetFiles().OrderByDescending(p => p.LastWriteTimeUtc);
var csxFile = files.FirstOrDefault(p => p.Extension.Equals(".csx", StringComparison.OrdinalIgnoreCase));
var asmFile = files.FirstOrDefault(p => p.Extension.Equals(".dll", StringComparison.OrdinalIgnoreCase));
LogMessage("SourceWatcher : Start");
string scriptText = string.Empty;
if(csxFile != default(FileInfo))
DirectoryInfo srcDirectoryInfo = new DirectoryInfo(_localScriptsPath);
foreach (DirectoryInfo srcSubDirInfo in srcDirectoryInfo.GetDirectories())
{
scriptText = await File.ReadAllTextAsync(csxFile.FullName);
LogMessage($"Scanning in folder : {srcSubDirInfo.FullName}");
var files = srcSubDirInfo.GetFiles().OrderByDescending(p => p.LastWriteTimeUtc);
var csxFile = files.FirstOrDefault(p => p.Extension.Equals(".csx", StringComparison.OrdinalIgnoreCase));
var asmFile = files.FirstOrDefault(p => p.Extension.Equals(".dll", StringComparison.OrdinalIgnoreCase));
string scriptText = string.Empty;
if (csxFile != default(FileInfo))
{
scriptText = await File.ReadAllTextAsync(csxFile.FullName);
}
EntityMetadata scriptMetadata = new EntityMetadata(scriptText);
EntityInvoker invoker = new EntityInvoker(scriptMetadata);
if (asmFile == default(FileInfo))
{
LogWarning($"No Assembly file (.dll). Skipping cache update");
continue;
}
LogMessage($"Loading assembly : {asmFile.FullName}");
Assembly asm = Assembly.LoadFrom(asmFile.FullName);
invoker.InitializeEntryPoint(asm);
LogMessage($"Updating cache with new invoker with id : {invoker.EntryPointDefinitionAttribute.Id}");
_invokerCache.AddOrUpdate(invoker.EntryPointDefinitionAttribute.Id, invoker);
}
EntityMetadata scriptMetadata = new EntityMetadata(scriptText);
EntityInvoker invoker = new EntityInvoker(scriptMetadata);
if(asmFile == default(FileInfo))
{
// TODO : Log Error of missing dll
continue;
}
Assembly asm = Assembly.LoadFrom(asmFile.FullName);
invoker.InitializeEntryPoint(asm);
_invokerCache.AddOrUpdate(invoker.EntryPointDefinitionAttribute.Id, invoker);
}
catch (Exception ex)
{
LogException(ex.Message, ex);
}
finally
{
LogMessage("SourceWatcher : End");
}
}

51
src/Diagnostics.RuntimeHost/Services/SourceWatcher/Watchers/SourceWatcherBase.cs Normal file
Просмотреть файл

@ -0,0 +1,51 @@
using Diagnostics.Logger;
using Diagnostics.Scripts;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.Configuration;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
namespace Diagnostics.RuntimeHost.Services.SourceWatcher
{
public abstract class SourceWatcherBase : ISourceWatcher
{
protected IHostingEnvironment _env;
protected IConfiguration _config;
protected ICache<string, EntityInvoker> _invokerCache;
protected string _eventSource;
protected abstract Task FirstTimeCompletionTask { get; }
public abstract void Start();
public Task WaitForFirstCompletion() => FirstTimeCompletionTask;
public SourceWatcherBase(IHostingEnvironment env, IConfiguration configuration, ICache<string, EntityInvoker> invokerCache, string eventSource)
{
_env = env;
_config = configuration;
_invokerCache = invokerCache;
_eventSource = eventSource;
}
protected void LogMessage(string message)
{
DiagnosticsETWProvider.Instance.LogSourceWatcherMessage(_eventSource, message);
}
protected void LogWarning(string message)
{
DiagnosticsETWProvider.Instance.LogSourceWatcherWarning(_eventSource, message);
}
protected void LogException(string message, Exception ex)
{
string exceptionType = ex != null ? ex.GetType().ToString() : string.Empty;
string exceptionDetails = ex != null ? ex.ToString() : string.Empty;
DiagnosticsETWProvider.Instance.LogSourceWatcherException(_eventSource, message, exceptionType, exceptionDetails);
}
}
}