diff --git a/UsageDataCollector/Project/Collector/BulkImport/App.config b/UsageDataCollector/Project/Collector/BulkImport/App.config index 155e6f8..217047a 100644 --- a/UsageDataCollector/Project/Collector/BulkImport/App.config +++ b/UsageDataCollector/Project/Collector/BulkImport/App.config @@ -1,6 +1,6 @@  - + \ No newline at end of file diff --git a/UsageDataCollector/Project/Collector/CollectorServiceLibrary/CollectorServiceLibrary.csproj b/UsageDataCollector/Project/Collector/CollectorServiceLibrary/CollectorServiceLibrary.csproj index f70dbf9..a8adaae 100644 --- a/UsageDataCollector/Project/Collector/CollectorServiceLibrary/CollectorServiceLibrary.csproj +++ b/UsageDataCollector/Project/Collector/CollectorServiceLibrary/CollectorServiceLibrary.csproj @@ -46,6 +46,8 @@ + + diff --git a/UsageDataCollector/Project/Collector/CollectorServiceLibrary/Import/BulkImport.cs b/UsageDataCollector/Project/Collector/CollectorServiceLibrary/Import/BulkImport.cs index 45b7341..94c6592 100644 --- a/UsageDataCollector/Project/Collector/CollectorServiceLibrary/Import/BulkImport.cs +++ b/UsageDataCollector/Project/Collector/CollectorServiceLibrary/Import/BulkImport.cs @@ -20,26 +20,23 @@ namespace ICSharpCode.UsageDataCollector.ServiceLibrary.Import public static void SketchOut() { // wrong schema errors by moving contracts to /contracts namespace - // UsageDataMessage currentMessage = FileImporter.ReadMessage(@"D:\Daten\SharpDevelop\trunk\SharpDevelopServers\UsageDataCollector\Project\Collector\CollectorServiceTestClient\SharpDevelopUsageData.xml.gz"); + UsageDataMessage message = + FileImporter.ReadMessage(@"D:\Daten\SharpDevelop\trunk\SharpDevelopServers\UsageDataCollector\SampleData\_Debugger_Exception_ab7a92f4-3d0e-44ac-afc9-a4d6090603b0.xml.gz"); using (var context = CollectorRepository.CreateContext()) { + CollectorRepository repo = new CollectorRepository(); + repo.Context = context; + + CrackAndStoreMessage processor = new CrackAndStoreMessage(message, repo); + processor.ProcessMessage(); + + // Dictionary features = context.Features.ToDictionary(f => f.Name, f => f.Id); // var features = context.Features.ToList().AsReadOnly(); // features: a, b, c // usage features: b, c, d --> find d - List knownFeatures = context.Features.Select(f => f.Name).ToList(); - - /* - var activationMethod = new ActivationMethod() - { - Name = "test" - }; - - context.ActivationMethods.AddObject(activationMethod); - context.SaveChanges(); - */ - + // List knownFeatures = context.Features.Select(f => f.Name).ToList(); } } } diff --git a/UsageDataCollector/Project/Collector/CollectorServiceLibrary/Import/CrackAndStoreMessage.cs b/UsageDataCollector/Project/Collector/CollectorServiceLibrary/Import/CrackAndStoreMessage.cs new file mode 100644 index 0000000..dced98b --- /dev/null +++ b/UsageDataCollector/Project/Collector/CollectorServiceLibrary/Import/CrackAndStoreMessage.cs @@ -0,0 +1,139 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using ICSharpCode.UsageDataCollector.Contracts; +using ICSharpCode.UsageDataCollector.DataAccess.Collector; + +namespace ICSharpCode.UsageDataCollector.ServiceLibrary.Import +{ + public class CrackAndStoreMessage + { + UsageDataMessage message = null; + CollectorRepository repository = null; + + public CrackAndStoreMessage(UsageDataMessage msg, CollectorRepository repo) + { + message = msg; + repository = repo; + } + + public void ProcessMessage() + { + string userGuid = message.UserID.ToString(); + if (String.IsNullOrEmpty(userGuid)) + { + return; + } + + // Preprocessing of type tables (don't insert any usage data unless type updates went through properly) + PreProcessEnvironmentDataNames(); + PreProcessActivationMethods(); + PreProcessFeatures(); + // TODO: Exceptions + + User modelUser = repository.FindUserByGuid(userGuid); + if (null == modelUser) + { + modelUser = new User() + { + AssociatedGuid = userGuid + }; + + repository.Context.Users.AddObject(modelUser); + + // we intentionally don't build the full model in memory first (user -> sessions -> data tables) + // avoiding concurrency issues (eg type tables) is more important than fewer database writes + repository.Context.SaveChanges(); + } + } + + protected void PreProcessEnvironmentDataNames() + { + List distinctMsgEnvProperties = (from s in message.Sessions + from p in s.EnvironmentProperties + select p.Name).Distinct().ToList(); + + // did we receive environment data at all? + if (distinctMsgEnvProperties.Count > 0) + { + List knownDataNames = repository.GetEnvironmentDataNames().ToList(); // cacheable + List missing = distinctMsgEnvProperties.Except(knownDataNames).ToList(); + + // this happens rarely for environment data names + if (missing.Count > 0) + { + foreach (string envdn in missing) + { + EnvironmentDataName modelEdn = new EnvironmentDataName() + { + Name = envdn + }; + + repository.Context.EnvironmentDataNames.AddObject(modelEdn); + } + + repository.Context.SaveChanges(); + } + } + } + + protected void PreProcessActivationMethods() + { + List distinctMsgActivationMethods = (from s in message.Sessions + from fu in s.FeatureUses + select fu.ActivationMethod).Distinct().ToList(); + + if (distinctMsgActivationMethods.Count > 0) + { + List knownActivationMethods = repository.GetActivationMethodNames().ToList(); // cacheable + List missing = distinctMsgActivationMethods.Except(knownActivationMethods).ToList(); + + if (missing.Count > 0) + { + foreach (string am in missing) + { + ActivationMethod modelAM = new ActivationMethod() + { + Name = am + }; + + repository.Context.ActivationMethods.AddObject(modelAM); + } + + repository.Context.SaveChanges(); + } + } + } + + protected void PreProcessFeatures() + { + List distinctMsgFeatures = (from s in message.Sessions + from fu in s.FeatureUses + select fu.FeatureName).Distinct().ToList(); + + if (distinctMsgFeatures.Count > 0) + { + List knownFeatures = repository.GetFeatureNames().ToList(); // cacheable + List missing = distinctMsgFeatures.Except(knownFeatures).ToList(); + + if (missing.Count > 0) + { + foreach (string fn in missing) + { + Feature modelFeature = new Feature() + { + Name = fn + }; + + repository.Context.Features.AddObject(modelFeature); + } + + repository.Context.SaveChanges(); + } + } + } + + } +} diff --git a/UsageDataCollector/Project/Collector/CollectorServiceLibrary/Import/ExceptionGroupImport.cs b/UsageDataCollector/Project/Collector/CollectorServiceLibrary/Import/ExceptionGroupImport.cs new file mode 100644 index 0000000..a3a18e4 --- /dev/null +++ b/UsageDataCollector/Project/Collector/CollectorServiceLibrary/Import/ExceptionGroupImport.cs @@ -0,0 +1,67 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace ICSharpCode.UsageDataCollector.ServiceLibrary.Import +{ + sealed class ExceptionGroupImport + { + public string Fingerprint; + + public string CrashID + { + get { return unchecked((uint)this.Fingerprint.GetHashCode() % 10000u).ToString("d4"); } + } + + public string Type + { + get + { + return ExceptionHelpers.SplitLines(this.Fingerprint).First(); + } + } + + static readonly Type[] argumentExceptions = { typeof(ArgumentException), typeof(ArgumentNullException), typeof(ArgumentOutOfRangeException) }; + + public string Location + { + get + { + List stackTrace = ExceptionHelpers.SplitLines(this.Fingerprint).Skip(1).ToList(); + // ignore any ThrowHelper (etc.) methods at the top of the stack + if (stackTrace.Count > 0 && GetFunctionName(stackTrace[0]).Contains("Throw")) + stackTrace.RemoveAt(0); + + if (stackTrace.Count == 0) + return "unknown"; + string type = this.Type; + if (argumentExceptions.Any(e => e.FullName == type) && ExceptionHelpers.IsUserCode(stackTrace[0])) + { + // find first stack frame supplying the invalid argument + string functionName = GetFunctionName(stackTrace[0]); + string result = stackTrace.FirstOrDefault(l => GetFunctionName(l) != functionName); + // report it if it's user code + if (result != null && ExceptionHelpers.IsUserCode(result)) + return result; + else + return stackTrace[0]; + } + else + { + // report first user-code stack frame + return stackTrace.FirstOrDefault(ExceptionHelpers.IsUserCode) ?? stackTrace[0]; + } + } + } + + static string GetFunctionName(string stackTraceLine) + { + int pos = stackTraceLine.IndexOf('('); + if (pos > 0) + return stackTraceLine.Substring(0, pos); + else + return stackTraceLine; + } + } +} diff --git a/UsageDataCollector/Project/Common/DataAccess/Collector/CollectorModel.Designer.cs b/UsageDataCollector/Project/Common/DataAccess/Collector/CollectorModel.Designer.cs index a1db21c..f8f21e0 100644 --- a/UsageDataCollector/Project/Common/DataAccess/Collector/CollectorModel.Designer.cs +++ b/UsageDataCollector/Project/Common/DataAccess/Collector/CollectorModel.Designer.cs @@ -65,22 +65,6 @@ namespace ICSharpCode.UsageDataCollector.DataAccess.Collector #region ObjectSet Properties - /// - /// No Metadata Documentation available. - /// - public ObjectSet ActivationMethods - { - get - { - if ((_ActivationMethods == null)) - { - _ActivationMethods = base.CreateObjectSet("ActivationMethods"); - } - return _ActivationMethods; - } - } - private ObjectSet _ActivationMethods; - /// /// No Metadata Documentation available. /// @@ -145,22 +129,6 @@ namespace ICSharpCode.UsageDataCollector.DataAccess.Collector } private ObjectSet _FeatureUses; - /// - /// No Metadata Documentation available. - /// - public ObjectSet Sessions - { - get - { - if ((_Sessions == null)) - { - _Sessions = base.CreateObjectSet("Sessions"); - } - return _Sessions; - } - } - private ObjectSet _Sessions; - /// /// No Metadata Documentation available. /// @@ -208,18 +176,42 @@ namespace ICSharpCode.UsageDataCollector.DataAccess.Collector } } private ObjectSet _Exceptions; + + /// + /// No Metadata Documentation available. + /// + public ObjectSet Sessions + { + get + { + if ((_Sessions == null)) + { + _Sessions = base.CreateObjectSet("Sessions"); + } + return _Sessions; + } + } + private ObjectSet _Sessions; + + /// + /// No Metadata Documentation available. + /// + public ObjectSet ActivationMethods + { + get + { + if ((_ActivationMethods == null)) + { + _ActivationMethods = base.CreateObjectSet("ActivationMethods"); + } + return _ActivationMethods; + } + } + private ObjectSet _ActivationMethods; #endregion #region AddTo Methods - /// - /// Deprecated Method for adding a new object to the ActivationMethods EntitySet. Consider using the .Add method of the associated ObjectSet<T> property instead. - /// - public void AddToActivationMethods(ActivationMethod activationMethod) - { - base.AddObject("ActivationMethods", activationMethod); - } - /// /// Deprecated Method for adding a new object to the EnvironmentDatas EntitySet. Consider using the .Add method of the associated ObjectSet<T> property instead. /// @@ -252,14 +244,6 @@ namespace ICSharpCode.UsageDataCollector.DataAccess.Collector base.AddObject("FeatureUses", featureUse); } - /// - /// Deprecated Method for adding a new object to the Sessions EntitySet. Consider using the .Add method of the associated ObjectSet<T> property instead. - /// - public void AddToSessions(Session session) - { - base.AddObject("Sessions", session); - } - /// /// Deprecated Method for adding a new object to the Users EntitySet. Consider using the .Add method of the associated ObjectSet<T> property instead. /// @@ -283,6 +267,22 @@ namespace ICSharpCode.UsageDataCollector.DataAccess.Collector { base.AddObject("Exceptions", exception); } + + /// + /// Deprecated Method for adding a new object to the Sessions EntitySet. Consider using the .Add method of the associated ObjectSet<T> property instead. + /// + public void AddToSessions(Session session) + { + base.AddObject("Sessions", session); + } + + /// + /// Deprecated Method for adding a new object to the ActivationMethods EntitySet. Consider using the .Add method of the associated ObjectSet<T> property instead. + /// + public void AddToActivationMethods(ActivationMethod activationMethod) + { + base.AddObject("ActivationMethods", activationMethod); + } #endregion } @@ -306,12 +306,10 @@ namespace ICSharpCode.UsageDataCollector.DataAccess.Collector /// Create a new ActivationMethod object. /// /// Initial value of the Id property. - /// Initial value of the Name property. - public static ActivationMethod CreateActivationMethod(global::System.Int32 id, global::System.String name) + public static ActivationMethod CreateActivationMethod(global::System.Int32 id) { ActivationMethod activationMethod = new ActivationMethod(); activationMethod.Id = id; - activationMethod.Name = name; return activationMethod; } @@ -348,7 +346,7 @@ namespace ICSharpCode.UsageDataCollector.DataAccess.Collector /// /// No Metadata Documentation available. /// - [EdmScalarPropertyAttribute(EntityKeyProperty=false, IsNullable=false)] + [EdmScalarPropertyAttribute(EntityKeyProperty=false, IsNullable=true)] [DataMemberAttribute()] public global::System.String Name { @@ -360,7 +358,7 @@ namespace ICSharpCode.UsageDataCollector.DataAccess.Collector { OnNameChanging(value); ReportPropertyChanging("Name"); - _Name = StructuralObject.SetValidValue(value, false); + _Name = StructuralObject.SetValidValue(value, true); ReportPropertyChanged("Name"); OnNameChanged(); } @@ -1256,14 +1254,14 @@ namespace ICSharpCode.UsageDataCollector.DataAccess.Collector /// /// Create a new Session object. /// - /// Initial value of the Id property. + /// Initial value of the SessionId property. /// Initial value of the ClientSessionId property. /// Initial value of the StartTime property. /// Initial value of the UserId property. - public static Session CreateSession(global::System.Int32 id, global::System.Int32 clientSessionId, global::System.DateTime startTime, global::System.Int32 userId) + public static Session CreateSession(global::System.Int32 sessionId, global::System.Int64 clientSessionId, global::System.DateTime startTime, global::System.Int32 userId) { Session session = new Session(); - session.Id = id; + session.SessionId = sessionId; session.ClientSessionId = clientSessionId; session.StartTime = startTime; session.UserId = userId; @@ -1278,34 +1276,34 @@ namespace ICSharpCode.UsageDataCollector.DataAccess.Collector /// [EdmScalarPropertyAttribute(EntityKeyProperty=true, IsNullable=false)] [DataMemberAttribute()] - public global::System.Int32 Id + public global::System.Int32 SessionId { get { - return _Id; + return _SessionId; } set { - if (_Id != value) + if (_SessionId != value) { - OnIdChanging(value); - ReportPropertyChanging("Id"); - _Id = StructuralObject.SetValidValue(value); - ReportPropertyChanged("Id"); - OnIdChanged(); + OnSessionIdChanging(value); + ReportPropertyChanging("SessionId"); + _SessionId = StructuralObject.SetValidValue(value); + ReportPropertyChanged("SessionId"); + OnSessionIdChanged(); } } } - private global::System.Int32 _Id; - partial void OnIdChanging(global::System.Int32 value); - partial void OnIdChanged(); + private global::System.Int32 _SessionId; + partial void OnSessionIdChanging(global::System.Int32 value); + partial void OnSessionIdChanged(); /// /// No Metadata Documentation available. /// [EdmScalarPropertyAttribute(EntityKeyProperty=false, IsNullable=false)] [DataMemberAttribute()] - public global::System.Int32 ClientSessionId + public global::System.Int64 ClientSessionId { get { @@ -1320,8 +1318,8 @@ namespace ICSharpCode.UsageDataCollector.DataAccess.Collector OnClientSessionIdChanged(); } } - private global::System.Int32 _ClientSessionId; - partial void OnClientSessionIdChanging(global::System.Int32 value); + private global::System.Int64 _ClientSessionId; + partial void OnClientSessionIdChanging(global::System.Int64 value); partial void OnClientSessionIdChanged(); /// diff --git a/UsageDataCollector/Project/Common/DataAccess/Collector/CollectorModel.edmx b/UsageDataCollector/Project/Common/DataAccess/Collector/CollectorModel.edmx index 56e6d4c..2524c33 100644 --- a/UsageDataCollector/Project/Common/DataAccess/Collector/CollectorModel.edmx +++ b/UsageDataCollector/Project/Common/DataAccess/Collector/CollectorModel.edmx @@ -20,14 +20,14 @@ - - + + - + @@ -36,14 +36,14 @@ - + - + @@ -55,7 +55,7 @@ - + @@ -66,14 +66,14 @@ - + - + @@ -84,8 +84,8 @@ - - + + @@ -94,7 +94,7 @@ - + @@ -102,23 +102,16 @@ - - + + - - - - - - - @@ -153,16 +146,6 @@ - - - - - - - - - - @@ -193,20 +176,29 @@ + + + + + + + + + + + + + + + + + - - - - - - - - @@ -245,17 +237,6 @@ - - - - - - - - - - - @@ -289,6 +270,25 @@ + + + + + + + + + + + + + + + + + + + @@ -310,15 +310,15 @@ - - + + diff --git a/UsageDataCollector/Project/Common/DataAccess/Collector/CollectorRepository.cs b/UsageDataCollector/Project/Common/DataAccess/Collector/CollectorRepository.cs index 560174f..6849e02 100644 --- a/UsageDataCollector/Project/Common/DataAccess/Collector/CollectorRepository.cs +++ b/UsageDataCollector/Project/Common/DataAccess/Collector/CollectorRepository.cs @@ -19,6 +19,19 @@ namespace ICSharpCode.UsageDataCollector.DataAccess.Collector return Context.Users.FirstOrDefault(u => u.AssociatedGuid == guid); } + public IEnumerable GetEnvironmentDataNames() + { + return Context.EnvironmentDataNames.Select(dn => dn.Name); + } + public IEnumerable GetActivationMethodNames() + { + return Context.ActivationMethods.Select(am => am.Name); + } + + public IEnumerable GetFeatureNames() + { + return Context.Features.Select(f => f.Name); + } } }