From 46a066fa136c8dccd5829a2f7ec13f2f1b08a2b5 Mon Sep 17 00:00:00 2001 From: Christian Wade Date: Tue, 18 Apr 2017 18:48:52 -0700 Subject: [PATCH] 1400 v3 --- .../SampleConfiguration.sql | 2 + .../usp_LastProcessingLogs.sql | 24 ++- .../dbo/Tables/ModelConfiguration.sql | 2 + .../dbo/Tables/ProcessingLog.sql | 1 + .../dbo/Views/vPartitioningConfiguration.sql | 2 + .../Program.cs | 6 +- .../ConfigDatabaseHelper.cs | 28 ++- .../ModelConfiguration.cs | 16 ++ .../PartitionProcessor.cs | 175 ++++++++++-------- 9 files changed, 164 insertions(+), 92 deletions(-) diff --git a/AsPartitionProcessing/AsPartitionProcessing.ConfigurationLogging/SampleConfiguration.sql b/AsPartitionProcessing/AsPartitionProcessing.ConfigurationLogging/SampleConfiguration.sql index 036f20c..b660671 100644 --- a/AsPartitionProcessing/AsPartitionProcessing.ConfigurationLogging/SampleConfiguration.sql +++ b/AsPartitionProcessing/AsPartitionProcessing.ConfigurationLogging/SampleConfiguration.sql @@ -8,6 +8,8 @@ VALUES( ,1 --[IntegratedAuth] ,-1 --[MaxParallelism] ,-1 --[CommitTimeout] + ,0 --[RetryAttempts] + ,0 --[RetryWaitTimeSeconds] ); INSERT INTO [dbo].[TableConfiguration] diff --git a/AsPartitionProcessing/AsPartitionProcessing.ConfigurationLogging/dbo/Stored Procedures/usp_LastProcessingLogs.sql b/AsPartitionProcessing/AsPartitionProcessing.ConfigurationLogging/dbo/Stored Procedures/usp_LastProcessingLogs.sql index 21163ae..c2cb2fb 100644 --- a/AsPartitionProcessing/AsPartitionProcessing.ConfigurationLogging/dbo/Stored Procedures/usp_LastProcessingLogs.sql +++ b/AsPartitionProcessing/AsPartitionProcessing.ConfigurationLogging/dbo/Stored Procedures/usp_LastProcessingLogs.sql @@ -1,8 +1,16 @@ -CREATE PROC [dbo].[usp_LastProcessingLogs] AS - SELECT [Message] - FROM [dbo].[ProcessingLog] - WHERE ExecutionID = - ( SELECT MAX([ExecutionID]) FROM [dbo].[ProcessingLog] - WHERE [LogDateTime] = (SELECT MAX([LogDateTime]) FROM [dbo].[ProcessingLog]) - ) - ORDER BY [LogDateTime] \ No newline at end of file +CREATE PROC [dbo].[usp_LastProcessingLogs] + @ExecutionCount tinyint = 1, + @ErrorsOnly bit = 0 +AS + SELECT --l.ExecutionID, + l.[LogDateTime], + l.[Message] + FROM [dbo].[ProcessingLog] l + INNER JOIN + ( SELECT TOP (@ExecutionCount) [ExecutionID], MAX([LogDateTime]) [MaxLogDateTime] + FROM [dbo].[ProcessingLog] + GROUP BY ExecutionID + ORDER BY [MaxLogDateTime] DESC + ) dt ON l.ExecutionID = dt.ExecutionID + WHERE @ErrorsOnly = 0 OR (@ErrorsOnly = 1 AND l.MessageType = 'Error') + ORDER BY [LogDateTime] diff --git a/AsPartitionProcessing/AsPartitionProcessing.ConfigurationLogging/dbo/Tables/ModelConfiguration.sql b/AsPartitionProcessing/AsPartitionProcessing.ConfigurationLogging/dbo/Tables/ModelConfiguration.sql index 9eb63ee..7d7a677 100644 --- a/AsPartitionProcessing/AsPartitionProcessing.ConfigurationLogging/dbo/Tables/ModelConfiguration.sql +++ b/AsPartitionProcessing/AsPartitionProcessing.ConfigurationLogging/dbo/Tables/ModelConfiguration.sql @@ -7,6 +7,8 @@ [IntegratedAuth] BIT NOT NULL, [MaxParallelism] INT NOT NULL, [CommitTimeout] INT NOT NULL, + [RetryAttempts] TINYINT NOT NULL, + [RetryWaitTimeSeconds] INT NOT NULL, CONSTRAINT [PK_ModelConfiguration] PRIMARY KEY CLUSTERED ([ModelConfigurationID] ASC) ); diff --git a/AsPartitionProcessing/AsPartitionProcessing.ConfigurationLogging/dbo/Tables/ProcessingLog.sql b/AsPartitionProcessing/AsPartitionProcessing.ConfigurationLogging/dbo/Tables/ProcessingLog.sql index e66ce8f..284f764 100644 --- a/AsPartitionProcessing/AsPartitionProcessing.ConfigurationLogging/dbo/Tables/ProcessingLog.sql +++ b/AsPartitionProcessing/AsPartitionProcessing.ConfigurationLogging/dbo/Tables/ProcessingLog.sql @@ -4,6 +4,7 @@ [ExecutionID] CHAR (36) NOT NULL, [LogDateTime] DATETIME NOT NULL, [Message] VARCHAR (8000) NOT NULL, + [MessageType] NVARCHAR(50) NOT NULL, CONSTRAINT [PK_ProcessingLog] PRIMARY KEY CLUSTERED ([PartitioningLogID] ASC), CONSTRAINT [FK_ProcessingLog_ModelConfiguration] FOREIGN KEY ([ModelConfigurationID]) REFERENCES [dbo].[ModelConfiguration] ([ModelConfigurationID]) ); diff --git a/AsPartitionProcessing/AsPartitionProcessing.ConfigurationLogging/dbo/Views/vPartitioningConfiguration.sql b/AsPartitionProcessing/AsPartitionProcessing.ConfigurationLogging/dbo/Views/vPartitioningConfiguration.sql index 890b767..8d39b76 100644 --- a/AsPartitionProcessing/AsPartitionProcessing.ConfigurationLogging/dbo/Views/vPartitioningConfiguration.sql +++ b/AsPartitionProcessing/AsPartitionProcessing.ConfigurationLogging/dbo/Views/vPartitioningConfiguration.sql @@ -11,6 +11,8 @@ SELECT m.[ModelConfigurationID] ,m.[IntegratedAuth] ,m.[MaxParallelism] ,m.[CommitTimeout] + ,m.[RetryAttempts] + ,m.[RetryWaitTimeSeconds] ,t.[TableConfigurationID] ,t.[AnalysisServicesTable] ,t.[DoNotProcess] diff --git a/AsPartitionProcessing/AsPartitionProcessing.SampleClient/Program.cs b/AsPartitionProcessing/AsPartitionProcessing.SampleClient/Program.cs index ecb880f..d672bca 100644 --- a/AsPartitionProcessing/AsPartitionProcessing.SampleClient/Program.cs +++ b/AsPartitionProcessing/AsPartitionProcessing.SampleClient/Program.cs @@ -138,6 +138,8 @@ namespace AsPartitionProcessing.SampleClient password: "", maxParallelism: -1, commitTimeout: -1, + retryAttempts: 0, + retryWaitTimeSeconds: 0, tableConfigurations: new List { @@ -279,7 +281,7 @@ namespace AsPartitionProcessing.SampleClient } } - private static void LogMessage(string message, ModelConfiguration partitionedModel) + private static void LogMessage(string message, MessageType messageType, ModelConfiguration partitionedModel) { //Can provide custom logger here @@ -287,7 +289,7 @@ namespace AsPartitionProcessing.SampleClient { if (!(_executionMode == ExecutionMode.InitializeInline)) { - ConfigDatabaseHelper.LogMessage(message, partitionedModel); + ConfigDatabaseHelper.LogMessage(message, messageType, partitionedModel); } Console.WriteLine(message); diff --git a/AsPartitionProcessing/AsPartitionProcessing/ConfigDatabaseHelper.cs b/AsPartitionProcessing/AsPartitionProcessing/ConfigDatabaseHelper.cs index f935803..1524888 100644 --- a/AsPartitionProcessing/AsPartitionProcessing/ConfigDatabaseHelper.cs +++ b/AsPartitionProcessing/AsPartitionProcessing/ConfigDatabaseHelper.cs @@ -27,6 +27,8 @@ namespace AsPartitionProcessing ,[IntegratedAuth] ,[MaxParallelism] ,[CommitTimeout] + ,[RetryAttempts] + ,[RetryWaitTimeSeconds] ,[TableConfigurationID] ,[AnalysisServicesTable] ,[Partitioned] @@ -37,7 +39,6 @@ namespace AsPartitionProcessing ,[MaxDateIsNow] ,[MaxDate] ,[IntegerDateKey] - ,[TemplateSourceQuery] FROM [dbo].[vPartitioningConfiguration] WHERE [DoNotProcess] = 0 {0} @@ -79,6 +80,8 @@ namespace AsPartitionProcessing modelConfig.IntegratedAuth = Convert.ToBoolean(reader["IntegratedAuth"]); modelConfig.MaxParallelism = Convert.ToInt32(reader["MaxParallelism"]); modelConfig.CommitTimeout = Convert.ToInt32(reader["CommitTimeout"]); + modelConfig.RetryAttempts = Convert.ToInt32(reader["RetryAttempts"]); + modelConfig.RetryWaitTimeSeconds = Convert.ToInt32(reader["RetryWaitTimeSeconds"]); modelConfig.ConfigDatabaseConnectionInfo = connectionInfo; currentModelConfigurationID = modelConfig.ModelConfigurationID; @@ -142,7 +145,7 @@ namespace AsPartitionProcessing /// /// Message to be logged. /// Partitioned model with configuration information. - public static void LogMessage(string message, ModelConfiguration partitionedModel) + public static void LogMessage(string message, MessageType messageType, ModelConfiguration partitionedModel) { using (var connection = new SqlConnection(GetConnectionString(partitionedModel.ConfigDatabaseConnectionInfo))) { @@ -156,13 +159,15 @@ namespace AsPartitionProcessing ([ModelConfigurationID] ,[ExecutionID] ,[LogDateTime] - ,[Message]) + ,[Message] + ,[MessageType]) VALUES (@ModelConfigurationID ,@ExecutionID ,@LogDateTime - ,@Message);"; - + ,@Message + ,@MessageType);"; + SqlParameter parameter; parameter = new SqlParameter("@ModelConfigurationID", SqlDbType.Int); @@ -181,6 +186,10 @@ namespace AsPartitionProcessing parameter.Value = message; command.Parameters.Add(parameter); + parameter = new SqlParameter("@MessageType", SqlDbType.VarChar, 50); + parameter.Value = messageType.ToString(); + command.Parameters.Add(parameter); + command.ExecuteNonQuery(); } } @@ -201,4 +210,13 @@ namespace AsPartitionProcessing return connectionString; } } + + /// + /// Enumeration of log message types. + /// + public enum MessageType + { + Informational, + Error + } } diff --git a/AsPartitionProcessing/AsPartitionProcessing/ModelConfiguration.cs b/AsPartitionProcessing/AsPartitionProcessing/ModelConfiguration.cs index f7b156e..76c737f 100644 --- a/AsPartitionProcessing/AsPartitionProcessing/ModelConfiguration.cs +++ b/AsPartitionProcessing/AsPartitionProcessing/ModelConfiguration.cs @@ -58,6 +58,16 @@ namespace AsPartitionProcessing /// public int CommitTimeout { get; set; } + /// + /// Number of times a retry of the processing operation will be performed if an error occurs. Use for near-real time scenarios and environments with network reliability issues. + /// + public int RetryAttempts { get; set; } + + /// + /// Number of seconds to wait before a retry attempt. + /// + public int RetryWaitTimeSeconds { get; set; } + /// /// Collection of partitioned tables containing configuration information. /// @@ -86,6 +96,8 @@ namespace AsPartitionProcessing /// Only applies when integratedAuth=false. Used for Azure AD UPNs to connect to Azure AS. /// Sets the maximum number of threads on which to run processing commands in parallel. -1 will not set the value. /// Set to override of CommitTimeout server property value for the connection. -1 will not override; the server value will be used. + /// Number of times a retry of the processing operation will be performed if an error occurs. Use for near-real time scenarios and environments with network reliability issues. + /// Number of seconds to wait before a retry attempt. /// Collection of partitioned tables containing configuration information. public ModelConfiguration( int modelConfigurationID, @@ -98,6 +110,8 @@ namespace AsPartitionProcessing string password, int maxParallelism, int commitTimeout, + int retryAttempts, + int retryWaitTimeSeconds, List tableConfigurations ) { @@ -111,6 +125,8 @@ namespace AsPartitionProcessing Password = password; MaxParallelism = maxParallelism; CommitTimeout = commitTimeout; + RetryAttempts = retryAttempts; + RetryWaitTimeSeconds = retryWaitTimeSeconds; TableConfigurations = tableConfigurations; ExecutionID = Guid.NewGuid().ToString(); } diff --git a/AsPartitionProcessing/AsPartitionProcessing/PartitionProcessor.cs b/AsPartitionProcessing/AsPartitionProcessing/PartitionProcessor.cs index a3215f9..360e795 100644 --- a/AsPartitionProcessing/AsPartitionProcessing/PartitionProcessor.cs +++ b/AsPartitionProcessing/AsPartitionProcessing/PartitionProcessor.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using Microsoft.AnalysisServices.Tabular; +using System.Threading; //----------- @@ -20,7 +21,7 @@ namespace AsPartitionProcessing /// /// The message to be logged /// Configuration info for the model - public delegate void LogMessageDelegate(string message, ModelConfiguration modelConfiguration); + public delegate void LogMessageDelegate(string message, MessageType messageType, ModelConfiguration modelConfiguration); /// /// Processor of partitions in AS tabular models @@ -31,6 +32,7 @@ namespace AsPartitionProcessing private static ModelConfiguration _modelConfiguration; private static LogMessageDelegate _messageLogger; + private static int _retryAttempts; #endregion @@ -45,7 +47,13 @@ namespace AsPartitionProcessing { _modelConfiguration = modelConfiguration; _messageLogger = messageLogger; + _retryAttempts = modelConfiguration.RetryAttempts; + PerformProcessing(); + } + + private static void PerformProcessing() + { Server server = new Server(); try { @@ -53,9 +61,9 @@ namespace AsPartitionProcessing Connect(server, out database); Console.ForegroundColor = ConsoleColor.White; - LogMessage($"Start: {DateTime.Now.ToString("hh:mm:ss tt")}", false); - LogMessage($"Server: {_modelConfiguration.AnalysisServicesServer}", false); - LogMessage($"Database: {_modelConfiguration.AnalysisServicesDatabase}", false); + LogMessage($"Start: {DateTime.Now.ToString("hh:mm:ss tt")}", MessageType.Informational, false); + LogMessage($"Server: {_modelConfiguration.AnalysisServicesServer}", MessageType.Informational, false); + LogMessage($"Database: {_modelConfiguration.AnalysisServicesDatabase}", MessageType.Informational, false); Console.ForegroundColor = ConsoleColor.Yellow; foreach (TableConfiguration tableConfiguration in _modelConfiguration.TableConfigurations) @@ -69,18 +77,18 @@ namespace AsPartitionProcessing if (tableConfiguration.PartitioningConfigurations.Count == 0) { //Non-partitioned table. Process at table level. - LogMessage("", false); - LogMessage($"Non-partitioned processing for table {tableConfiguration.AnalysisServicesTable}", false); - LogMessage(new String('-', tableConfiguration.AnalysisServicesTable.Length + 37), false); + LogMessage("", MessageType.Informational, false); + LogMessage($"Non-partitioned processing for table {tableConfiguration.AnalysisServicesTable}", MessageType.Informational, false); + LogMessage(new String('-', tableConfiguration.AnalysisServicesTable.Length + 37), MessageType.Informational, false); if (_modelConfiguration.IncrementalOnline) { - LogMessage($"Process table {tableConfiguration.AnalysisServicesTable} /Full", true); + LogMessage($"Process table {tableConfiguration.AnalysisServicesTable} /Full", MessageType.Informational, true); table.RequestRefresh(RefreshType.Full); } else { - LogMessage($"Process table {tableConfiguration.AnalysisServicesTable} /DataOnly", true); + LogMessage($"Process table {tableConfiguration.AnalysisServicesTable} /DataOnly", MessageType.Informational, true); table.RequestRefresh(RefreshType.DataOnly); } } @@ -99,9 +107,9 @@ namespace AsPartitionProcessing //Process based on partitioning configuration(s). foreach (PartitioningConfiguration partitioningConfiguration in tableConfiguration.PartitioningConfigurations) { - LogMessage("", false); - LogMessage($"Rolling-window partitioning for table {tableConfiguration.AnalysisServicesTable}", false); - LogMessage(new String('-', tableConfiguration.AnalysisServicesTable.Length + 38), false); + LogMessage("", MessageType.Informational, false); + LogMessage($"Rolling-window partitioning for table {tableConfiguration.AnalysisServicesTable}", MessageType.Informational, false); + LogMessage(new String('-', tableConfiguration.AnalysisServicesTable.Length + 38), MessageType.Informational, false); //Figure out what processing needs to be done List partitionKeysCurrent = GetPartitionKeysCurrent(table, partitioningConfiguration.Granularity); @@ -109,19 +117,19 @@ namespace AsPartitionProcessing List partitionKeysForProcessing = GetPartitionKeysTarget(true, partitioningConfiguration, partitioningConfiguration.Granularity); DisplayPartitionRange(partitionKeysCurrent, true, partitioningConfiguration.Granularity); DisplayPartitionRange(partitionKeysNew, false, partitioningConfiguration.Granularity); - LogMessage("", false); - LogMessage("=>Actions & progress:", false); + LogMessage("", MessageType.Informational, false); + LogMessage("=>Actions & progress:", MessageType.Informational, false); //Check for old partitions that need to be removed foreach (string partitionKey in partitionKeysCurrent) { if (Convert.ToInt32(partitionKey) < Convert.ToInt32(partitionKeysNew[0])) { - LogMessage($"Remove old partition {DateFormatPartitionKey(partitionKey, partitioningConfiguration.Granularity)}", true); + LogMessage($"Remove old partition {DateFormatPartitionKey(partitionKey, partitioningConfiguration.Granularity)}", MessageType.Informational, true); table.Partitions.Remove(partitionKey); } } - + //Process partitions foreach (string partitionKey in partitionKeysForProcessing) { @@ -130,7 +138,7 @@ namespace AsPartitionProcessing if (partitionToProcess == null) { partitionToProcess = CreateNewPartition(table, templatePartition, partitioningConfiguration, partitionKey, partitioningConfiguration.Granularity); - LogMessage($"Create new partition {DateFormatPartitionKey(partitionKey, partitioningConfiguration.Granularity)}", true); + LogMessage($"Create new partition {DateFormatPartitionKey(partitionKey, partitioningConfiguration.Granularity)}", MessageType.Informational, true); if (!_modelConfiguration.InitialSetUp) { @@ -148,14 +156,14 @@ namespace AsPartitionProcessing if (partitionToProcess.State != ObjectState.Ready) { //Process new partitions sequentially during initial setup so don't run out of memory - LogMessage($"Sequentially process {DateFormatPartitionKey(partitionKey, partitioningConfiguration.Granularity)} /DataOnly", true); + LogMessage($"Sequentially process {DateFormatPartitionKey(partitionKey, partitioningConfiguration.Granularity)} /DataOnly", MessageType.Informational, true); partitionToProcess.RequestRefresh(RefreshType.DataOnly); database.Model.SaveChanges(); } else { //Partition already exists during initial setup (and is fully processed), so ignore it - LogMessage($"Partition {DateFormatPartitionKey(partitionKey, partitioningConfiguration.Granularity)} already exists and is processed", true); + LogMessage($"Partition {DateFormatPartitionKey(partitionKey, partitioningConfiguration.Granularity)} already exists and is processed", MessageType.Informational, true); } } } @@ -165,7 +173,7 @@ namespace AsPartitionProcessing if (_modelConfiguration.InitialSetUp) { string beginParam = GetDateKey("19010102", Granularity.Daily, tableConfiguration.PartitioningConfigurations[0].IntegerDateKey, false, templatePartition.Source is MPartitionSource); - string endParam = GetDateKey("19010101", Granularity.Daily, tableConfiguration.PartitioningConfigurations[0].IntegerDateKey, false, templatePartition.Source is MPartitionSource); + string endParam = GetDateKey("19010101", Granularity.Daily, tableConfiguration.PartitioningConfigurations[0].IntegerDateKey, false, templatePartition.Source is MPartitionSource); //Query generated will always return nothing string query = tableConfiguration.PartitioningConfigurations[0].TemplateSourceQuery.Replace("{0}", beginParam).Replace("{1}", endParam); @@ -185,47 +193,56 @@ namespace AsPartitionProcessing //Commit the data changes, and bring model back online if necessary - LogMessage("", false); - LogMessage("Final operations", false); - LogMessage(new String('-', 16), false); + LogMessage("", MessageType.Informational, false); + LogMessage("Final operations", MessageType.Informational, false); + LogMessage(new String('-', 16), MessageType.Informational, false); //Save changes setting MaxParallelism if necessary if (_modelConfiguration.MaxParallelism == -1) { - LogMessage("Save changes ...", true); + LogMessage("Save changes ...", MessageType.Informational, true); database.Model.SaveChanges(); } else { - LogMessage($"Save changes with MaxParallelism={Convert.ToString(_modelConfiguration.MaxParallelism)}...", true); + LogMessage($"Save changes with MaxParallelism={Convert.ToString(_modelConfiguration.MaxParallelism)}...", MessageType.Informational, true); database.Model.SaveChanges(new SaveOptions() { MaxParallelism = _modelConfiguration.MaxParallelism }); } //Perform recalc if necessary if (_modelConfiguration.InitialSetUp || (!_modelConfiguration.InitialSetUp && !_modelConfiguration.IncrementalOnline)) { - LogMessage("Recalc model to bring back online ...", true); + LogMessage("Recalc model to bring back online ...", MessageType.Informational, true); database.Model.RequestRefresh(RefreshType.Calculate); database.Model.SaveChanges(); } Console.ForegroundColor = ConsoleColor.White; - LogMessage("", false); - LogMessage("Finish: " + DateTime.Now.ToString("hh:mm:ss tt"), false); + LogMessage("", MessageType.Informational, false); + LogMessage("Finish: " + DateTime.Now.ToString("hh:mm:ss tt"), MessageType.Informational, false); } catch (Exception exc) { Console.ForegroundColor = ConsoleColor.Red; - LogMessage("", false); - LogMessage($"Exception occurred: {DateTime.Now.ToString("hh:mm:ss tt")}", false); - LogMessage($"Exception message: {exc.Message}", false); + LogMessage("", MessageType.Informational, false); + LogMessage($"Exception occurred: {DateTime.Now.ToString("hh:mm:ss tt")}", MessageType.Error, false); + LogMessage($"Exception message: {exc.Message}", MessageType.Error, false); if (exc.InnerException != null) { - LogMessage($"Inner exception message: {exc.InnerException.Message}", false); + LogMessage($"Inner exception message: {exc.InnerException.Message}", MessageType.Error, false); } - LogMessage("", false); + LogMessage("", MessageType.Informational, false); Console.ForegroundColor = ConsoleColor.White; + + //Auto retry + if (_retryAttempts > 0) + { + LogMessage($"Retry attempts remaining: {Convert.ToString(_retryAttempts)}. Will wait {Convert.ToString(_modelConfiguration.RetryWaitTimeSeconds)} seconds and then attempt retry.", MessageType.Informational, false); + _retryAttempts -= 1; + Thread.Sleep(_modelConfiguration.RetryWaitTimeSeconds * 1000); + PerformProcessing(); + } } finally { @@ -255,11 +272,11 @@ namespace AsPartitionProcessing Server server = new Server(); try { - LogMessage("", false); - LogMessage($"Merge partitions into {partitionKey} for table {analysisServicesTable}", false); - LogMessage(new String('-', partitionKey.Length + analysisServicesTable.Length + 33), false); - LogMessage("", false); - LogMessage("=>Actions & progress:", false); + LogMessage("", MessageType.Informational, false); + LogMessage($"Merge partitions into {partitionKey} for table {analysisServicesTable}", MessageType.Informational, false); + LogMessage(new String('-', partitionKey.Length + analysisServicesTable.Length + 33), MessageType.Informational, false); + LogMessage("", MessageType.Informational, false); + LogMessage("=>Actions & progress:", MessageType.Informational, false); //Check target granularity if (targetGranularity == Granularity.Daily) @@ -323,40 +340,40 @@ namespace AsPartitionProcessing List partitionsToBeMerged = GetPartitionsCurrent(table, childGranularity, partitionKey); if (partitionsToBeMerged.Count == 0) { - LogMessage($"No partitinos found in {analysisServicesTable} to be merged into {partitionKey}.", false); + LogMessage($"No partitinos found in {analysisServicesTable} to be merged into {partitionKey}.", MessageType.Informational, false); } else { //Done with validation, so go ahead ... - LogMessage("", false); - LogMessage($"Create new merged partition {DateFormatPartitionKey(partitionKey, targetGranularity)} for table {analysisServicesTable}", true); + LogMessage("", MessageType.Informational, false); + LogMessage($"Create new merged partition {DateFormatPartitionKey(partitionKey, targetGranularity)} for table {analysisServicesTable}", MessageType.Informational, true); Partition newPartition = CreateNewPartition(table, templatePartition, partitionConfig, partitionKey, targetGranularity); foreach (Partition partition in partitionsToBeMerged) { - LogMessage($"Partition {partition.Name} to be merged into {DateFormatPartitionKey(partitionKey, targetGranularity)}", true); + LogMessage($"Partition {partition.Name} to be merged into {DateFormatPartitionKey(partitionKey, targetGranularity)}", MessageType.Informational, true); } newPartition.RequestMerge(partitionsToBeMerged); - LogMessage($"Save changes for table {analysisServicesTable} ...", true); + LogMessage($"Save changes for table {analysisServicesTable} ...", MessageType.Informational, true); database.Model.SaveChanges(); Console.ForegroundColor = ConsoleColor.White; - LogMessage("", false); - LogMessage("Finish: " + DateTime.Now.ToString("hh:mm:ss tt"), false); + LogMessage("", MessageType.Informational, false); + LogMessage("Finish: " + DateTime.Now.ToString("hh:mm:ss tt"), MessageType.Informational, false); } } catch (Exception exc) { Console.ForegroundColor = ConsoleColor.Red; - LogMessage("", false); - LogMessage($"Exception occurred: {DateTime.Now.ToString("hh:mm:ss tt")}", false); - LogMessage($"Exception message: {exc.Message}", false); + LogMessage("", MessageType.Informational, false); + LogMessage($"Exception occurred: {DateTime.Now.ToString("hh:mm:ss tt")}", MessageType.Error, false); + LogMessage($"Exception message: {exc.Message}", MessageType.Error, false); if (exc.InnerException != null) { - LogMessage($"Inner exception message: {exc.InnerException.Message}", false); + LogMessage($"Inner exception message: {exc.InnerException.Message}", MessageType.Error, false); } - LogMessage("", false); + LogMessage("", MessageType.Informational, false); Console.ForegroundColor = ConsoleColor.White; } finally @@ -388,16 +405,16 @@ namespace AsPartitionProcessing Connect(server, out database); Console.ForegroundColor = ConsoleColor.White; - LogMessage($"Start: {DateTime.Now.ToString("hh:mm:ss tt")}", false); - LogMessage($"Server: {_modelConfiguration.AnalysisServicesServer}", false); - LogMessage($"Database: {_modelConfiguration.AnalysisServicesDatabase}", false); + LogMessage($"Start: {DateTime.Now.ToString("hh:mm:ss tt")}", MessageType.Informational, false); + LogMessage($"Server: {_modelConfiguration.AnalysisServicesServer}", MessageType.Informational, false); + LogMessage($"Database: {_modelConfiguration.AnalysisServicesDatabase}", MessageType.Informational, false); Console.ForegroundColor = ConsoleColor.Yellow; - LogMessage("", false); - LogMessage($"Defrag partitioned tables in database {_modelConfiguration.AnalysisServicesDatabase}", false); - LogMessage(new String('-', _modelConfiguration.AnalysisServicesDatabase.Length + 38), false); - LogMessage("", false); - LogMessage("=>Actions & progress:", false); + LogMessage("", MessageType.Informational, false); + LogMessage($"Defrag partitioned tables in database {_modelConfiguration.AnalysisServicesDatabase}", MessageType.Informational, false); + LogMessage(new String('-', _modelConfiguration.AnalysisServicesDatabase.Length + 38), MessageType.Informational, false); + LogMessage("", MessageType.Informational, false); + LogMessage("=>Actions & progress:", MessageType.Informational, false); foreach (TableConfiguration tableConfiguration in _modelConfiguration.TableConfigurations) { @@ -410,27 +427,27 @@ namespace AsPartitionProcessing throw new Microsoft.AnalysisServices.ConnectionException($"Could not connect to table {tableConfiguration.AnalysisServicesTable}."); } - LogMessage($"Defrag table {tableConfiguration.AnalysisServicesTable} ...", true); + LogMessage($"Defrag table {tableConfiguration.AnalysisServicesTable} ...", MessageType.Informational, true); table.RequestRefresh(RefreshType.Defragment); database.Model.SaveChanges(); } } Console.ForegroundColor = ConsoleColor.White; - LogMessage("", false); - LogMessage("Finish: " + DateTime.Now.ToString("hh:mm:ss tt"), false); + LogMessage("", MessageType.Informational, false); + LogMessage("Finish: " + DateTime.Now.ToString("hh:mm:ss tt"), MessageType.Informational, false); } catch (Exception exc) { Console.ForegroundColor = ConsoleColor.Red; - LogMessage("", false); - LogMessage($"Exception occurred: {DateTime.Now.ToString("hh:mm:ss tt")}", false); - LogMessage($"Exception message: {exc.Message}", false); + LogMessage("", MessageType.Informational, false); + LogMessage($"Exception occurred: {DateTime.Now.ToString("hh:mm:ss tt")}", MessageType.Error, false); + LogMessage($"Exception message: {exc.Message}", MessageType.Error, false); if (exc.InnerException != null) { - LogMessage($"Inner exception message: {exc.InnerException.Message}", false); + LogMessage($"Inner exception message: {exc.InnerException.Message}", MessageType.Error, false); } - LogMessage("", false); + LogMessage("", MessageType.Informational, false); Console.ForegroundColor = ConsoleColor.White; } finally @@ -453,19 +470,19 @@ namespace AsPartitionProcessing { if (_modelConfiguration.IncrementalOnline) { - LogMessage($"Parallel process partition {DateFormatPartitionKey(partitionKey, granularity)} /Full", true); + LogMessage($"Parallel process partition {DateFormatPartitionKey(partitionKey, granularity)} /Full", MessageType.Informational, true); partitionToProcess.RequestRefresh(RefreshType.Full); } else { - LogMessage($"Parallel process partition {DateFormatPartitionKey(partitionKey, granularity)} /DataOnly", true); + LogMessage($"Parallel process partition {DateFormatPartitionKey(partitionKey, granularity)} /DataOnly", MessageType.Informational, true); partitionToProcess.RequestRefresh(RefreshType.DataOnly); } } - private static void LogMessage(string message, bool indented) + private static void LogMessage(string message, MessageType messageType, bool indented) { - _messageLogger($"{(indented ? new String(' ', 3) : "")}{message}", _modelConfiguration); + _messageLogger($"{(indented ? new String(' ', 3) : "")}{message}", messageType, _modelConfiguration); } private static string DateFormatPartitionKey(string partitionKey, Granularity granularity) @@ -494,18 +511,18 @@ namespace AsPartitionProcessing private static void DisplayPartitionRange(List partitionKeys, bool current, Granularity granularity) { - LogMessage("", false); + LogMessage("", MessageType.Informational, false); if (partitionKeys.Count > 0) { - LogMessage($"=>{(current ? "Current" : "New")} partition range ({Convert.ToString(granularity)}):", false); - LogMessage($"MIN partition: {DateFormatPartitionKey(partitionKeys[0], granularity)}", true); - LogMessage($"MAX partition: {DateFormatPartitionKey(partitionKeys[partitionKeys.Count - 1], granularity)}", true); - LogMessage($"Partition count: {partitionKeys.Count}", true); + LogMessage($"=>{(current ? "Current" : "New")} partition range ({Convert.ToString(granularity)}):", MessageType.Informational, false); + LogMessage($"MIN partition: {DateFormatPartitionKey(partitionKeys[0], granularity)}", MessageType.Informational, true); + LogMessage($"MAX partition: {DateFormatPartitionKey(partitionKeys[partitionKeys.Count - 1], granularity)}", MessageType.Informational, true); + LogMessage($"Partition count: {partitionKeys.Count}", MessageType.Informational, true); } else { - LogMessage("=>Table not yet partitioned", false); + LogMessage("=>Table not yet partitioned", MessageType.Informational, false); } } @@ -513,7 +530,11 @@ namespace AsPartitionProcessing { //Connect and get main objects string serverConnectionString = $"Provider=MSOLAP;{(_modelConfiguration.CommitTimeout == -1 ? "" : $"CommitTimeout={Convert.ToString(_modelConfiguration.CommitTimeout)};")}Data Source={_modelConfiguration.AnalysisServicesServer};"; - if (!_modelConfiguration.IntegratedAuth) + if (_modelConfiguration.IntegratedAuth) + { + serverConnectionString += $"Integrated Security=SSPI;"; + } + else { serverConnectionString += $"User ID={_modelConfiguration.UserName};Password={_modelConfiguration.Password};Persist Security Info=True;Impersonation Level=Impersonate;"; }