diff --git a/.gitignore b/.gitignore index f44bd54..acbb46e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,32 @@ ################################################################################ /packages +/Smv2Sql/bin/Release +/Smv2Sql/obj +/Smv2Sql/Smv2Sql.csproj.user +/SmvAccessor/bin/Release +/SmvAccessor/obj +/SmvAccessorDemo/obj +/SMVActionsTable/bin/Release +/SMVActionsTable/obj +/SmvCloud/bin/Release +/SmvCloud/csx/Release +/SmvCloud/obj/Release +/SmvCloud/rcf/Release/SmvCloudWorkerContent +/SmvCloudWorker/bin/Release +/SmvCloudWorker/obj +/SmvDb/bin +/SmvDb/obj/Release +/SmvInterceptor/bin/Release +/SmvInterceptor/obj +/SmvInterceptorWrapper/bin/Release +/SmvInterceptorWrapper/obj +/SmvLibrary/bin +/SmvLibrary/obj +/SmvLineCounter/bin/Release +/SmvLineCounter/obj +/SmvSdv/bin/Release +/SmvSkeleton/bin +/SmvSkeleton/obj +/SmvTest/bin/Release +/SmvTest/obj diff --git a/Backup/ServiceConfiguration.Cloud.cscfg b/Backup/ServiceConfiguration.Cloud.cscfg new file mode 100644 index 0000000..4ea646e --- /dev/null +++ b/Backup/ServiceConfiguration.Cloud.cscfg @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/Backup/ServiceConfiguration.Local.cscfg b/Backup/ServiceConfiguration.Local.cscfg new file mode 100644 index 0000000..4ea646e --- /dev/null +++ b/Backup/ServiceConfiguration.Local.cscfg @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/Backup/ServiceDefinition.csdef b/Backup/ServiceDefinition.csdef new file mode 100644 index 0000000..b9875b7 --- /dev/null +++ b/Backup/ServiceDefinition.csdef @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/Backup/SmvCloud.ccproj b/Backup/SmvCloud.ccproj new file mode 100644 index 0000000..48d8944 --- /dev/null +++ b/Backup/SmvCloud.ccproj @@ -0,0 +1,73 @@ + + + + + Debug + AnyCPU + 2.3 + 3bbc4fad-85a1-4451-8975-bc00eaf81e63 + Library + Properties + SmvCloud + SmvCloud + True + SmvCloud + False + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + SmvCloudWorker + {00d32586-d675-4ef4-b4f3-06dc45bbf079} + True + Worker + SmvCloudWorker + True + + + + + + + + Content + + + Content + + + Content + + + Content + + + + + 10.0 + $(MSBuildExtensionsPath)\Microsoft\VisualStudio\v$(VisualStudioVersion)\Windows Azure Tools\2.3\ + + + \ No newline at end of file diff --git a/Backup/SmvCloudWorkerContent/CloudConfig.cs b/Backup/SmvCloudWorkerContent/CloudConfig.cs new file mode 100644 index 0000000..b27f318 --- /dev/null +++ b/Backup/SmvCloudWorkerContent/CloudConfig.cs @@ -0,0 +1,29 @@ +using System.Collections.Generic; +using System.Xml.Serialization; + +namespace SmvLibrary +{ + [XmlRootAttribute(Namespace = "")] + public class SMVCloudConfig + { + [XmlElementAttribute("StorageConnectionString")] + public StorageConnectionStringElement StorageConnectionString; + + [XmlElementAttribute("ServiceBusConnectionString")] + public ServiceBusConnectionStringElement ServiceBusConnectionString; + } + + [XmlRootAttribute("StorageConnectionString", Namespace = "")] + public class StorageConnectionStringElement + { + [XmlAttributeAttribute()] + public string Value { get; set; } + } + + [XmlRootAttribute("ServiceBusConnectionString", Namespace = "")] + public class ServiceBusConnectionStringElement + { + [XmlAttributeAttribute()] + public string Value { get; set; } + } +} \ No newline at end of file diff --git a/Backup/SmvCloudWorkerContent/CloudConfig.xsd b/Backup/SmvCloudWorkerContent/CloudConfig.xsd new file mode 100644 index 0000000..af09923 --- /dev/null +++ b/Backup/SmvCloudWorkerContent/CloudConfig.xsd @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Backup/SmvCloudWorkerContent/diagnostics.wadcfg b/Backup/SmvCloudWorkerContent/diagnostics.wadcfg new file mode 100644 index 0000000..74a9c23 --- /dev/null +++ b/Backup/SmvCloudWorkerContent/diagnostics.wadcfg @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Doc/CloudSMVActionScheduler.docx b/Doc/CloudSMVActionScheduler.docx new file mode 100644 index 0000000..9f652e9 Binary files /dev/null and b/Doc/CloudSMVActionScheduler.docx differ diff --git a/Doc/SMVCloudWorker.docx b/Doc/SMVCloudWorker.docx new file mode 100644 index 0000000..8314c3f Binary files /dev/null and b/Doc/SMVCloudWorker.docx differ diff --git a/Doc/SmvConfiguration.docx b/Doc/SmvConfiguration.docx new file mode 100644 index 0000000..5678155 Binary files /dev/null and b/Doc/SmvConfiguration.docx differ diff --git a/Doc/Static Module Verifier.docx b/Doc/Static Module Verifier.docx new file mode 100644 index 0000000..16a4929 Binary files /dev/null and b/Doc/Static Module Verifier.docx differ diff --git a/SMVActionsTable/ActionsTableDataSource.cs b/SMVActionsTable/ActionsTableDataSource.cs new file mode 100644 index 0000000..dd026b0 --- /dev/null +++ b/SMVActionsTable/ActionsTableDataSource.cs @@ -0,0 +1,107 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Table; + +namespace SMVActionsTable +{ + /// + /// This class is a wrapper around the actions table that provides some useful operations. + /// + public class ActionsTableDataSource + { + private CloudTable actionsTable; /// Table where each row contains information about the action. + public ActionsTableDataSource(CloudTable table) + { + actionsTable = table; + actionsTable.CreateIfNotExists(); + } + + /// + /// Get an entry from the table. + /// + /// The partition key of the entry. + /// The row key of the entry. + /// The entry if it was found in the table, null otherwise. + public ActionsTableEntry GetEntry(string partitionKey, string rowKey) + { + var retrieveOperation = TableOperation.Retrieve(partitionKey, rowKey); + TableResult result = actionsTable.Execute(retrieveOperation); + if(result.Result == null) + { + return null; + } + return (ActionsTableEntry)result.Result; + } + + /// + /// Get all the entries with a certain status. WARNING: Does a query across all partitions, so may be very slow. + /// + /// The status to use in the query. + /// The entries in the table which have the given status. + public IEnumerable GetEntries(ActionStatus status) + { + var query = new TableQuery().Where(TableQuery.GenerateFilterConditionForInt("Status", + QueryComparisons.Equal, (int)status)); + var results = actionsTable.ExecuteQuery(query); + return results; + } + + /// + /// Adds an entry to the table. + /// + /// The entry to be added. + public void AddEntry(ActionsTableEntry entry) + { + var insertOperation = TableOperation.Insert(entry); + actionsTable.Execute(insertOperation); + } + + /// + /// Updates the status of an entry in the table. + /// + /// Partition key of the entry. + /// Row key of the entry. + /// The new status of the entry. + public void UpdateStatus(string partitionKey, string rowKey, ActionStatus newStatus) + { + var retrieveOperation = TableOperation.Retrieve(partitionKey, rowKey); + TableResult result = actionsTable.Execute(retrieveOperation); + if (result.Result == null) + { + throw new ArgumentException(string.Format("Could not find an entry in the table for the given partitionKey, rowKey tuple: {0}, {1}", + partitionKey, rowKey)); + } + var entry = (ActionsTableEntry)result.Result; + entry.Status = (int)newStatus; + + var replaceOperation = TableOperation.Replace(entry); + actionsTable.Execute(replaceOperation); + } + + /// + /// Updates the SerializedAction field of an entry in the table. + /// + /// Partition key of the entry. + /// Row key of the entry. + /// The new value for the serialized action field. + public void UpdateAction(string partitionKey, string rowKey, byte[] serializedAction) + { + var retrieveOperation = TableOperation.Retrieve(partitionKey, rowKey); + TableResult result = actionsTable.Execute(retrieveOperation); + if (result.Result == null) + { + throw new ArgumentException(string.Format("Could not find an entry in the table for the given partitionKey, rowKey tuple: {0}, {1}", + partitionKey, rowKey)); + } + var entry = (ActionsTableEntry)result.Result; + entry.SerializedAction = serializedAction; + + var replaceOperation = TableOperation.Replace(entry); + actionsTable.Execute(replaceOperation); + } + } +} diff --git a/SMVActionsTable/ActionsTableEntry.cs b/SMVActionsTable/ActionsTableEntry.cs new file mode 100644 index 0000000..a0e9bf8 --- /dev/null +++ b/SMVActionsTable/ActionsTableEntry.cs @@ -0,0 +1,85 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Table; +using System.Globalization; + +namespace SMVActionsTable +{ + /// + /// Denotes the status of the action. + /// + public enum ActionStatus { NotStarted, InProgress, Complete, Error }; + + /// + /// Represents a row in the actions table. The scheduler instance GUID is used as the partition key and the rule GUID is used as the row key. + /// + public class ActionsTableEntry : TableEntity + { + /// + /// Name of the action. + /// + public string ActionName { get; set; } + + /// + /// Status of the action. + /// + public int Status { get; set; } + + /// + /// The serialized action. + /// + public byte[] SerializedAction { get; set; } + + /// + /// Which version of SMV to use to run this action. + /// + public string Version { get; set; } + + /// + /// Path to the plugin used to run this action, relative to %smv%. + /// + public string PluginPath { get; set; } + + /// + /// Hash of the module required to run this action. + /// + public string ModuleHash { get; set; } + + public ActionsTableEntry() + { + ActionName = "Invalid"; + RowKey = "Invalid"; + PartitionKey = "Invalid"; + SerializedAction = null; + Version = "Invalid"; + PluginPath = "Invalid"; + ModuleHash = "Invalid"; + Status = (int)ActionStatus.Error; + } + + public ActionsTableEntry(string actionName, string actionGuid, string schedulerInstanceGuid, byte[] serializedAction, + string version, string plugin, string moduleHash) + { + ActionName = actionName; + RowKey = actionGuid; + PartitionKey = schedulerInstanceGuid; + SerializedAction = serializedAction; + Version = version; + PluginPath = plugin; + ModuleHash = moduleHash; + Status = (int)ActionStatus.NotStarted; + } + + public override string ToString() + { + return "Scheduler Instance GUID: " + PartitionKey + Environment.NewLine + + "Action Name: " + ActionName + Environment.NewLine + + "Action Guid: " + RowKey + Environment.NewLine + + "Status: " + Status.ToString(CultureInfo.CurrentCulture); + } + } +} diff --git a/SMVActionsTable/Properties/AssemblyInfo.cs b/SMVActionsTable/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..d7a6d0a --- /dev/null +++ b/SMVActionsTable/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("SMVActionsTable")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("SMVActionsTable")] +[assembly: AssemblyCopyright("Copyright © 2014")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("ef2ca0f4-755d-47d9-b028-a1dbd66c4e10")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/SMVActionsTable/SMVActionsTable.csproj b/SMVActionsTable/SMVActionsTable.csproj new file mode 100644 index 0000000..fd16651 --- /dev/null +++ b/SMVActionsTable/SMVActionsTable.csproj @@ -0,0 +1,59 @@ + + + + + Debug + AnyCPU + {6F655D35-3867-4351-8929-23B4578E5190} + Library + Properties + SMVActionsTable + SMVActionsTable + v4.5 + 512 + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/SmvCloudWorker/Properties/AssemblyInfo.cs b/SmvCloudWorker/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..f14a261 --- /dev/null +++ b/SmvCloudWorker/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("SmvCloudWorker")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("SmvCloudWorker")] +[assembly: AssemblyCopyright("Copyright © 2014")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("b6c2f531-9088-40be-ae66-6e1308bdd26a")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/SmvCloudWorker/SmvCloudWorker.csproj b/SmvCloudWorker/SmvCloudWorker.csproj new file mode 100644 index 0000000..744e870 --- /dev/null +++ b/SmvCloudWorker/SmvCloudWorker.csproj @@ -0,0 +1,110 @@ + + + + Debug + AnyCPU + 8.0.30703 + 2.0 + {00D32586-D675-4EF4-B4F3-06DC45BBF079} + Library + Properties + SmvCloudWorker + SmvCloudWorker + v4.5 + 512 + Worker + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + ..\packages\Microsoft.Data.Edm.5.6.2\lib\net40\Microsoft.Data.Edm.dll + True + + + ..\packages\Microsoft.Data.OData.5.6.2\lib\net40\Microsoft.Data.OData.dll + True + + + ..\packages\Microsoft.Data.Services.Client.5.6.2\lib\net40\Microsoft.Data.Services.Client.dll + True + + + ..\packages\WindowsAzure.ServiceBus.3.2.0\lib\net45-full\Microsoft.ServiceBus.dll + True + + + ..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll + + + True + + + False + + + + ..\packages\Newtonsoft.Json.5.0.6\lib\net45\Newtonsoft.Json.dll + + + + + + + + + ..\packages\System.Spatial.5.6.2\lib\net40\System.Spatial.dll + True + + + + + + + + + + + + + + + + + + {5a923bec-71fc-48e3-b076-2a2aa81ebd03} + SmvAccessor + + + {6f655d35-3867-4351-8929-23b4578e5190} + SMVActionsTable + + + {e03dc0df-84cc-420c-91b1-2a937e88ff31} + SmvLibrary + + + + + \ No newline at end of file diff --git a/SmvCloudWorker/WorkerRole.cs b/SmvCloudWorker/WorkerRole.cs new file mode 100644 index 0000000..3759cb6 --- /dev/null +++ b/SmvCloudWorker/WorkerRole.cs @@ -0,0 +1,410 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.IO.Compression; +using System.Diagnostics; +using System.IO; +using System.Xml.Serialization; +using Microsoft.WindowsAzure.ServiceRuntime; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using Microsoft.WindowsAzure.Storage.Table; +using Microsoft.WindowsAzure.Storage.Queue; +using SmvLibrary; +using SMVActionsTable; +using Microsoft.WindowsAzure.Storage.RetryPolicies; +using Microsoft.ServiceBus.Messaging; +using Microsoft.ServiceBus; + +namespace SmvCloudWorker2 +{ + public class WorkerRole : RoleEntryPoint + { + private TopicClient outputTopic; /// Output topic to inform the clients of results. + private SMVCloudConfig cloudConfig; /// Object containing connection strings. + private bool acceptingMessages = true; /// Set to false when the worker stops accepting new messages. + private BlobRequestOptions options; /// Options for blob storage. + private CloudQueue inputQueue; /// Queue with the actions to be executed. + private CloudBlobContainer jobsContainer; /// Jobs are downloaded from here. + private CloudBlobContainer resultsContainer; /// Results are uploaded here. + private CloudBlobContainer versionsContainer; /// SMV versions are downloaded from here. + private ActionsTableDataSource tableDataSource; /// Used to interact with the actions table. + private string workingDirectory; /// The working directory for SMV. + private string resultsDirectory; /// A temporary location for the result zip files. + private string smvVersionsDirectory; /// The directory containing the SMV versions. + private string smvDirectory; /// The directory containing the current SMV version. + private CloudQueueMessage currentMessage; /// The message being currently processed. + + /// Maps names of SMV versions to their locations on the file system. + private IDictionary> smvVersions = new Dictionary>(); + + private void SendMessageToTopic(BrokeredMessage msg) + { + bool done = false; + for (int retryCount = 1; !done; retryCount++) + { + done = true; + try + { + outputTopic = TopicClient.CreateFromConnectionString(cloudConfig.ServiceBusConnectionString.value, CloudConstants.ResultsTopicName); + outputTopic.Send(msg); + outputTopic.Close(); + } + catch (Exception) + { + if (retryCount == CloudConstants.MaxRetries) + { + throw; + } + done = false; + System.Threading.Thread.Sleep(CloudConstants.RetryBackoffInterval); + } + } + } + + public override void Run() + { + while (acceptingMessages) + { + System.Threading.Thread.Sleep(30 * 1000); + + try + { + currentMessage = inputQueue.GetMessage(TimeSpan.FromHours(1)); + if (currentMessage != null) + { + // Parse the message. + string[] msgParts = currentMessage.AsString.Split(','); + string schedulerInstanceGuid = msgParts[0]; + string actionGuid = msgParts[1]; + + // Get the table entry. + ActionsTableEntry tableEntry = tableDataSource.GetEntry(schedulerInstanceGuid, actionGuid); + tableDataSource.UpdateStatus(schedulerInstanceGuid, actionGuid, ActionStatus.InProgress); + CloudBlockBlob jobBlob = jobsContainer.GetBlockBlobReference(actionGuid + ".zip"); + using (var outputMsg = new BrokeredMessage()) + { + outputMsg.Properties["SchedulerInstanceGuid"] = schedulerInstanceGuid; + outputMsg.Properties["ActionGuid"] = actionGuid; + outputMsg.Properties["DequeueCount"] = currentMessage.DequeueCount; + outputMsg.Properties["WaitTime"] = DateTime.Now - currentMessage.InsertionTime; + + // Check if we have tried to process this message too many times. + // If so, delete it and report an error back to the client. + if (currentMessage.DequeueCount >= CloudConstants.MaxDequeueCount) + { + tableDataSource.UpdateStatus(schedulerInstanceGuid, actionGuid, ActionStatus.Error); + SendMessageToTopic(outputMsg); + inputQueue.DeleteMessage(currentMessage); + jobBlob.Delete(); + continue; + } + + // Switch the version of SMV if required. + if (!SetSmvVersion(tableEntry.Version)) + { + Trace.TraceError("Could not set SMV version."); + tableDataSource.UpdateStatus(schedulerInstanceGuid, actionGuid, ActionStatus.Error); + SendMessageToTopic(outputMsg); + inputQueue.DeleteMessage(currentMessage); + jobBlob.Delete(); + continue; + } + + // Load the plugin. + if (!string.IsNullOrEmpty(tableEntry.PluginPath)) + { + string pluginPath = Environment.ExpandEnvironmentVariables(tableEntry.PluginPath); + var assembly = System.Reflection.Assembly.LoadFrom(pluginPath); + string fullName = assembly.ExportedTypes.First().FullName; + Utility.plugin = (ISMVPlugin)assembly.CreateInstance(fullName); + if (Utility.plugin == null) + { + throw new Exception("Could not load plugin: " + tableEntry.PluginPath); + } + Utility.plugin.Initialize(); + } + + // Get the module object, if any. + if (!string.IsNullOrEmpty(tableEntry.ModuleHash)) + { + SmvAccessor.ISmvAccessor accessor = Utility.GetSmvSQLAccessor(); + Utility.smvModule = accessor.GetModuleByHash(tableEntry.ModuleHash); + if(Utility.smvModule == null) + { + throw new Exception("Could not load module with hash: " + tableEntry.ModuleHash); + } + } + + // Download the job and extract it to the working directory. + Utility.ClearDirectory(workingDirectory); + string jobZipPath = Path.Combine(workingDirectory, "job.zip"); + jobBlob.DownloadToFile(jobZipPath, FileMode.CreateNew); + ZipFile.ExtractToDirectory(jobZipPath, workingDirectory); + File.Delete(jobZipPath); + + // Deserialize the action. + SMVAction action = (SMVAction)Utility.ByteArrayToObject(tableEntry.SerializedAction); + + // Get ready to execute the action. + // We substitute the value of assemblyDir and workingDir with the values on this machine. + string oldWorkingDir = action.variables["workingDir"].ToLower(); + string oldAssemblyDir = action.variables["assemblyDir"].ToLower(); + string newAssemblyDir = Path.Combine(smvDirectory, "bin").ToLower(); + workingDirectory = workingDirectory.ToLower(); + var keys = new List(action.variables.Keys); + foreach (var key in keys) + { + if (!string.IsNullOrEmpty(action.variables[key])) + { + if (action.variables[key].ToLower().StartsWith(oldAssemblyDir)) + { + action.variables[key] = action.variables[key].ToLower().Replace(oldAssemblyDir, newAssemblyDir); + } + else if (action.variables[key].ToLower().StartsWith(oldWorkingDir)) + { + action.variables[key] = action.variables[key].ToLower().Replace(oldWorkingDir, workingDirectory); + } + } + } + // NOTE: We set the Path attribute in the action to null because the action is always processed in the working directory. + var path = action.Path; + action.Path = null; + + Utility.SetSmvVar("workingDir", workingDirectory); + + // Execute the action. + SMVActionResult result = Utility.ExecuteAction(action); + + // Change the paths back to their old values. + foreach (var key in keys) + { + if (!string.IsNullOrEmpty(action.variables[key])) + { + if (action.variables[key].ToLower().StartsWith(newAssemblyDir)) + { + action.variables[key] = action.variables[key].ToLower().Replace(newAssemblyDir, oldAssemblyDir); + } + else if (action.variables[key].ToLower().StartsWith(workingDirectory)) + { + action.variables[key] = action.variables[key].ToLower().Replace(workingDirectory, oldWorkingDir); + } + } + } + + // Now set the path attribute again because the client needs it. + action.Path = path; + + // Zip up the working directory and upload it as the result. + string resultsZipPath = Path.Combine(resultsDirectory, actionGuid + ".zip"); + ZipFile.CreateFromDirectory(workingDirectory, resultsZipPath); + CloudBlockBlob resultsBlob = resultsContainer.GetBlockBlobReference(actionGuid + ".zip"); + resultsBlob.UploadFromFile(resultsZipPath, FileMode.Open); + File.Delete(resultsZipPath); + + // Job done! + tableDataSource.UpdateAction(schedulerInstanceGuid, actionGuid, Utility.ObjectToByteArray(action)); + tableDataSource.UpdateStatus(schedulerInstanceGuid, actionGuid, ActionStatus.Complete); + SendMessageToTopic(outputMsg); + if (currentMessage != null) + { + inputQueue.DeleteMessage(currentMessage); + currentMessage = null; + } + jobBlob.DeleteIfExists(); + Utility.ClearDirectory(workingDirectory); + } + } + } + catch (Exception e) + { + Trace.TraceError("Exception while processing queue item:" + e.ToString()); + if (currentMessage != null) + { + inputQueue.UpdateMessage(currentMessage, TimeSpan.FromSeconds(5), MessageUpdateFields.Visibility); + } + System.Threading.Thread.Sleep(5000); + } + } + } + + private bool SetSmvVersion(string version) + { + // The version string cannot be empty. + if (String.IsNullOrEmpty(version)) + { + return false; + } + + // First, check if the SMV version already exists in our local machine. + string smvName = "smv-" + version; + string smvFilename = smvName + ".zip"; + CloudBlockBlob blob = versionsContainer.GetBlockBlobReference(smvFilename); + + blob.FetchAttributes(); + if (smvVersions.ContainsKey(version) && smvVersions[version].Item2 >= blob.Properties.LastModified.Value.UtcDateTime) + { + smvVersionsDirectory = smvVersions[version].Item1; + Environment.SetEnvironmentVariable("smv", smvVersionsDirectory); + return true; + } + + // Download the SMV version if it's not available locally. + string zipPath = Path.Combine(smvVersionsDirectory, smvFilename); + if (File.Exists(zipPath)) + { + File.Delete(zipPath); + } + blob.DownloadToFile(zipPath, FileMode.CreateNew); + + // Delete the version's directory to clear it of any old files. + string dir = Path.Combine(smvVersionsDirectory, smvName); + if (Directory.Exists(dir)) + { + Directory.Delete(dir, true); + } + + // Now we'll unpack our version of SMV. + try + { + Directory.CreateDirectory(dir); + ZipFile.ExtractToDirectory(zipPath, dir); + File.Delete(zipPath); + } + catch (Exception e) + { + Trace.TraceError("Error unzipping smv.zip: Exception: {0}", e.ToString()); + return false; + } + + // We've setup a version of SMV successfully. Set smvPath and add this version to the list of avaiable SMV versions. + smvDirectory = dir; + Environment.SetEnvironmentVariable("smv", smvDirectory); + smvVersions[version] = new Tuple(smvDirectory, blob.Properties.LastModified.Value.UtcDateTime); + + return true; + } + + public override bool OnStart() + { + try + { + // Set the maximum number of concurrent connections . + System.Net.ServicePointManager.DefaultConnectionLimit = 12; + + // Set options for blob storage. + options = new BlobRequestOptions(); + options.ServerTimeout = TimeSpan.FromMinutes(10); + options.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(CloudConstants.RetryBackoffInterval), CloudConstants.MaxRetries); + + // Get the connection strings. + string assemblyDir = Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location); + string cloudConfigPath = Path.Combine(assemblyDir, CloudConstants.CloudConfigXmlFileName); + string cloudConfigContents = Utility.ReadFile(cloudConfigPath); + string schemaPath = Path.Combine(assemblyDir, CloudConstants.CloudConfigXsdFileName); + using (var c = new StringReader(cloudConfigContents)) + { + if (!Utility.ValidateXmlFile(schemaPath, c)) + { + Trace.TraceError("Could not load cloud config from file: " + cloudConfigPath); + return false; + } + } + var serializer = new XmlSerializer(typeof(SMVCloudConfig)); + using (var reader = new StringReader(cloudConfigContents)) + { + cloudConfig = (SMVCloudConfig)serializer.Deserialize(reader); + } + + bool done = false; + while (!done) + { + try + { + var storageAccount = CloudStorageAccount.Parse(cloudConfig.StorageConnectionString.value); + + // Setup queue storage. + CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient(); + queueClient.DefaultRequestOptions = new QueueRequestOptions(); + queueClient.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(CloudConstants.RetryBackoffInterval), + CloudConstants.MaxRetries); + inputQueue = queueClient.GetQueueReference(CloudConstants.InputQueueName); + inputQueue.CreateIfNotExists(); + + // Setup blob storage. + CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); + blobClient.DefaultRequestOptions = new BlobRequestOptions(); + blobClient.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(CloudConstants.RetryBackoffInterval), + CloudConstants.MaxRetries); + jobsContainer = blobClient.GetContainerReference(CloudConstants.InputBlobContainerName); + jobsContainer.CreateIfNotExists(); + resultsContainer = blobClient.GetContainerReference(CloudConstants.OutputBlobContainerName); + resultsContainer.CreateIfNotExists(); + versionsContainer = blobClient.GetContainerReference(CloudConstants.VersionsContainerName); + versionsContainer.CreateIfNotExists(); + + // Setup table storage. + var tableStorage = storageAccount.CreateCloudTableClient(); + tableStorage.DefaultRequestOptions = new TableRequestOptions(); + tableStorage.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(CloudConstants.RetryBackoffInterval), CloudConstants.MaxRetries); + CloudTable table = tableStorage.GetTableReference(CloudConstants.TableName); + tableDataSource = new ActionsTableDataSource(table); + + // Setup the service bus topic. + var namespaceManager = NamespaceManager.CreateFromConnectionString(cloudConfig.ServiceBusConnectionString.value); + if (!namespaceManager.TopicExists(CloudConstants.ResultsTopicName)) + { + namespaceManager.CreateTopic(CloudConstants.ResultsTopicName); + } + + done = true; + } + catch (Exception e) + { + Trace.TraceError("Failure trying to connect to Azure storage: " + e.ToString()); + System.Threading.Thread.Sleep(5000); + } + } + + // Get paths to important directories in the file system. Remove the trailing slash from the paths. + LocalResource localResource = RoleEnvironment.GetLocalResource(CloudConstants.SmvWorkingDirectoryResourceName); + workingDirectory = localResource.RootPath.Remove(localResource.RootPath.Length - 1); + + localResource = RoleEnvironment.GetLocalResource(CloudConstants.SmvResultsDirectoryResourceName); + resultsDirectory = localResource.RootPath.Remove(localResource.RootPath.Length - 1); + + localResource = RoleEnvironment.GetLocalResource(CloudConstants.SmvDirectoryResourceName); + smvVersionsDirectory = localResource.RootPath.Remove(localResource.RootPath.Length - 1); + } + catch (Exception e) + { + Trace.TraceError("Exception while running OnStart(): " + e.ToString()); + return false; + } + + Utility.result = null; + + return base.OnStart(); + } + + public override void OnStop() + { + try + { + // Stop accepting more messages. + acceptingMessages = false; + + // Make the message available to another worker. + if (currentMessage != null) + { + inputQueue.UpdateMessage(currentMessage, TimeSpan.FromSeconds(5), MessageUpdateFields.Visibility); + currentMessage = null; + } + } + catch (Exception e) + { + Trace.TraceError("An exception occurred while running OnStop(): " + e.ToString()); + } + } + } +} \ No newline at end of file diff --git a/SmvCloudWorker/app.config b/SmvCloudWorker/app.config new file mode 100644 index 0000000..a002034 --- /dev/null +++ b/SmvCloudWorker/app.config @@ -0,0 +1,62 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/SmvCloudWorker/packages.config b/SmvCloudWorker/packages.config new file mode 100644 index 0000000..8e5ff4e --- /dev/null +++ b/SmvCloudWorker/packages.config @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/SmvInterceptorWrapper/App.config b/SmvInterceptorWrapper/App.config new file mode 100644 index 0000000..8e15646 --- /dev/null +++ b/SmvInterceptorWrapper/App.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/SmvInterceptorWrapper/GlobalSuppressions.cs b/SmvInterceptorWrapper/GlobalSuppressions.cs new file mode 100644 index 0000000..091fe25 Binary files /dev/null and b/SmvInterceptorWrapper/GlobalSuppressions.cs differ diff --git a/SmvInterceptorWrapper/Program.cs b/SmvInterceptorWrapper/Program.cs new file mode 100644 index 0000000..e71e494 --- /dev/null +++ b/SmvInterceptorWrapper/Program.cs @@ -0,0 +1,497 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Diagnostics; +using System.Text.RegularExpressions; +using System.IO; +using System.Globalization; + +namespace SmvInterceptorWrapper +{ + class Program + { + static void Main(string[] args) + { + // Get the output dir from the Environment variable set by SMV + string smvOutDir = Environment.GetEnvironmentVariable("SMV_OUTPUT_DIR"); + + List iargs = args.Where(x => !x.Contains("/iwrap:") && !x.Contains(".rsp") && !x.Contains("/plugin:")).ToList(); + + #region cl.exe + if (args.Contains("/iwrap:cl.exe")) + { + Console.WriteLine("iwrap: cl.exe called with args " + string.Join(",", args)); + string[] unsupportedClExtensions = {".inf", ".mof", ".src", ".pdb" }; + + // check for inf and other unsupported files and skip + if (args.Where(a => unsupportedClExtensions.Any(b => a.Contains(b))).Count() > 0) + { + return; + } + + // get the name of the plugin + string plugin = args.Where(x => x.Contains("/plugin:")).ToList().First().Replace("/plugin:", String.Empty); + + // call slamcl + string rspContents = string.Empty; + string rspFileContent = string.Empty; + string rspFile = args.ToList().Find(x => x.Contains(".rsp") || x.StartsWith("@", StringComparison.OrdinalIgnoreCase)); + if (!string.IsNullOrEmpty(rspFile)) + { + rspFile = rspFile.Replace("@", ""); + rspContents = System.IO.File.ReadAllText(rspFile); + rspContents = rspContents.Replace("/analyze-", ""); + } + + // remove unsupported flags. currently we are not removing anything. + Regex[] unsupportedFlags = {//new Regex(@"\s+\/Yu[^\s]+\s{1}", System.Text.RegularExpressions.RegexOptions.IgnoreCase), + //new Regex(@"\s+\/Fp[^\s]+\s{1}", System.Text.RegularExpressions.RegexOptions.IgnoreCase), + //new Regex(@"\s+\/Yc[^\s]+\s{1}", System.Text.RegularExpressions.RegexOptions.IgnoreCase), + //new Regex(@"\s+(\/d1nodatetime)", System.Text.RegularExpressions.RegexOptions.IgnoreCase), + //new Regex(@"\s+(\/d1trimfile:[^\s]*)", System.Text.RegularExpressions.RegexOptions.IgnoreCase), + //new Regex(@"\s+(\/d2AllowCompatibleILVersions)", System.Text.RegularExpressions.RegexOptions.IgnoreCase), + //new Regex(@"\s+(\/d2Zi\+)", System.Text.RegularExpressions.RegexOptions.IgnoreCase), + //new Regex(@"\s+(\/d1nosafedelete)", System.Text.RegularExpressions.RegexOptions.IgnoreCase), + //new Regex(@"\s+(\-nosafedelete)", System.Text.RegularExpressions.RegexOptions.IgnoreCase), + //new Regex(@"-DDBG=\d\s{1}|\/DDBG=\d\s{1}", System.Text.RegularExpressions.RegexOptions.IgnoreCase), + //new Regex(@"-DDEBUG\s{1}|\/DDEBUG\s{1}", System.Text.RegularExpressions.RegexOptions.IgnoreCase), + //new Regex("\\s+\\/Fp\"[^\"]+\"{1}", System.Text.RegularExpressions.RegexOptions.None) + }; + + foreach (Regex rgx in unsupportedFlags) + { + rspContents = rgx.Replace(rspContents, " "); + } + + rspContents = rspContents.Replace("\r", " "); + rspContents = rspContents.Replace("\n", " "); + rspContents = rspContents.Replace("/W4", string.Empty); + rspContents = rspContents.Replace("/W3", string.Empty); + rspContents = rspContents.Replace("/W2", string.Empty); + rspContents = rspContents.Replace("/W1", string.Empty); + rspContents = rspContents.Replace("/WX", string.Empty); + + rspFileContent = rspContents; + + rspContents = Environment.ExpandEnvironmentVariables(" /nologo /w /Y- /analyze:only /analyze:plugin \"" + plugin + + "\" /errorReport:none" + " " + string.Join(" ", iargs)) + " " + rspContents; + + // get out dir + string outDir = smvOutDir; + + // Persist the RSP file + // Remove file names (*.c) from the content + Regex fileNameRegex = new Regex(@"(\w+\.c)", System.Text.RegularExpressions.RegexOptions.IgnoreCase); + rspFileContent = fileNameRegex.Replace(rspFileContent, String.Empty); + + using (System.IO.StreamWriter str = new StreamWriter(Path.Combine(outDir, "sdv_cl.rsp"), false)) + { + str.Write(rspFileContent); + } + + // call CL.exe + ProcessStartInfo psi = new ProcessStartInfo(System.IO.Path.GetFullPath(Environment.ExpandEnvironmentVariables("%SMV_ANALYSIS_COMPILER%")), + Environment.ExpandEnvironmentVariables(rspContents)); + psi.RedirectStandardError = true; + psi.RedirectStandardOutput = true; + psi.UseShellExecute = false; + + + + if (!psi.EnvironmentVariables.ContainsKey("esp.cfgpersist.persistfile")) + { + psi.EnvironmentVariables.Add("esp.cfgpersist.persistfile", outDir + "\\$SOURCEFILE.rawcfgf"); + psi.EnvironmentVariables.Add("Esp.CfgPersist.ExpandLocalStaticInitializer", "1"); + psi.EnvironmentVariables.Add("ESP.BplFilesDir", outDir); + } + + //Console.WriteLine("iwrap: cl.exe --> " + psi.FileName + " " + psi.Arguments); + + Process p = System.Diagnostics.Process.Start(psi); + using(System.IO.StreamWriter sw = new System.IO.StreamWriter(outDir + "\\smvcl.log", true)) + { + sw.Write(p.StandardOutput.ReadToEnd()); + sw.Write(p.StandardError.ReadToEnd()); + } + + File.AppendAllText(outDir + "\\smvcl.log", psi.Arguments); + } + #endregion + #region smv2sql.exe + else if (args.Contains("/iwrap:smv2sql.exe")) + { + // get rsp contents + string rspContents = string.Empty; + string rspFile = args.ToList().Find(x => x.Contains(".rsp") || x.StartsWith("@", StringComparison.OrdinalIgnoreCase)); + if (!string.IsNullOrEmpty(rspFile)) + { + rspFile = rspFile.Replace("@", ""); + rspContents = System.IO.File.ReadAllText(rspFile); + rspContents = rspContents.Replace("\r", " "); + rspContents = rspContents.Replace("\n", " "); + } + + // get directory where build artifacts are being placed + // Note: for Link, this is not just getting the /out: + // since Link is using inputs, and the location of the inputs + // is the real location where rawcfgf/li files are going to + // TODO: think about whether the LI files should be placed + // in the outdir + string outDir = smvOutDir; + Console.WriteLine("iwrap: smv2sql.exe --> outdir is " + outDir); + + // May not be the best way to get the source dir. + File.Copy(Environment.ExpandEnvironmentVariables(@"%SMV%\bin\smv2sql.exe.config"), Path.Combine(outDir, "smv2sql.exe.config"), true); + File.Copy(Environment.ExpandEnvironmentVariables(@"%SMV%\bin\smvaccessor.dll"), Path.Combine(outDir, "smvaccessor.dll"), true); + + // Don't let the Smv2Sql phase run twice. + if (File.Exists(Path.Combine(outDir, "smv2sql.log"))) + { + Console.WriteLine("iwrap: link.exe --> quitting since this phase is already complete"); + return; + } + + // Run Smv2Sql to put the LI and BPL files into a DB. + Process smv2SqlProcess; + var psi = new ProcessStartInfo(System.IO.Path.GetFullPath(Environment.ExpandEnvironmentVariables(@"%SMV%\bin\smv2sql.exe"))); + psi.RedirectStandardError = true; + psi.RedirectStandardOutput = true; + psi.UseShellExecute = false; + psi.WorkingDirectory = outDir; + psi.Arguments = outDir + " /log verbose"; + Console.WriteLine("iwrap: link.exe --> " + psi.FileName + " " + psi.Arguments); + smv2SqlProcess = System.Diagnostics.Process.Start(psi); + using (StreamWriter sw = new System.IO.StreamWriter(outDir + "\\smv2sql.log", true)) + { + sw.Write(smv2SqlProcess.StandardOutput.ReadToEnd()); + sw.Write(smv2SqlProcess.StandardError.ReadToEnd()); + } + } + #endregion + #region link.exe + else if (args.Contains("/iwrap:link.exe")) + { + Console.WriteLine("iwrap: link.exe called with args " + string.Join(" ", iargs)); + Console.WriteLine("iwrap: link.exe --> " + Environment.ExpandEnvironmentVariables("slamcl_writer.exe")); + + // get rsp contents + string rspContents = string.Empty; + string rspFile = args.ToList().Find(x => x.Contains(".rsp") || x.StartsWith("@", StringComparison.OrdinalIgnoreCase)); + if (!string.IsNullOrEmpty(rspFile)) + { + rspFile = rspFile.Replace("@", ""); + rspContents = System.IO.File.ReadAllText(rspFile); + rspContents = rspContents.Replace("\r", " "); + rspContents = rspContents.Replace("\n", " "); + } + + // get out dir + string outDir = smvOutDir; + Console.WriteLine("iwrap: link.exe --> outdir is " + outDir); + + // get rid of previous LI files, log files etc. + foreach (string liFile in Directory.GetFiles(outDir, "*.li")) + { + File.Delete(liFile); + } + if(File.Exists(Path.Combine(outDir, "smvlink.log"))) + { + File.Delete(Path.Combine(outDir, "smvlink.log")); + } + + //TODO: if link is called multiple times to create multiple binaries + // we will still only create 1 LI file for all the LI files that are available + // This happens in cases where a directory is used to store all the .obj + // files and then link is called multiple times with a subset of the .obj + // files to produce various binaries. + // The solution is to look at the link command in its entirety, + // and extract the obj files that are being used in it. + // we should then produce the LI for the corresponding obj.li files. + // the obj files are specified in the link rsp or command line and can + // be extracted using a regex, similar to how we extract lib files and locations. + + string[] files = System.IO.Directory.GetFiles(outDir, "*.rawcfgf"); + + files = files.Select(x => System.IO.Path.GetFileNameWithoutExtension(System.IO.Path.GetFileName(x))).ToArray(); + + files = files.Where(x => !x.Contains(".obj")).ToArray(); + + ProcessStartInfo psi; + Process p; + StreamWriter sw; + + psi = new ProcessStartInfo(Environment.ExpandEnvironmentVariables("slamcl_writer.exe"), "--smv *"); + psi.RedirectStandardError = true; + psi.RedirectStandardOutput = true; + psi.UseShellExecute = false; + psi.WorkingDirectory = outDir; + + Console.WriteLine("iwrap: link.exe --> " + psi.FileName + " " + psi.Arguments); + + p = System.Diagnostics.Process.Start(psi); + using (sw = new System.IO.StreamWriter(outDir + "\\smvlink.log", true)) + { + sw.Write(p.StandardOutput.ReadToEnd()); + sw.Write(p.StandardError.ReadToEnd()); + } + + files = files.Select(x => x + ".rawcfgf.obj").ToArray(); + + #region BPL for compilation units + /* + * IF WE WANT TO PRODUCE BPLs for each compilation unit then we uncommend the following section + // Produce BPL files from the LI files. + Process li2BplProcess; + File.Copy(Environment.ExpandEnvironmentVariables(@"%SMV%\bin\liConversion.txt"), Path.Combine(outDir, "liConversion.txt"), true); + psi = new ProcessStartInfo(System.IO.Path.GetFullPath(Environment.ExpandEnvironmentVariables(@"%SMV%\bin\li2bpl.exe"))); + psi.RedirectStandardError = true; + psi.RedirectStandardOutput = true; + psi.UseShellExecute = false; + psi.WorkingDirectory = outDir; + sw = new System.IO.StreamWriter(outDir + "\\smvli2bpl.log", true); + foreach(string file in files) + { + psi.Arguments = " -liFile " + Path.Combine(outDir, file); + Console.WriteLine("iwrap: li2bpl.exe --> " + psi.FileName + " " + psi.Arguments); + li2BplProcess = System.Diagnostics.Process.Start(psi); + sw.Write(li2BplProcess.StandardOutput.ReadToEnd()); + sw.Write(li2BplProcess.StandardError.ReadToEnd()); + // Rename all the BPL files, appending "compilationunit." to the name of each file. + // Can li2bpl be modified to take the name of the output bpl file as an argument? + string oldBplPath = Path.Combine(outDir, "li2c_prog.bpl"); + string newBplPath = Path.Combine(outDir, "compilationunit." + file + ".bpl"); + if (File.Exists(oldBplPath)) + { + File.Copy(oldBplPath, newBplPath, true); + } + } + sw.Close(); + */ + #endregion + + Process slamLinkProcess; + + // if only 1 li file then just copy that to slam.li + if (files.Length == 1) + { + File.Copy(outDir + "\\" + files.ElementAt(0) + ".li", outDir + "\\slam.li", true); + } + else + { + // many LI files, link all LIs together + psi = new ProcessStartInfo(Environment.ExpandEnvironmentVariables("slamlink.exe ")); + psi.RedirectStandardError = true; + psi.RedirectStandardOutput = true; + psi.UseShellExecute = false; + psi.WorkingDirectory = outDir; + psi.Arguments = " --lib " + string.Join(" ", files); + Console.WriteLine("iwrap: link.exe --> " + psi.FileName + " " + psi.Arguments); + slamLinkProcess = System.Diagnostics.Process.Start(psi); + using (sw = new System.IO.StreamWriter(outDir + "\\smvlink.log", true)) + { + sw.Write(slamLinkProcess.StandardOutput.ReadToEnd()); + sw.Write(slamLinkProcess.StandardError.ReadToEnd()); + } + + // copy the slam.lib.li produced by slamlink to slam.li + if (File.Exists(outDir + "\\slam.lib.li")) + { + File.Copy(outDir + "\\slam.lib.li", outDir + "\\slam.li", true); + } + } + + // create copy for linking with libs + if (File.Exists(outDir + "\\slam.li")) + { + File.Copy(outDir + "\\slam.li", outDir + "\\slamout.obj.li", true); + } + else + { + Console.WriteLine("iwrap: link.exe --> No slam.li found in " + outDir); + } + + // get any libs that need to be added and the corresponding rawcfgs + List libs = GetLibs(string.Join(" ", iargs) + " " + rspContents + " "); + + libs.RemoveAll(l => string.IsNullOrEmpty(l)); + + foreach (string l in libs) + { + Console.WriteLine("lib is " + l); + if (l.Equals(outDir)) continue; + try + { + string[] liFilesInLibDir = Directory.GetFiles(l, "slam.li"); + + foreach (string liFile in liFilesInLibDir) + { + Console.WriteLine("iwrap: Linking " + liFile + " " + outDir + "\\slam.obj.li"); + + File.Copy(liFile, outDir + "\\slamlib.obj.li", true); + File.Copy(outDir + "\\slamout.obj.li", outDir + "\\slamorig.obj.li"); + + psi = new ProcessStartInfo(Environment.ExpandEnvironmentVariables("slamlink.exe")); + psi.RedirectStandardError = true; + psi.RedirectStandardOutput = true; + psi.UseShellExecute = false; + psi.WorkingDirectory = outDir; + psi.Arguments = " --lib slamorig.obj slamlib.obj /out:slamout.obj"; + slamLinkProcess = System.Diagnostics.Process.Start(psi); + using (sw = new System.IO.StreamWriter(outDir + "\\smvlink.log", true)) + { + sw.Write(slamLinkProcess.StandardOutput.ReadToEnd()); + sw.Write(slamLinkProcess.StandardError.ReadToEnd()); + } + } + } + catch(Exception e) + { + Console.WriteLine(e.ToString()); + } + + if (File.Exists(outDir + "\\slamout.obj.li")) + { + File.Copy(outDir + "\\slamout.obj.li", outDir + "\\slam.li", true); + } + + } + } + #endregion + #region lib.exe + else if (args.Contains("/iwrap:lib.exe")) + { + Console.WriteLine("iwrap: Currently unimplemented. In general link functionality should be used."); + } + #endregion + } + + /// + /// Given a list of arguments, figures out what obj/lib files are + /// present, and returns the list + /// + /// arguments to process + /// list of libraries + static List GetLibs(string args) + { + // Extract all matches containing paths of lib and obj files which are enclosed by an optional space and does not have a space within + MatchCollection matchList = Regex.Matches(args, @"[\s]?([^\s""]+\.((lib)|(obj)))[\s]$?", RegexOptions.IgnoreCase); + List spaceTokens = matchList.Cast() + .Select(match => match.Value.Trim().Replace(@"/implib:", string.Empty)) + .Where(match => !match.StartsWith(@"/out", StringComparison.OrdinalIgnoreCase)) + .Where(match => !match.StartsWith(@"\\out", StringComparison.OrdinalIgnoreCase)) + .Select(match => System.IO.Path.GetDirectoryName(match.Trim())).ToList(); + + // Extract all matches containing paths of lib and obj files which are enclosed within quotes + matchList = Regex.Matches(args, @"""([^""]+\.(lib|obj))""", RegexOptions.IgnoreCase); + List quoteTokens = matchList.Cast() + .Select(match => match.Value.Trim().Replace(@"/implib:", string.Empty)) + .Where(match => !match.StartsWith(@"/out", StringComparison.OrdinalIgnoreCase)) + .Where(match => !match.StartsWith(@"\\out", StringComparison.OrdinalIgnoreCase)) + .Select(match => System.IO.Path.GetDirectoryName(match.Replace("\"", ""))).ToList(); + + return spaceTokens.Union(quoteTokens).ToList(); + } + + /// + /// Given arguments to compiler, gets the output folder + /// by matching on a regular expression + /// + /// arguments to process + /// compilers output directory + static string GetOutDirCl(string args) + { + string regex = String.Format(CultureInfo.InvariantCulture, "/F{0}\"(\\S+)\"|/F{0}(\\S+)", "[a|d|m|p|R|e|o|r|i]"); + Match match = Regex.Match(args, regex); + if(match.Success) + { + if (match.Groups[1].Success) return System.IO.Path.GetDirectoryName(match.Groups[1].Value); + if (match.Groups[2].Success) return System.IO.Path.GetDirectoryName(match.Groups[2].Value); + } + return Environment.GetEnvironmentVariable("OBJECT_ROOT"); + } + + /// + /// arguments to link are processed to figure out the + /// output directory of link + /// + /// arguments to process + /// output directory of link + static string GetOutDirLink(string args) + { + Match match = Regex.Match(args, "/out:\"(.[^\"]+)\"|/out:(\\S+)", RegexOptions.IgnoreCase); + string retVal = string.Empty; + if (match.Success) + { + if(match.Groups[1].Success) retVal = System.IO.Path.GetDirectoryName(match.Groups[1].Value); + if (match.Groups[2].Success) retVal = System.IO.Path.GetDirectoryName(match.Groups[2].Value); + } + + if (string.IsNullOrEmpty(retVal)) + { + retVal = Environment.GetEnvironmentVariable("OBJECT_ROOT"); + } + else + { + // check for relative path + if (!Path.IsPathRooted(retVal)) + { + retVal = Path.Combine(Environment.CurrentDirectory, retVal); + } + } + return retVal; + } + + /// + /// Extract build path from output + /// + /// Output of the build. + public static string ExtractBuildPath() + { + string lines = string.Empty; + using (FileStream fs = new FileStream("smvbuild.log", FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) + { + using (StreamReader sr = new StreamReader(fs)) + { + while (!sr.EndOfStream) + { + lines += sr.ReadLine() + Environment.NewLine; + } + } + } + + if (!String.IsNullOrEmpty(lines)) + { + lines = lines.Replace("\r\n", Environment.NewLine); + Match match = Regex.Match(lines, @"\/\/iwrap: outdir is (.*?)$", RegexOptions.Multiline); + string path = String.Empty; + + if (match.Success) + { + try + { + string key = match.Groups[1].Value; + path = key.Trim(); + } + catch (Exception) + { + Console.WriteLine("iwrap: Could not extract build path"); + } + Console.WriteLine("iwrap: Build path found - " + path); + return path; + } + else + { + Console.WriteLine("iwrap: Could not extract build path"); + return string.Empty; + } + } + else + { + Console.WriteLine("iwrap: Could not extract build path"); + return string.Empty; + } + } + } +} diff --git a/SmvInterceptorWrapper/Properties/AssemblyInfo.cs b/SmvInterceptorWrapper/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..b301ac6 --- /dev/null +++ b/SmvInterceptorWrapper/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("SmvInterceptorWrapper")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("SmvInterceptorWrapper")] +[assembly: AssemblyCopyright("Copyright © 2013")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("0eb021eb-9999-4aae-ba63-34a734ee5f95")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/SmvInterceptorWrapper/SmvInterceptorWrapper.csproj b/SmvInterceptorWrapper/SmvInterceptorWrapper.csproj new file mode 100644 index 0000000..7a682e0 --- /dev/null +++ b/SmvInterceptorWrapper/SmvInterceptorWrapper.csproj @@ -0,0 +1,60 @@ + + + + + Debug + AnyCPU + {8F9EA492-E86B-4F1C-ADC5-BF407466F448} + Exe + Properties + SmvInterceptorWrapper + SmvInterceptorWrapper + v4.5 + 512 + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + AllRules.ruleset + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/SmvInterceptorWrapper/iwrap.csproj b/SmvInterceptorWrapper/iwrap.csproj new file mode 100644 index 0000000..9d9dc2d --- /dev/null +++ b/SmvInterceptorWrapper/iwrap.csproj @@ -0,0 +1,58 @@ + + + + + Debug + AnyCPU + {8F9EA492-E86B-4F1C-ADC5-BF407466F448} + Exe + Properties + iwrap + iwrap + v4.5 + 512 + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/SmvLibrary/ActionsQueueEntry.cs b/SmvLibrary/ActionsQueueEntry.cs new file mode 100644 index 0000000..5097ec7 --- /dev/null +++ b/SmvLibrary/ActionsQueueEntry.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace SmvLibrary +{ + /// + /// Each instance of this class represents an entry in the ActionsQueue for the local and master action schedulers. + /// + public class ActionsQueueEntry + { + /// + /// The action to be executed. + /// + public SMVAction Action { get; set; } + + /// + /// This delegate is called after the action is complete. + /// + public SMVActionCompleteCallBack Callback { get; set; } + + /// + /// A list of results, one for this action and one for each action that was a child of this action. + /// + public List Results { get; set; } + + /// + /// Context object passed to the callback. + /// + public object Context { get; set; } + + public ActionsQueueEntry(SMVAction action, SMVActionCompleteCallBack callback, object context) + { + this.Action = action; + this.Callback = callback; + this.Context = context; + this.Results = new List(); + } + } +} diff --git a/SmvLibrary/CloudConfig.cs b/SmvLibrary/CloudConfig.cs new file mode 100644 index 0000000..d861537 --- /dev/null +++ b/SmvLibrary/CloudConfig.cs @@ -0,0 +1,29 @@ +using System.Collections.Generic; +using System.Xml.Serialization; + +namespace SmvLibrary +{ + [XmlRootAttribute(Namespace = "")] + public class SMVCloudConfig + { + [XmlElementAttribute("StorageConnectionString")] + public StorageConnectionStringElement StorageConnectionString; + + [XmlElementAttribute("ServiceBusConnectionString")] + public ServiceBusConnectionStringElement ServiceBusConnectionString; + } + + [XmlRootAttribute("StorageConnectionString", Namespace = "")] + public class StorageConnectionStringElement + { + [XmlAttributeAttribute()] + public string value { get; set; } + } + + [XmlRootAttribute("ServiceBusConnectionString", Namespace = "")] + public class ServiceBusConnectionStringElement + { + [XmlAttributeAttribute()] + public string value { get; set; } + } +} \ No newline at end of file diff --git a/SmvLibrary/CloudConfig.xsd b/SmvLibrary/CloudConfig.xsd new file mode 100644 index 0000000..af09923 --- /dev/null +++ b/SmvLibrary/CloudConfig.xsd @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/SmvLibrary/CloudConstants.cs b/SmvLibrary/CloudConstants.cs new file mode 100644 index 0000000..aaaf951 --- /dev/null +++ b/SmvLibrary/CloudConstants.cs @@ -0,0 +1,30 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace SmvLibrary +{ + /// + /// Common constants needed by both SmvCloudWorker and CloudSmvActionScheduler. + /// + public static class CloudConstants + { + public static int MaxRetries = 6; /// Maximum number of times to try to connect to blob storage before giving up. + public const int RetryBackoffInterval = 4; /// Time to wait in seconds before retrying a request. + public const int BeginReceiveTimeoutInHours = 9; /// Timeout for the BeginReceive() call. + public const string InputBlobContainerName = "smvactions"; /// Container action data is uploaded to. + public const string OutputBlobContainerName = "smvresults"; /// Container results are downloaded from. + public const string VersionsContainerName = "smvversions"; /// Container where different versions of SMV are stored. + public const string InputQueueName = "smvactions"; /// Name of the actions queue. + public const string ResultsTopicName = "smvresults"; /// Name of the service bus topic used to deliver result messages to the client. + public const string TableName = "actionstable"; /// Name of the table used to store information about the actions. + public const string CloudConfigXmlFileName = "cloudconfig.xml"; /// Name of the XML file that contains the connection strings. + public const string CloudConfigXsdFileName = "cloudconfig.xsd"; /// Name of XML schema file for the cloud config files. + public const int MaxDequeueCount = 5; /// The maximum number of times a message can be dequeued for processing. + public const string SmvWorkingDirectoryResourceName = "SMVWorking"; /// Used to get the path to the SMV working directory. + public const string SmvResultsDirectoryResourceName = "SMVResults"; /// Used to get the path to the results directory. + public const string SmvDirectoryResourceName = "SMVExec"; /// Used to get the location to SMV. + } +} diff --git a/SmvLibrary/CloudSMVActionScheduler.cs b/SmvLibrary/CloudSMVActionScheduler.cs new file mode 100644 index 0000000..71b1404 --- /dev/null +++ b/SmvLibrary/CloudSMVActionScheduler.cs @@ -0,0 +1,265 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Queue; +using Microsoft.WindowsAzure.Storage.Blob; +using Microsoft.WindowsAzure.Storage.Table; +using Microsoft.WindowsAzure.Storage.RetryPolicies; +using Microsoft.ServiceBus; +using Microsoft.ServiceBus.Messaging; +using System.Globalization; +using System.IO; +using System.IO.Compression; +using SMVActionsTable; + +namespace SmvLibrary +{ + public class CloudSMVActionScheduler : ISMVActionScheduler + { + private bool disposed = false; /// Used for implementing IDisposable. + private string schedulerInstanceGuid; /// A unique identifier for this instance of the CloudSMVActionScheduler object. + private string storageConnectionString; /// Connection string to connect to Azure storage. + private string serviceBusConnectionString; /// Connection string to connect to Azure Service Bus. + private CloudBlobContainer inputContainer; /// Container where actions are uploaded to. + private CloudBlobContainer outputContainer; /// Container where results are downloaded from. + private ActionsTableDataSource tableDataSource; /// Data source object used to query the actions table. + private CloudQueue actionsQueue; /// Cloud queue where each message is an action to be processed by a worker. + private NamespaceManager namespaceManager; /// Namespace manager used for working with service bus entities. + private SubscriptionClient subscriptionClient; /// Subscription client used to communicate with the service bus. + + private class CloudActionCompleteContext + { + public SMVAction action; + public SMVActionCompleteCallBack callback; + public object context; + + public CloudActionCompleteContext(SMVAction _action, SMVActionCompleteCallBack _callback, object _context) + { + action = _action; + callback = _callback; + context = _context; + } + } + + /// + /// This variable maps action GUIDs to CloudActionCompleteContext objects so we can get some information about the + /// action that completed when ActionComplete() is called. + /// + private Dictionary contextDictionary = new Dictionary(); + + public CloudSMVActionScheduler(SMVCloudConfig config) + { + // Set the instance GUID. + schedulerInstanceGuid = Guid.NewGuid().ToString(); + Log.LogInfo("Scheduler Instance GUID: " + schedulerInstanceGuid); + + // Check if the connection strings are set properly. + storageConnectionString = config.StorageConnectionString.value; + serviceBusConnectionString = config.ServiceBusConnectionString.value; + if (string.IsNullOrEmpty(storageConnectionString)) + { + throw new Exception("Connection string \"Microsoft.WindowsAzure.Storage.ConnectionString\" is not set. Please contact rahulku@microsoft.com for a " + + "valid connection string."); + } + if (string.IsNullOrEmpty(serviceBusConnectionString)) + { + throw new Exception("Connection string \"Microsoft.ServiceBus.ConnectionString\" is not set. Please contact rahulku@microsoft.com for a " + + "valid connection string."); + } + + int retriesLeft = CloudConstants.MaxRetries; + while (true) + { + try + { + // Connect to cloud storage. + var storageAccount = CloudStorageAccount.Parse(storageConnectionString); + var queueStorage = storageAccount.CreateCloudQueueClient(); + var blobStorage = storageAccount.CreateCloudBlobClient(); + var tableStorage = storageAccount.CreateCloudTableClient(); + + // Set up blob storage. + blobStorage.DefaultRequestOptions = new BlobRequestOptions(); + blobStorage.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(CloudConstants.RetryBackoffInterval), CloudConstants.MaxRetries); + inputContainer = blobStorage.GetContainerReference(CloudConstants.InputBlobContainerName); + inputContainer.CreateIfNotExists(); + outputContainer = blobStorage.GetContainerReference(CloudConstants.OutputBlobContainerName); + outputContainer.CreateIfNotExists(); + + // Set up our queue. + queueStorage.DefaultRequestOptions = new QueueRequestOptions(); + queueStorage.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(CloudConstants.RetryBackoffInterval), CloudConstants.MaxRetries); + actionsQueue = queueStorage.GetQueueReference(CloudConstants.InputQueueName); + actionsQueue.CreateIfNotExists(); + + // Set up table storage. + tableStorage.DefaultRequestOptions = new TableRequestOptions(); + tableStorage.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(CloudConstants.RetryBackoffInterval), CloudConstants.MaxRetries); + CloudTable table = tableStorage.GetTableReference(CloudConstants.TableName); + tableDataSource = new ActionsTableDataSource(table); + + // Set up the service bus subscription. + namespaceManager = NamespaceManager.CreateFromConnectionString(serviceBusConnectionString); + var filter = new SqlFilter(String.Format(CultureInfo.CurrentCulture, "(SchedulerInstanceGuid = '{0}')", schedulerInstanceGuid)); + if (!namespaceManager.TopicExists(CloudConstants.ResultsTopicName)) + { + namespaceManager.CreateTopic(CloudConstants.ResultsTopicName); + } + var subDesc = new SubscriptionDescription(CloudConstants.ResultsTopicName, schedulerInstanceGuid); + subDesc.AutoDeleteOnIdle = TimeSpan.FromDays(7.0); + if (!namespaceManager.SubscriptionExists(CloudConstants.ResultsTopicName, schedulerInstanceGuid)) + { + namespaceManager.CreateSubscription(subDesc, filter); + } + subscriptionClient = SubscriptionClient.CreateFromConnectionString(serviceBusConnectionString, + CloudConstants.ResultsTopicName, schedulerInstanceGuid); + subscriptionClient.RetryPolicy = new RetryExponential( + TimeSpan.FromSeconds(CloudConstants.RetryBackoffInterval), + TimeSpan.FromSeconds(CloudConstants.RetryBackoffInterval * CloudConstants.MaxRetries), + CloudConstants.MaxRetries); + subscriptionClient.OnMessage((msg) => + { + try + { + ActionComplete(msg); + } + catch (Exception) + { + msg.Abandon(); + } + }); + // If we're here, we've successfully initialized everything and can break out of the retry loop. + break; + } + catch (Exception e) + { + // We can fix the three exceptions below by using retry logic. But there's no point retrying for other exceptions. + if ((e is TimeoutException || e is ServerBusyException || e is MessagingCommunicationException) && retriesLeft > 0) + { + retriesLeft--; + } + else + { + throw; + } + } + System.Threading.Thread.Sleep(5000); + } + } + + public void AddAction(SMVAction action, SMVActionCompleteCallBack callback, object context) + { + string actionGuid = Guid.NewGuid().ToString(); + + // Upload action directory to blob storage. + + string actionPath = Utility.GetActionDirectory(action); + string zipPath = Path.GetTempFileName(); + File.Delete(zipPath); + ZipFile.CreateFromDirectory(actionPath, zipPath); + + CloudBlockBlob blob = inputContainer.GetBlockBlobReference(actionGuid + ".zip"); + blob.UploadFromFile(zipPath, FileMode.Open); + File.Delete(zipPath); + + // Add entry to table storage. + + // TODO: Due to constraints on sizes of properties in Azure table entities, serializedAction cannot be larger + // than 64kB. Fix this if this becomes an issue. + byte[] serializedAction = Utility.ObjectToByteArray(action); + string moduleHash = Utility.smvModule == null ? string.Empty : Utility.smvModule.Hash; + ActionsTableEntry entry = new ActionsTableEntry(action.name, actionGuid, schedulerInstanceGuid, serializedAction, + Utility.version, Utility.pluginPath, moduleHash); + tableDataSource.AddEntry(entry); + + // Add message to queue. + Log.LogInfo("Executing: " + action.GetFullName() + " [cloud id:" + actionGuid + "]"); + string messageString = schedulerInstanceGuid + "," + actionGuid; + var message = new CloudQueueMessage(messageString); + actionsQueue.AddMessage(message); + + // Start listening for a message from the service bus. + + contextDictionary[actionGuid] = new CloudActionCompleteContext(action, callback, context); + OnMessageOptions options = new OnMessageOptions(); + options.AutoComplete = false; + options.AutoRenewTimeout = TimeSpan.FromHours(CloudConstants.BeginReceiveTimeoutInHours); + + //subscriptionClient.BeginReceive(TimeSpan.FromHours(CloudConstants.BeginReceiveTimeoutInHours), + // new AsyncCallback(ActionComplete), null); + } + + /// + /// Callback that is called when SubscriptionClient.BeginReceive() receives a message. + /// + /// + private void ActionComplete(BrokeredMessage message) + { + + var actionGuid = (string)message.Properties["ActionGuid"]; + var waitTime = (TimeSpan)message.Properties["WaitTime"]; // The amount of time the rule had to wait before it started being processed. + var dequeueCount = (int)message.Properties["DequeueCount"]; // The number of times the message we sent was dequeued by the workers. + + message.Complete(); + + CloudActionCompleteContext context = contextDictionary[actionGuid]; + ActionsTableEntry entry = tableDataSource.GetEntry(schedulerInstanceGuid, actionGuid); + var action = (SMVAction)Utility.ByteArrayToObject(entry.SerializedAction); + + Console.WriteLine("ActionComplete for " + action.GetFullName() + " [cloud id " + actionGuid + "]"); + + // Populate the original action object so that the master scheduler gets the changes to the action object. + context.action.analysisProperty = action.analysisProperty; + context.action.result = action.result; + context.action.variables = action.variables; + + var results = new SMVActionResult[] { context.action.result }; + if(entry.Status != (int)ActionStatus.Complete) + { + Log.LogError(string.Format("Failed to complete action: {0} ({1})", actionGuid, context.action.name)); + context.callback(new SMVActionResult[] { context.action.result }, context.context); + } + + // Download and extract the results. + CloudBlockBlob resultsBlob = outputContainer.GetBlockBlobReference(actionGuid + ".zip"); + string zipPath = Path.GetTempFileName(); + File.Delete(zipPath); + resultsBlob.DownloadToFile(zipPath, FileMode.CreateNew); + resultsBlob.Delete(); + string actionDirectory = Utility.GetActionDirectory(context.action); + Utility.ClearDirectory(actionDirectory); + ZipFile.ExtractToDirectory(zipPath, actionDirectory); + File.Delete(zipPath); + + // Write to the cloudstats.txt file. + using (StreamWriter writer = File.AppendText(Path.Combine(actionDirectory, "cloudstats.txt"))) + { + writer.WriteLine("Wait Time: " + waitTime.ToString()); + writer.WriteLine("Dequeue Count: " + dequeueCount); + } + + context.callback(results, context.context); + } + + protected virtual void Dispose(bool disposing) + { + if (!disposed) + { + if (disposing) + { + // Clean up managed resources. + } + } + disposed = true; + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + } +} diff --git a/SmvLibrary/GlobalSuppressions.cs b/SmvLibrary/GlobalSuppressions.cs new file mode 100644 index 0000000..4e055a9 Binary files /dev/null and b/SmvLibrary/GlobalSuppressions.cs differ diff --git a/SmvLibrary/ISMVActionScheduler.cs b/SmvLibrary/ISMVActionScheduler.cs new file mode 100644 index 0000000..2399011 --- /dev/null +++ b/SmvLibrary/ISMVActionScheduler.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Runtime.Serialization; + +namespace SmvLibrary +{ + /// + /// Delegate used by the scheduler to indicate that an action and all its children have completed. + /// + /// The results of all the actions that have been executed as a result of this action. + /// Context object passed to the callback. + public delegate void SMVActionCompleteCallBack(IEnumerable results, object context); + + /// + /// Defines an interface for classes that want to support scheduling SMVActions. + /// + public interface ISMVActionScheduler : IDisposable + { + /// + /// Adds an action to be scheduled. Once the action has completed, will be called with + /// as the argument. + /// + /// The action to be performed. + /// Delegate that will be called once the action has been performed. + /// Object passed to the delegate when the action has been performed. + void AddAction(SMVAction action, SMVActionCompleteCallBack callback, object context); + } +} diff --git a/SmvLibrary/ISMVPlugin.cs b/SmvLibrary/ISMVPlugin.cs new file mode 100644 index 0000000..a148fbb --- /dev/null +++ b/SmvLibrary/ISMVPlugin.cs @@ -0,0 +1,58 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace SmvLibrary +{ + public interface ISMVPlugin + { + /// + /// Initialize the Plugin + /// + void Initialize(); + + /// + /// Prints help text specific to the plugin + /// + void PrintPluginHelp(); + + /// + /// Process custom arguments for use by the Plugin + /// + /// The arguments passed to smv. + void ProcessPluginArgument(string[] args); + + /// + /// Called before an action is run + /// + /// The action being run. + void PreAction(SMVAction action); + + /// + /// Called after an action is run + /// + /// The action being run. + void PostAction(SMVAction action); + + /// + /// Called after build, if build node is present in the SMVConfig. + /// + /// List of build actions. + void PostBuild(SMVAction[] buildActions); + + /// + /// Do analysis if /analyze argument is not passed to SMV + /// + /// List of analysis actions. + /// true on success, false on failure. + bool DoPluginAnalysis(SMVAction[] analysisActions); + + /// + /// Called after analysis, if analysis node is present in the SMVConfig. + /// + /// List of result of the analysis actions. + void PostAnalysis(SMVAction[] analysisActions); + } +} diff --git a/SmvLibrary/LocalSMVActionScheduler.cs b/SmvLibrary/LocalSMVActionScheduler.cs new file mode 100644 index 0000000..9b0d91e --- /dev/null +++ b/SmvLibrary/LocalSMVActionScheduler.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections.Generic; +using System.Collections.Concurrent; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Threading; + +namespace SmvLibrary +{ + public class LocalSMVActionScheduler : ISMVActionScheduler + { + private ConcurrentQueue actionsQueue = new ConcurrentQueue(); + private List executeThreads = new List(); + + private bool disposed = false; + + /// + /// Starts threads, each of which will start dequeueing actions from the actions + /// queue. The threads stop executing once the object is disposed of. + /// + /// The number of threads to start. + public LocalSMVActionScheduler(int numberOfThreads) + { + for(int i = 0; i < numberOfThreads; i++) + { + Thread t = new Thread(new ThreadStart(Execute)); + executeThreads.Add(t); + t.Start(); + } + } + + public void AddAction(SMVAction action, SMVActionCompleteCallBack callback, object context) + { + var entry = new ActionsQueueEntry(action, callback, context); + actionsQueue.Enqueue(entry); + } + + private void Execute() + { + try + { + while(true) + { + ActionsQueueEntry entry; + while(actionsQueue.TryDequeue(out entry)) + { + Log.LogInfo("Executing: " + entry.Action.GetFullName()); + SMVActionResult result = Utility.ExecuteAction(entry.Action); + entry.Results.Add(result); + entry.Callback(entry.Results, entry.Context); + } + Thread.Sleep(2000); + } + } + catch(ThreadAbortException) + { + // Do nothing here, we just need this so we can call Thread.Abort() to kill the thread. + } + } + + protected virtual void Dispose(bool disposing) + { + if (!disposed) + { + if (disposing) + { + // Clean up managed resources. + foreach(Thread t in executeThreads) + { + t.Abort(); + } + } + } + disposed = true; + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + } +} diff --git a/SmvLibrary/MasterSMVActionScheduler.cs b/SmvLibrary/MasterSMVActionScheduler.cs new file mode 100644 index 0000000..4a29baa --- /dev/null +++ b/SmvLibrary/MasterSMVActionScheduler.cs @@ -0,0 +1,149 @@ +using System; +using System.Collections.Generic; +using System.Collections.Concurrent; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace SmvLibrary +{ + public class MasterSMVActionScheduler : ISMVActionScheduler + { + private ConcurrentQueue actionsQueue = new ConcurrentQueue(); + private IDictionary schedulers = new Dictionary(); + + private bool disposed = false; + private bool done = false; + + private delegate void ExecuteDelegate(); + + /// + /// Start running a thread that handles dequeueing actions from the actionsQueue and sending them to the appropriate scheduler. + /// + public MasterSMVActionScheduler() + { + new ExecuteDelegate(this.Execute).BeginInvoke(null, null); + } + + /// + /// Add a new scheduler to the list of schedulers used by the master scheduler. + /// + /// The type of scheduler you're adding. E.g., "local", "cloud" + /// The scheduler to be added. + public void AddScheduler(string type, ISMVActionScheduler scheduler) + { + schedulers[type] = scheduler; + } + + public void AddAction(SMVAction action, SMVActionCompleteCallBack callback, object context) + { + Log.LogInfo("Queuing action: " + action.GetFullName()); + var entry = new ActionsQueueEntry(action, callback, context); + actionsQueue.Enqueue(entry); + } + + /// + /// This function runs in its own thread, dequeueing actions from the actions queue and sending them to the + /// appropriate scheduler. + /// + private void Execute() + { + while (!done) + { + ActionsQueueEntry entry; + while (!done && actionsQueue.TryDequeue(out entry)) + { + SMVAction action = entry.Action; + string schedulerType = action.executeOn; + + if (!schedulers.ContainsKey(schedulerType)) + { + Log.LogFatalError("Could not find scheduler of type: " + schedulerType + + " while executing action " + action.name); + } + else + { + ISMVActionScheduler scheduler = schedulers[schedulerType]; + lock (Utility.lockObject) + { + Utility.result.Add(action.GetFullName(), "Skipped"); + } + scheduler.AddAction(action, new SMVActionCompleteCallBack(ActionComplete), entry); + } + } + System.Threading.Thread.Sleep(2000); + } + } + + /// + /// This callback function is called once an action and all its children have executed. + /// + /// A list of results, one for each action (the action added to the queue and its children). + /// A context object. + private void ActionComplete(IEnumerable results, object context) + { + var entry = context as ActionsQueueEntry; + SMVAction action = entry.Action; + SMVActionCompleteCallBack callback = entry.Callback; + entry.Results.AddRange(results); + + Log.LogInfo("Completed action: " + action.GetFullName()); + + // Add result to our global result set. + string result = "Failed"; + if(action.result != null && action.result.isSuccessful) + { + result = "Success"; + } + lock (Utility.lockObject) + { + Utility.result[action.GetFullName()] = result; + } + + // If there was an error, simply call the callback function with whatever results we have, the callback is + // expected to handle the errors by looking at the list of results. + if (action.result == null || action.result.breakExecution) + { + entry.Callback(entry.Results, entry.Context); + } + // Otherwise, add the next action to the queue, if any. + else + { + SMVAction nextAction = Utility.GetNextAction(action); + + if (nextAction != null) + { + nextAction.analysisProperty = action.analysisProperty; + nextAction.variables = new Dictionary(entry.Action.variables); + //var newEntry = new ActionsQueueEntry(nextAction, entry.Callback, entry.Context); + //newEntry.Results = entry.Results; + //actionsQueue.Enqueue(newEntry); + this.AddAction(nextAction, entry.Callback, entry.Context); + } + else + { + entry.Callback(entry.Results, entry.Context); + } + } + } + + protected virtual void Dispose(bool disposing) + { + if(!disposed) + { + if(disposing) + { + // Clean up managed resources. + done = true; + } + } + disposed = true; + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + } +} diff --git a/SmvLibrary/Properties/AssemblyInfo.cs b/SmvLibrary/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..a779950 --- /dev/null +++ b/SmvLibrary/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("SMVLibrary")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("SMVLibrary")] +[assembly: AssemblyCopyright("Copyright © 2014")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("b7dc81e0-23b7-44d6-a136-9967ac20548c")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/SmvLibrary/SmvLibrary.csproj b/SmvLibrary/SmvLibrary.csproj new file mode 100644 index 0000000..d42a185 --- /dev/null +++ b/SmvLibrary/SmvLibrary.csproj @@ -0,0 +1,123 @@ + + + + + Debug + AnyCPU + {E03DC0DF-84CC-420C-91B1-2A937E88FF31} + Library + Properties + SmvLibrary + SmvLibrary + v4.5 + 512 + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + AllRules.ruleset + true + + + + ..\packages\Microsoft.Data.Edm.5.6.2\lib\net40\Microsoft.Data.Edm.dll + True + + + ..\packages\Microsoft.Data.OData.5.6.2\lib\net40\Microsoft.Data.OData.dll + True + + + ..\packages\WindowsAzure.ServiceBus.3.2.0\lib\net45-full\Microsoft.ServiceBus.dll + True + + + + + + + + + + + ..\packages\System.Spatial.5.6.2\lib\net40\System.Spatial.dll + True + + + + + + + + + + + CloudConfig.xsd + + + + + + + Config.xsd + + + + + + + + + + + + Designer + + + Designer + + + + + + {5a923bec-71fc-48e3-b076-2a2aa81ebd03} + SmvAccessor + + + {6f655d35-3867-4351-8929-23b4578e5190} + SMVActionsTable + + + + + + + + + + + copy $(ProjectDir)Config.xsd $(TargetDir) +copy $(ProjectDir)CloudConfig.xsd $(TargetDir) +copy $(ProjectDir)CloudConfig.xml $(TargetDir) + + + \ No newline at end of file diff --git a/SmvLibrary/Utility.cs b/SmvLibrary/Utility.cs new file mode 100644 index 0000000..ada5edd --- /dev/null +++ b/SmvLibrary/Utility.cs @@ -0,0 +1,1047 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Collections.Concurrent; +using System.Collections.Specialized; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using System.Xml; +using System.Xml.Schema; +using SmvAccessor; +using System.Configuration; +using System.Globalization; +using System.Threading; +using System.Runtime.Serialization.Formatters.Binary; + +[assembly: CLSCompliant(true)] +namespace SmvLibrary +{ + sealed public class Utility + { + public static SmvAccessor.Module smvModule = null; /// The module if a module and plugin is provided. + public static ISmvAccessor accessor; /// Accessor object used to interact with the data source. + public static ISMVPlugin plugin = null; /// The plugin used for this run of SMV. + public static string pluginPath = string.Empty; /// Name of the plugin. Used for cloud scheduling. + public static MasterSMVActionScheduler scheduler; /// Used to schedule actions. + public static OrderedDictionary result = new OrderedDictionary(); /// Stores the result of the actions. + public static string version; /// Name for this version of SMV. Used for cloud scheduling. + public static Dictionary smvVars = new Dictionary(); /// Dictionary to store the current run specific variables + public static bool debugMode = false; + + private static IDictionary actionsDictionary = new Dictionary(); + public static object lockObject = new object(); + private static List actionResults; + + + private Utility() { } + + /// + /// Add actions to the actions dictionary that will be used by the scheduler. + /// + /// The list of actions to be added to the dictionary. + public static void PopulateActionsDictionary(IEnumerable actions) + { + foreach (SMVAction action in actions) + { + actionsDictionary.Add(action.name, action); + } + } + + /// + /// Returns all the root actions (actions which are not the nextAction of any action). + /// + /// The list of actions. + public static SMVAction[] GetRootActions(IEnumerable actionList) + { + List result = new List(); + List dependentActions = new List(); + + foreach (SMVAction action in actionList) + { + if (!string.IsNullOrEmpty(action.nextAction)) + { + dependentActions.Add(action.nextAction); + } + } + + foreach (SMVAction action in actionList) + { + if (!dependentActions.Contains(action.name)) + { + result.Add(action); + } + } + + return result.ToArray(); + } + + /// + /// Helper function that reads all lines in the files into one string, each line separated by a newline. + /// + /// The file to be read. + /// The contents of the file. + public static string ReadFile(string filePath) + { + string lines = string.Empty; + + try + { + using (FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) + { + using (StreamReader sr = new StreamReader(fs)) + { + while (!sr.EndOfStream) + { + lines += sr.ReadLine() + Environment.NewLine; + } + } + } + + return lines; + } + catch (FileNotFoundException e) + { + Log.LogError(e.Message); + return null; + } + } + + /// + /// Copies a file and prints a logging message. + /// + /// Path to the file to be copied. + /// Destination path of the file. Cannot be a directory. + public static void CopyFile(string source, string destination, TextWriter logger) + { + Log.LogInfo(String.Format(CultureInfo.InvariantCulture, "Copying file {0} to {1}.", source, destination), logger); + File.Copy(source, destination, true); + } + + /// + /// Copy directory contents recursively + /// + public static void CopyDirectory(string sourceDirName, string destDirName, bool copySubDirs) + { + DirectoryInfo dir = new DirectoryInfo(sourceDirName); + DirectoryInfo[] dirs = dir.GetDirectories(); + + if (!dir.Exists) + { + throw new DirectoryNotFoundException( + "Source directory does not exist or could not be found: " + + sourceDirName); + } + + if (!Directory.Exists(destDirName)) + { + Directory.CreateDirectory(destDirName); + } + + FileInfo[] files = dir.GetFiles(); + foreach (FileInfo file in files) + { + string temppath = Path.Combine(destDirName, file.Name); + file.CopyTo(temppath, true); + } + + if (copySubDirs) + { + foreach (DirectoryInfo subdir in dirs) + { + string temppath = Path.Combine(destDirName, subdir.Name); + CopyDirectory(subdir.FullName, temppath, copySubDirs); + } + } + } + + /// + /// Deletes a file and prints a logging message. + /// + /// Path to the file to be deleted. + public static void DeleteFile(string file, TextWriter logger) + { + Log.LogInfo(String.Format(CultureInfo.InvariantCulture, "Deleting file {0}", file), logger); + File.SetAttributes(file, FileAttributes.Normal); + File.Delete(file); + } + + /// + /// Validates the config file against the schema. + /// + /// Path to the schema file. + /// Content of the config file + /// Boolean based on whether the validation failed + public static bool ValidateXmlFile(string schemaPath, TextReader configFile) + { + XmlReader reader; + Log.LogInfo("Validating XML against schema: " + schemaPath); + + // Load and validate the XML document. + XmlReaderSettings settings = new XmlReaderSettings(); + settings.Schemas.Add("", schemaPath); + settings.ValidationType = ValidationType.Schema; + + try + { + reader = XmlReader.Create(configFile, settings); + } + catch (FileNotFoundException) + { + Log.LogError("Config file not found. Put it in the current directory or pass it in the arguments [/config:]"); + return false; + } + + ValidationEventHandler handler = new ValidationEventHandler(ValidationEventHandlerCallback); + XmlDocument doc = new XmlDocument(); + + try + { + doc.Load(reader); + doc.Validate(handler); + } + catch (Exception e) + { + Log.LogError(e.ToString()); + return false; + } + + // We do some validation ourselves because we can't use XSD 1.1 yet. + foreach (XmlNode actionNode in doc.SelectNodes("SMVConfig/Analysis/Action")) + { + foreach (XmlNode copyArtifactNode in actionNode.SelectNodes("CopyArtifact")) + { + string type = Environment.ExpandEnvironmentVariables(copyArtifactNode.Attributes["type"].Value).ToLowerInvariant(); + string entity = Environment.ExpandEnvironmentVariables(copyArtifactNode.Attributes["entity"].Value).ToLowerInvariant(); + + if ((type == "rawcfgf" && entity != "compilationunit") || + (type == "li" && entity == "functionunit") || + (type == "bpl" && entity == "compilationunit")) + { + Log.LogError(String.Format(CultureInfo.InvariantCulture, "\"type\" attribute cannot have value \"{0}\" when \"entity\" attribute has value \"{1}\".", type, entity)); + return false; + } + } + } + + return true; + } + + /// + /// Callback used by XmlDocument.Validate. + /// + /// Not used. + /// Not used. + private static void ValidationEventHandlerCallback(object sender, ValidationEventArgs e) + { + switch (e.Severity) + { + case XmlSeverityType.Error: + Log.LogError(e.Message); + break; + case XmlSeverityType.Warning: + Log.LogWarning(e.Message); + break; + } + } + + /// + /// Get the path to an action's working directory. + /// + /// The action object. + /// Full path to the action's working directory. + public static string GetActionDirectory(SMVAction action) + { + string path = string.Empty; + if(action.Path != null && action.Path.value != null) + { + path = action.Path.value; + } + return Utility.ExpandVariables(path, action.variables); + } + + /// + /// Deletes all files and directories inside a directory. + /// + /// The directory to clear. + public static void ClearDirectory(string directory) + { + DirectoryInfo di = new DirectoryInfo(directory); + + foreach (FileInfo f in di.GetFiles()) + { + f.Delete(); + } + + foreach (DirectoryInfo d in di.GetDirectories()) + { + d.Delete(true); + } + } + + /// + /// Serializes an object into an array of bytes. + /// + /// The object to be serialized. + /// The array of bytes representing the serialized object. + public static byte[] ObjectToByteArray(object obj) + { + if(obj == null) + { + return null; + } + var bf = new BinaryFormatter(); + using(var ms = new MemoryStream()) + { + bf.Serialize(ms, obj); + return ms.ToArray(); + } + } + + /// + /// Deserializes an object from an array of bytes. + /// + /// The array to be used for deserialization. + /// The deserialized object. + public static object ByteArrayToObject(byte[] bytes) + { + var bf = new BinaryFormatter(); + using(var ms = new MemoryStream()) + { + ms.Write(bytes, 0, bytes.Length); + ms.Seek(0, SeekOrigin.Begin); + return bf.Deserialize(ms); + } + } + + /// + /// Launches a process and executes the given command with the args provided. + /// + /// Command to execute. + /// Arguments to pass. + /// Directory in which to start the process + /// Environment variables + /// The process on success, null on failure. + public static Process LaunchProcess(String cmd, String args, string startDirectory, SMVEnvVar[] env, TextWriter logger) + { + try + { + var psi = new ProcessStartInfo(cmd, args); + psi.RedirectStandardError = true; + psi.RedirectStandardInput = true; + psi.RedirectStandardOutput = true; + psi.UseShellExecute = false; + + // Set environment variables for this process. + SetEnvironmentVariables(psi, env, logger); + + Log.LogInfo(String.Format(CultureInfo.InvariantCulture, "Launching {0} with arguments: {1} ", cmd, args), logger); + + if (!String.IsNullOrEmpty(startDirectory)) + { + psi.WorkingDirectory = startDirectory; + Log.LogInfo("PATH: " + startDirectory, logger); + } + + return Process.Start(psi); + } + catch (Exception) + { + Log.LogFatalError(String.Format(CultureInfo.InvariantCulture, "Could not start process: {0} with args {1}, working directory: {2}", cmd, args, startDirectory)); + return null; + } + } + + /// + /// Sets certain SMV variables related to a module's attributes. + /// + /// The module whose attributes will be used. + /// The dictionary where the variables will be set. If this is null, the global variables dictionary is used. + static void SetModuleVariables(SmvAccessor.Module module, IDictionary dict = null) + { + List invalidChars = Path.GetInvalidFileNameChars().ToList(); + // We need to remove % characters from the paths to prevent environment variable expansion when trying to access these paths from the command line. + invalidChars.Add('%'); + string moduleName = module.Name; + foreach (char ch in invalidChars) + { + moduleName = moduleName.Replace(ch.ToString(), "."); + } + string dateCreated = module.DateCreated.ToString().Replace("/", "-").Replace(":", ".").Replace(" ", "."); + string lastModified = module.LastModified.ToString().Replace("/", "-").Replace(":", ".").Replace(" ", "."); + + if(dict == null) + { + dict = smvVars; + } + + dict["ModuleName"] = moduleName; + dict["ModuleDateCreated"] = dateCreated; + dict["ModuleId"] = module.Id.ToString(CultureInfo.InvariantCulture); + dict["ModuleLastModified"] = lastModified; + } + + /// + /// Gets the list of artifacts associated with the compilation units of a module. + /// + /// The accessor object used to interact with the data source. + /// The module whose artifacts we'll be getting. + /// The type of artifacts to get. + /// A list of artifacts for compilation units of a certain module. + static IEnumerable GetCompilationUnitArtifactsForModule(ISmvAccessor accessor, SmvAccessor.Module module, string type) + { + var artifacts = new List(); + IEnumerable cus = accessor.GetCompilationUnitsByModuleId(module.Id); + foreach (var cu in cus) + { + IEnumerable cuArtifacts = accessor.GetArtifactsForCompilationUnit(cu).Where(artifact => artifact.Type == type); + artifacts.InsertRange(0, cuArtifacts); + } + return artifacts; + } + + /// + /// Gets the list of artifacts associated with the function units of a module. + /// + /// The accessor object used to interact with the data source. + /// The module whose artifacts we'll be getting. + /// The type of artifacts to get. + /// A list of artifacts for function units of a certain module. + static IEnumerable GetFunctionUnitArtifactsForModule(ISmvAccessor accessor, SmvAccessor.Module module, string type) + { + var artifacts = new List(); + IEnumerable fus = accessor.GetFunctionUnitsByModuleId(module.Id); + foreach (var fu in fus) + { + IEnumerable fuArtifacts = accessor.GetArtifactsForFunctionUnit(fu).Where(artifact => artifact.Type == type); + artifacts.InsertRange(0, fuArtifacts); + } + return artifacts; + } + + /// + /// Copies artifacts for a module into a directory. + /// + /// The module whose artifacts are to be copied. + /// The type of artifact to be copied, e.g., "li", "bpl", "rawcfgf" + /// The entity whose artifacts will be copied, e.g., "module", "compilationunit", "functionunit". + /// The directory the artifacts will be copied to. + public static void CopyArtifacts(Module module, string type, string entity, string destDir, TextWriter logger = null) + { + Log.LogInfo("Copying files of type: " + type, logger); + Log.LogInfo("Copying files for entity: " + entity, logger); + + // Delete all files in the artifact directories before copying. Don't delete config.xml. + Directory.CreateDirectory(destDir); + Log.LogInfo(String.Format(CultureInfo.InvariantCulture, "Using {0} as destination directory..", destDir), logger); + Log.LogInfo("Deleting all files in " + destDir, logger); + foreach (string file in Directory.EnumerateFiles(destDir)) + { + Utility.DeleteFile(file, logger); + } + + // Copy artifacts based on entity type. + ISmvAccessor dbAccessor = GetSmvSQLAccessor(); + IEnumerable artifacts = null; + switch (entity) + { + case "module": + artifacts = dbAccessor.GetArtifactsForModule(module).Where(artifact => artifact.Type == type); + break; + + case "compilationunit": + artifacts = GetCompilationUnitArtifactsForModule(dbAccessor, module, type); + break; + + case "functionunit": + artifacts = GetFunctionUnitArtifactsForModule(dbAccessor, module, type); + break; + } + foreach (var artifact in artifacts) + { + string destPath = Path.Combine(destDir, artifact.Name); + Utility.CopyFile(artifact.Path, destPath, logger); + } + } + + /// + /// Copies artifacts based on a list of COPYARTIFACT nodes. + /// + /// The list of COPYARTIFACT nodes. + /// The module whose artifacts will be copied. + /// The directory the artifacts will be copied to. + /// Variables dictionary for the action that is the parent of the copy artifact nodes currently being processed. + static void CopyArtifacts(CopyArtifactType[] copyArtifactNodes, SmvAccessor.Module module, string workingDir, IDictionary variables, + TextWriter logger) + { + if (copyArtifactNodes != null) + { + Log.LogInfo("Copying artifacts for module: " + module.Name, logger); + foreach (CopyArtifactType node in copyArtifactNodes) + { + Log.LogInfo("Processing CopyArtifact node..", logger); + + // Get attributes from the node. + string name = Environment.ExpandEnvironmentVariables(node.name).ToLowerInvariant(); + string type = Environment.ExpandEnvironmentVariables(node.type).ToLowerInvariant(); + string entity = Environment.ExpandEnvironmentVariables(node.entity.ToString()).ToLowerInvariant(); + string dir = node.to; + + if (dir == null) + { + dir = String.Empty; + } + + dir = Environment.ExpandEnvironmentVariables(dir); + + // Set a variable with the name of the node as the key and the directory as the value; we'll need this + // when we run commands for this action. + string destDir = Path.Combine(workingDir, dir); + destDir = Utility.ExpandVariables(destDir, variables); + + string copyArtifactVar = string.Empty; + if (variables.ContainsKey(name)) + { + copyArtifactVar = variables[name]; + } + copyArtifactVar += " " + destDir; + variables[name] = copyArtifactVar; + + // Copy the artifacts. + CopyArtifacts(module, type, entity, destDir, logger); + } + } + } + + /// + /// Gets the child action for an action. + /// + /// The parent action. + /// The child action if one exists, else null. + public static SMVAction GetNextAction(SMVAction action) + { + if(action.nextAction == null || !actionsDictionary.ContainsKey(action.nextAction)) + { + return null; + } + SMVAction template = actionsDictionary[action.nextAction]; + SMVAction nextAction = new SMVAction(template, string.Empty); + return nextAction; + } + + /// + /// Helper function that calls adds an array of actions to Utitlity.scheduler, waits until they all complete + /// and returns the results. Call Utility.scheduler.AddAction() directory for finer-grained control. + /// + /// The list of actions to be executed. + /// The list of results of the actions that were executed. + public static List ExecuteActions(SMVAction[] actions) + { + var waitHandle = new CountdownEvent(actions.Length); + actionResults = new List(); + + foreach (SMVAction action in actions) + { + action.variables = new Dictionary(Utility.smvVars); + if (!string.IsNullOrEmpty(action.analysisProperty)) + { + action.variables.Add("analysisProperty", action.analysisProperty); + } + Utility.scheduler.AddAction(action, new SMVActionCompleteCallBack(DoneExecuteAction), waitHandle); + } + + waitHandle.Wait(); + return actionResults; + } + + /// + /// Call back used by ExecuteActions(). + /// + /// + /// + static void DoneExecuteAction(IEnumerable results, object context) + { + actionResults.AddRange(results); + + var countDownEvent = context as CountdownEvent; + countDownEvent.Signal(); + } + + /// + /// Returns a singleton accessor object. + /// + /// The accessor object. + public static ISmvAccessor GetSmvSQLAccessor() + { + if (accessor == null) + { + // Connect to the database. + string connectionString = ConfigurationManager.ConnectionStrings["SmvDbConnectionString"].ConnectionString; + Log.LogInfo("Connecting to database with connection string: " + connectionString); + accessor = new SmvSqlAccessor(connectionString); + } + return accessor; + } + + /// + /// Called by schedulers to execute an action. + /// + /// The action to be executed. + /// An SMVActionResult object representing the result of executing the action. + public static SMVActionResult ExecuteAction(SMVAction action) + { + // NOTE: The code in this function must be thread safe. + if(action == null) + { + return null; + } + + // If there is a plugin, call PreAction first. + if(plugin != null) + { + plugin.PreAction(action); + } + + using(MemoryStream stream = new MemoryStream()) + { + // We use a logger for writing messages since we can't output to the console in this function (As this + // may be running in multiple threads). + StreamWriter logger = new StreamWriter(stream); + IDictionary variables = action.variables; + string actionPath = variables["workingDir"]; + string actionOutput = string.Empty; + + // Get the name of the action. + string name = action.name; + if (variables.ContainsKey("analysisProperty")) + { + name = action.name + " - " + variables["analysisProperty"]; + } + variables["name"] = action.name; + Log.LogInfo("Running action: " + name, logger); + + // Get the path to the action. + if (action.Path != null) + { + actionPath = action.Path.value; + } + actionPath = ExpandVariables(actionPath, variables); + variables["actionPath"] = actionPath; + + // Launch a cmd.exe process to run commands in. + Process process = LaunchProcess("cmd.exe", "", actionPath, action.Env, logger); + if (process == null) + { + Log.LogFatalError("Could not launch process."); + } + + process.OutputDataReceived += (sender, e) => + { + Log.LogMessage(e.Data, logger); + actionOutput += e.Data + Environment.NewLine; + }; + + process.ErrorDataReceived += (sender, e) => + { + Log.LogMessage(e.Data, logger); + actionOutput += e.Data + Environment.NewLine; + }; + + // Copy artifacts and set module variables if there's a module involved. + if (smvModule != null) + { + SetModuleVariables(smvModule, variables); + CopyArtifacts(action.CopyArtifact, smvModule, variables["workingDir"], variables, logger); + } + + // Run the commands. + if (action.Command != null) + { + foreach (SMVCommand cmd in action.Command) + { + // Get the command and arguments, and expand all environment as well as SMV variables. + string cmdAttr = ExpandVariables(Environment.ExpandEnvironmentVariables(cmd.value), variables); + string argumentsAttr = string.Empty; + if (!string.IsNullOrEmpty(cmd.arguments)) + { + argumentsAttr = ExpandVariables(Environment.ExpandEnvironmentVariables(cmd.arguments), variables); + } + + try + { + Log.LogInfo(String.Format(CultureInfo.InvariantCulture, "Launching {0} with arguments: {1}", cmdAttr, argumentsAttr), logger); + process.StandardInput.WriteLine(String.Join(" ", new String[] { cmdAttr, argumentsAttr })); + } + catch (Exception e) + { + Log.LogInfo(e.ToString(), logger); + Log.LogInfo("Could not start process: " + cmdAttr, logger); + return null; + } + } + } + + process.StandardInput.WriteLine("Exit %errorlevel%"); + process.StandardInput.Close(); + + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + process.WaitForExit(); + + logger.Flush(); + stream.Position = 0; + string output = string.Empty; + + var sr = new StreamReader(stream); + output = sr.ReadToEnd(); + + if(debugMode) + { + Log.LogInfo(output); + } + + // Write logs to file if log argument is provided to SMV. + Log.WriteToFile(name, output); + + action.result = new SMVActionResult(action.name, output, (process.ExitCode == 0), + process.ExitCode != 0 && action.breakOnError); + + // Call plugin post action only if we were successful in executing the action. + if (process.ExitCode == 0) + { + // get the output directory and set the output of the action from the build log. + if (action.name.Equals("NormalBuild")) + { + string logPath = Path.Combine(variables["workingDir"], variables["smvLogFileNamePrefix"] + ".log"); + action.result.output = Utility.ReadFile(logPath); + + variables["outputDir"] = ExtractBuildPath(variables["workingDir"], action.result.output, logger); + Utility.SetSmvVar("outputDir", variables["outputDir"]); + } + + // Get the output directory and the analysis directory. + if (action.name.Equals("InterceptedBuild")) + { + string logPath = Path.Combine(variables["workingDir"], variables["smvLogFileNamePrefix"] + ".log"); + action.result.output = Utility.ReadFile(logPath); + } + + // Call the plugin's post action. + if (plugin != null) + { + plugin.PostAction(action); + } + } + else + { + Log.LogFatalError(String.Format("Action: {0}, failed.", name)); + } + + return action.result; + } + } + + /// + /// Checks the result of ExecuteActions() for success. + /// + /// The result from ExecuteActions(). + /// True on success, false on failure. + public static bool IsExecuteActionsSuccessful(List actionsResult) + { + foreach (SMVActionResult result in actionResults) + { + if (result.breakExecution) + { + return false; + } + } + + return true; + } + + /// + /// This function is needed because it supports case insenstive string replacement. + /// + /// The string to be processed. + /// The string to be replace. + /// The string that will replace + /// The type of comparison we will do. + /// The string with replacements. + public static string StringReplace(string str, string oldValue, string newValue, StringComparison comparison) + { + if (string.IsNullOrEmpty(str) || string.IsNullOrEmpty(oldValue)) + { + return null; + } + else + { + StringBuilder sb = new StringBuilder(); + + int previousIndex = 0; + int index = str.IndexOf(oldValue, comparison); + while (index != -1) + { + sb.Append(str.Substring(previousIndex, index - previousIndex)); + sb.Append(newValue); + index += oldValue.Length; + + previousIndex = index; + index = str.IndexOf(oldValue, index, comparison); + } + sb.Append(str.Substring(previousIndex)); + + return sb.ToString(); + } + } + + /// + /// Sets environment variables based on a list of ENV nodes. + /// + /// The list of ENV nodes. + public static void SetEnvironmentVariables(ProcessStartInfo processInfo, SMVEnvVar[] env, TextWriter logger) + { + if (processInfo == null || env == null) + return; + + Dictionary variablesSet = new Dictionary(); + variablesSet.Add("PATH", Environment.GetEnvironmentVariable("PATH")); + String varKey = null, varValue = null; + + Log.LogInfo("Setting environment variables..", logger); + + + foreach (SMVEnvVar envVar in env) + { + varKey = envVar.key; + varValue = envVar.value ?? String.Empty; + + // We need to do this because Environment.ExpandEnvironmentVariable does not expand environment variables specific to a process + + foreach (KeyValuePair pair in variablesSet) + { + varValue = Regex.Replace(varValue, String.Format(CultureInfo.InvariantCulture, "%{0}%", pair.Key), pair.Value, RegexOptions.IgnoreCase); + } + + varValue = ExpandSmvVariables(Environment.ExpandEnvironmentVariables(varValue)); + + if (variablesSet.ContainsKey(varKey)) + variablesSet.Remove(varKey); + + variablesSet.Add(varKey, varValue); + + if (processInfo.EnvironmentVariables.ContainsKey(varKey)) + processInfo.EnvironmentVariables.Remove(varKey); + + Log.LogInfo(String.Format(CultureInfo.InvariantCulture, "Setting environment variable: {0}={1}", varKey, varValue), logger); + processInfo.EnvironmentVariables.Add(varKey, varValue); + } + } + + /// + /// Prints the result of the Build Actions. + /// + /// List of the action names and if they succeeded. + public static void PrintResult(IDictionary result, double buildTime, double analysisTime) + { + if (result != null) + { + Log.LogMessage(Environment.NewLine); + Log.LogMessage("============================================================="); + Log.LogMessage("SMV Result:" + Environment.NewLine); + + foreach (DictionaryEntry actionResult in result) + { + Log.LogMessage(String.Format(CultureInfo.InvariantCulture, "{0} : {1}", actionResult.Key.ToString().PadRight(55), actionResult.Value)); + } + + Log.LogMessage("============================================================="); + Log.LogMessage(Environment.NewLine); + + if (buildTime > 0) + { + Log.LogMessage("Build time: " + buildTime); + } + + if (analysisTime > 0) + { + Log.LogMessage("Analysis time: " + analysisTime); + } + } + } + + /// + /// Get a list of distinct matches for a given regex and a haystack + /// + /// The regex pattern. + /// The string to search in. + /// Array of unique matches. + public static string[] GetUniqueRegexMatches(string pattern, string haystack) + { + MatchCollection matches = Regex.Matches(haystack, pattern); + HashSet result = new HashSet(matches.Cast().Select(m => m.Value)); + return result.ToArray(); + } + + /// + /// Returns the value corresponding to the key and null if not present + /// + /// The key to lookup. + /// Returns the value. + public static string GetSmvVar(string key) + { + if (smvVars.ContainsKey(key)) + { + return smvVars[key]; + } + + return null; + } + + /// + /// Sets the value corresponding to the key. + /// + /// The key to set. + /// The value to set corresponding to the key. + public static void SetSmvVar(string key, string value) + { + if (smvVars.ContainsKey(key)) + { + smvVars.Remove(key); + } + smvVars.Add(key, value); + } + + /// + /// Replaces the name of each variable embedded in the specified + /// string with the string equivalent of the value of the variable, then returns + /// the resulting string. + /// + /// A string containing the names of zero or more environment variables. Each + /// environment variable is quoted with the dollar sign character ($) and enclosed in [], for example, [$PATH]. + /// A string with each variable replaced by its value. + public static string ExpandVariables(string arg, IDictionary dict, TextWriter logger = null) + { + if (string.IsNullOrEmpty(arg)) + { + return string.Empty; + } + + string[] argVars = GetUniqueRegexMatches(@"\[\$(.*?)\]", arg); + foreach (String k in argVars) + { + // Extract the var name without the prefix ([$) and suffix (]) + string key = k.Substring(2, k.Length - 3); + string value = dict.ContainsKey(key) ? dict[key] : String.Empty; + + if (value == null) + { + Log.LogWarning(String.Format("Value of var ({0}) not set.", key), logger); + } + + arg = StringReplace(arg, k, value, StringComparison.InvariantCultureIgnoreCase); + } + + return arg; + } + + /// + /// Replaces the name of each SMV variable embedded in the specified + /// string with the string equivalent of the value of the variable, then returns + /// the resulting string. + /// + /// A string containing the names of zero or more environment variables. Each + /// environment variable is quoted with the dollar sign character ($) and enclosed in [], for example, [$PATH]. + /// A string with each SMV variable replaced by its value. + public static string ExpandSmvVariables(string arg) + { + return ExpandVariables(arg, smvVars); + } + + /// + /// Extract build path from output + /// + /// Output of the build. + public static string ExtractBuildPath(string workingDir, string output, TextWriter logger) + { + if (!String.IsNullOrEmpty(workingDir) && !String.IsNullOrEmpty(output)) + { + output = output.Replace("\r\n", Environment.NewLine); + + // Razzle + Match match = Regex.Match(output, @"cl.exe @(.*?)$", RegexOptions.Multiline); + string path = String.Empty; + + try + { + if (match.Success) + { + + string key = match.Groups[1].Value; + path = key.Trim(); + path = Path.GetDirectoryName(path); + + Log.LogInfo("Build path found - " + path, logger); + return path; + } + else + { + //MSBuild + string regex = String.Format(CultureInfo.InvariantCulture, "/F{0}\"(\\S+)\"|/F{0}(\\S+)", "[a|d|m|p|R|e|o|r|i]"); + match = Regex.Match(output, regex); + + if (match.Success) + { + string key = string.Empty; + if (!string.IsNullOrEmpty(match.Groups[1].Value)) + { + key = match.Groups[1].Value; + } + else if (!string.IsNullOrEmpty(match.Groups[2].Value)) + { + key = match.Groups[2].Value; + } + else + { + Log.LogFatalError("Cannot extract build path from the log file!"); + } + path = Path.Combine(workingDir, key.Trim()); + + //detect whether its a directory or file. If file get the parent directory + + //FileAttributes attr = File.GetAttributes(path); + + //if ((attr & FileAttributes.Directory) != FileAttributes.Directory) + //{ + // path = Path.GetDirectoryName(path); + //} + path = Path.GetDirectoryName(path); + Log.LogInfo("Build path found - " + path, logger); + return path; + } + else + { + Log.LogFatalError("Regex match failed, could not extract build path"); + return string.Empty; + } + } + } + catch (Exception e) + { + Console.WriteLine(e.ToString()); + Log.LogFatalError("Could not extract build path"); + return string.Empty; + } + } + else + { + Log.LogFatalError("No build output, could not extract build path"); + return string.Empty; + } + } + + } +} diff --git a/SmvLibrary/app.config b/SmvLibrary/app.config new file mode 100644 index 0000000..b26287c --- /dev/null +++ b/SmvLibrary/app.config @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/SmvLibrary/config.cs b/SmvLibrary/config.cs new file mode 100644 index 0000000..af6a4fd --- /dev/null +++ b/SmvLibrary/config.cs @@ -0,0 +1,182 @@ +using System.Collections.Generic; +using System.Xml.Serialization; +using System; + +// +// This source code was auto-generated by xsd, Version=4.0.30319.1 and then modified +// + +namespace SmvLibrary +{ + [XmlRootAttribute(Namespace = "")] + [Serializable] + public class SMVConfig + { + + [XmlArrayItemAttribute("SetVar", typeof(SetVar))] + public SetVar[] Variables { get; set; } + + [XmlArrayItemAttribute("Action", typeof(SMVAction))] + public SMVAction[] Build { get; set; } + + [XmlArrayItemAttribute("Action", typeof(SMVAction))] + public SMVAction[] Analysis { get; set; } + } + + [XmlRootAttribute("Action", Namespace = "")] + [Serializable] + public class SMVAction + { + public SMVAction() + { + this.breakOnError = true; + this.name = string.Empty; + this.executeOn = "local"; + this.variables = null; + } + public SMVAction(SMVCommand[] commands, string name) + { + this.breakOnError = true; + this.name = name; + this.Command = commands; + this.executeOn = "local"; + this.variables = null; + } + + public SMVAction(SMVAction orig, string analysisProperty) + { + this.breakOnError = orig.breakOnError; + this.name = orig.name; + this.Command = orig.Command; + this.Path = orig.Path; + this.Env = orig.Env; + this.CopyArtifact = orig.CopyArtifact; + this.executeOn = orig.executeOn; + this.nextAction = orig.nextAction; + this.variables = orig.variables; + this.analysisProperty = analysisProperty; + } + + public PathType Path { get; set; } + + [XmlElementAttribute("Env")] + public SMVEnvVar[] Env { get; set; } + + [XmlElementAttribute("CopyArtifact")] + public CopyArtifactType[] CopyArtifact { get; set; } + + [XmlElementAttribute("Command")] + public SMVCommand[] Command { get; set; } + + [XmlAttributeAttribute()] + public string name { get; set; } + + [XmlAttributeAttribute()] + public bool breakOnError { get; set; } + + [XmlAttributeAttribute()] + public string executeOn { get; set; } + + [XmlAttributeAttribute()] + public string nextAction { get; set; } + + [XmlIgnoreAttribute] + public SMVActionResult result { get; set; } + + [XmlIgnoreAttribute] + public string analysisProperty { get; set; } + + [XmlIgnoreAttribute] + public IDictionary variables { get; set; } + + + public string GetFullName() + { + return (string.IsNullOrEmpty(analysisProperty)) ? name : name + " - " + analysisProperty; + } + } + + [XmlRootAttribute("SetVar", Namespace = "")] + [Serializable] + public class SetVar + { + [XmlAttributeAttribute()] + public string key { get; set; } + + [XmlAttributeAttribute()] + public string value { get; set; } + } + + [XmlRootAttribute("Path", Namespace = "")] + [Serializable] + public class PathType + { + + [XmlAttributeAttribute()] + public string value { get; set; } + } + + [XmlRootAttribute("Env", Namespace = "")] + [Serializable] + public class SMVEnvVar + { + [XmlAttributeAttribute()] + public string key { get; set; } + + [XmlAttributeAttribute()] + public string value { get; set; } + } + + [XmlRootAttribute("CopyArtifact", Namespace = "")] + [Serializable] + public class CopyArtifactType + { + + [XmlAttributeAttribute()] + public string name { get; set; } + + [XmlAttributeAttribute()] + public string type { get; set; } + + [XmlAttributeAttribute()] + public entityAttrType entity { get; set; } + + [XmlAttributeAttribute()] + public string to { get; set; } + } + + public enum entityAttrType + { + Module, + CompilationUnit, + FunctionUnit, + } + + [XmlRootAttribute("Command", Namespace = "")] + [Serializable] + public class SMVCommand + { + [XmlAttributeAttribute()] + public string value { get; set; } + + [XmlAttributeAttribute()] + public string arguments { get; set; } + } + + [Serializable] + public class SMVActionResult + { + public SMVActionResult(string name, string output, bool isSuccessful, bool breakExecution) + { + this.name = name; + this.output = output; + this.isSuccessful = isSuccessful; + this.breakExecution = breakExecution; + } + + public string name { get; private set; } + public string output { get; set; } + public bool isSuccessful { get; private set; } + public bool breakExecution { get; private set; } + } +} \ No newline at end of file diff --git a/SmvLibrary/config.xsd b/SmvLibrary/config.xsd new file mode 100644 index 0000000..fc77194 --- /dev/null +++ b/SmvLibrary/config.xsd @@ -0,0 +1,109 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/SmvLibrary/log.cs b/SmvLibrary/log.cs new file mode 100644 index 0000000..2c4b2a8 --- /dev/null +++ b/SmvLibrary/log.cs @@ -0,0 +1,135 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace SmvLibrary +{ + sealed public class Log + { + private Log() + { + } + + static string logPath; + + public static void SetLogPath(string path) + { + if (!String.IsNullOrEmpty(path)) + { + logPath = path; + } + } + + /// + /// Logs a message to the Console or a logger + /// + /// The type of message. + /// The message to log. + static void WriteLog(string type, string message, TextWriter logger) + { + string result = String.Empty; + + if (String.IsNullOrEmpty(type)) + { + result = message; + } + else + { + result = String.Format(CultureInfo.InvariantCulture, "{0} : {1}", type, message); + } + + if(logger != null) + { + lock (logger) + { + logger.WriteLine(result); + logger.Flush(); + } + return; + } + + Console.WriteLine(result); + } + + /// + /// Logs a message to a file + /// + /// Filename. + /// The text to log. + public static void WriteToFile(string fileName, string text) + { + if (!String.IsNullOrEmpty(logPath)) + { + string path = Path.Combine(logPath, fileName + ".txt"); + using (StreamWriter sw = new StreamWriter(path)) + { + sw.WriteLine(text); + } + } + } + + /// + /// Logs a generic message + /// + /// The message to log. + public static void LogMessage(string message, TextWriter logger = null) + { + WriteLog("", message, logger); + } + + /// + /// Logs a INFO message + /// + /// The message to log. + public static void LogInfo(string message, TextWriter logger = null) + { + WriteLog("INFO", message, logger); + } + + /// + /// Logs a DEBUG message + /// + /// the message to log + /// + public static void LogDebug(string message, TextWriter logger = null) + { + if (Utility.debugMode) + { + WriteLog("DEBUG", message, logger); + } + } + + /// + /// Logs a ERROR message + /// + /// The message to log. + public static void LogError(string message, TextWriter logger = null) + { + WriteLog("ERROR", message, logger); + } + + /// + /// Logs a error message and terminates + /// + /// The message to log. + public static void LogFatalError(string message) + { + WriteLog("FATAL ERROR", message, null); + Environment.Exit(-1); + } + + /// + /// Logs a WARNING message + /// + /// The message to log. + public static void LogWarning(string message, TextWriter logger = null) + { + WriteLog("WARNING", message, logger); + } + + } +} diff --git a/SmvLibrary/packages.config b/SmvLibrary/packages.config new file mode 100644 index 0000000..47127a6 --- /dev/null +++ b/SmvLibrary/packages.config @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/SmvSkeleton/App.config b/SmvSkeleton/App.config new file mode 100644 index 0000000..8155ef9 --- /dev/null +++ b/SmvSkeleton/App.config @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/SmvSkeleton/GlobalSuppressions.cs b/SmvSkeleton/GlobalSuppressions.cs new file mode 100644 index 0000000..6a768d1 Binary files /dev/null and b/SmvSkeleton/GlobalSuppressions.cs differ diff --git a/SmvSkeleton/Program.cs b/SmvSkeleton/Program.cs new file mode 100644 index 0000000..4ddd51e --- /dev/null +++ b/SmvSkeleton/Program.cs @@ -0,0 +1,453 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using System.IO; +using System.Configuration; +using SmvAccessor; +using System.Xml; +using System.Xml.Schema; +using System.Diagnostics; +using System.Web; +using System.Collections.Specialized; +using System.Collections; +using System.Xml.Serialization; +using SmvSkeleton.Properties; +using SmvLibrary; +using System.Reflection; +using System.Globalization; + +namespace SmvSkeleton +{ + class Program + { + const int defaultLocalThreadCount = 5; + static SMVConfig smvConfig; + const string configXmlFileName = "Config.xml"; + const string configXsdFileName = "Config.xsd"; + const string cloudConfigXmlFileName = "CloudConfig.xml"; + const string cloudConfigXsdFileName = "CloudConfig.xsd"; + private static bool doAnalysis = false; + private static string buildLogFileNamePrefix = "smvbuild"; + + /// + /// Prints the usage string to the console. + /// + static void PrintUsage() + { + Console.WriteLine(Resources.UsageString); + } + + /// + /// Prints detailed help text to the console. + /// + static void PrintHelp() + { + PrintUsage(); + Log.LogInfo(Resources.HelpTextWithoutUsageString); + } + + /// + /// Processes command line arguments for Analysis. + /// + /// The list of command line arguments. + /// true on success, false on failure. + static bool ProcessArgs(string[] args) + { + bool help = false; + bool unsupportedArgument = false; + + for (int i = 0; i < args.Length; ) + { + args[i] = args[i].ToLowerInvariant(); + if (args[i].Equals("/help") || args[i].Equals("/?")) + { + help = true; + PrintHelp(); + break; + } + #region module + if (args[i].StartsWith("/module:", StringComparison.InvariantCulture)) + { + string path; + string dateTime = String.Empty; + Regex re = new Regex(@"^/module:([^@]+)@?(.+)?$"); + Match match = re.Match(args[i].Trim()); + if (!match.Success) + { + Log.LogError("Invalid argument: " + args[i]); + PrintUsage(); + return false; + } + + path = match.Groups[1].Value.Trim(); + if (match.Groups.Count == 3) + { + dateTime = match.Groups[2].Value; + } + + DateTime dt = default(DateTime); + if (!String.IsNullOrEmpty(dateTime)) + { + try + { + dt = DateTime.ParseExact(dateTime, "yyyyMMddHHmmss", System.Globalization.CultureInfo.InvariantCulture); + } + catch (FormatException) + { + Log.LogError(String.Format(CultureInfo.InvariantCulture, "Could not parse timestamp: {0} from module parameter: {1}", dateTime, args[i])); + return false; + } + } + + ISmvAccessor dbAccessor = Utility.GetSmvSQLAccessor(); + Utility.smvModule = dbAccessor.GetModuleByName(path, dt, true); + if (Utility.smvModule == null) + { + Log.LogError("Module with name: " + path + " does not exist in the data source."); + return false; + } + i++; + } + #endregion + else if (args[i].Equals("/getavailablemodules", StringComparison.InvariantCulture)) + { + Log.LogInfo("Printing list of all modules in the data source:"); + ISmvAccessor dbAccessor = Utility.GetSmvSQLAccessor(); + dbAccessor.PrintModuleStatistics(); + i++; + return false; + } + else if (args[i].StartsWith("/searchmodules:", StringComparison.InvariantCulture)) + { + string searchText = args[i].Replace("/searchmodules:", String.Empty); + Log.LogInfo(String.Format(CultureInfo.InvariantCulture, "INFO: Searching for modules with \"{0}\" in either the name field..", searchText)); + ISmvAccessor dbAccessor = Utility.GetSmvSQLAccessor(); + IEnumerable ms = dbAccessor.SearchModules(searchText); + dbAccessor.PrintModuleStatistics(ms); + i++; + return false; + } + + else if (args[i].StartsWith("/config:", StringComparison.InvariantCulture) || args[i].StartsWith("/log:", StringComparison.InvariantCulture)) + { + String[] tokens = args[i].Split(new char[] { ':' }, 2); + + if (tokens.Length == 2) + { + string value = tokens[1].Replace(@"""", String.Empty); + + if (tokens[0].Equals("/config")) + { + Utility.SetSmvVar("configFilePath", value); + } + else if (tokens[0].Equals("/log")) + { + if (!Directory.Exists(value)) + { + Log.LogFatalError("Log path does not exist."); + } + Log.SetLogPath(value); + } + } + i++; + } + else if (args[i].Equals("/analyze")) + { + doAnalysis = true; + i++; + } + else if (args[i].StartsWith("/plugin:", StringComparison.InvariantCulture)) + { + String[] tokens = args[i].Split(new char[] { ':' }, 2); + + if (File.Exists(tokens[1])) + { + Utility.pluginPath = tokens[1].Replace(Environment.GetEnvironmentVariable("smv"), "%smv%"); + Assembly assembly = Assembly.LoadFrom(tokens[1]); + string fullName = assembly.ExportedTypes.First().FullName; + Utility.plugin = (ISMVPlugin)assembly.CreateInstance(fullName); + + if (Utility.plugin == null) + { + Log.LogFatalError("Could not load plugin."); + } + Utility.plugin.Initialize(); + } + else + { + Log.LogFatalError("Plugin not found."); + } + i++; + } + else if (args[i].StartsWith("/projectfile:", StringComparison.InvariantCulture)) + { + String[] tokens = args[i].Split(new char[] { ':' }, 2); + Utility.SetSmvVar("projectFileArg", tokens[1]); + i++; + } + else if (args[i].Equals("/debug")) + { + Utility.debugMode = true; + i++; + } + else + { + unsupportedArgument = true; + i++; + } + } + + if (Utility.plugin != null) + { + Utility.plugin.ProcessPluginArgument(args); + } + else if (unsupportedArgument) + { + Log.LogFatalError("Unsupported arguments. Please provide a Plugin."); + } + + if (help) + { + if (Utility.plugin != null) + { + Utility.plugin.PrintPluginHelp(); + } + return false; + } + + return true; + } + + static void Main(string[] args) + { + Utility.SetSmvVar("workingDir", Directory.GetCurrentDirectory()); + Utility.SetSmvVar("logFilePath", null); + Utility.SetSmvVar("assemblyDir", Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location)); + Utility.SetSmvVar("configFilePath", Path.Combine(Utility.GetSmvVar("workingDir"), configXmlFileName)); + Utility.SetSmvVar("smvLogFileNamePrefix", buildLogFileNamePrefix); + + // Process commandline arguments. + // Note that ProcessArgs will return false if execution should not continue. + // This happens in cases such as /help, /getAvailableModules, /searchmodules + if (!ProcessArgs(args)) + { + return; + } + + // Get the SMV version name. + string smvVersionTxtPath = Path.Combine(Utility.GetSmvVar("assemblyDir"), "SmvVersionName.txt"); + if(!File.Exists(smvVersionTxtPath)) + { + Log.LogFatalError("SmvVersionName.txt must exist in the SMV bin directory."); + } + string[] lines = File.ReadAllLines(smvVersionTxtPath); + if(lines.Length < 1) + { + Log.LogFatalError("SmvVersionName.txt is empty."); + } + Utility.version = lines[0]; + + // Consume specified configuration file + smvConfig = GetSMVConfig(); + + if (smvConfig == null) + { + Log.LogFatalError("Could not load Config file"); + } + + // Set the variables defined in the Variables node in the config file + LoadGlobalVariables(smvConfig.Variables); + + // Project file value from command line overrides the Config value + if (!String.IsNullOrEmpty(Utility.GetSmvVar("projectFileArg"))) + { + Utility.SetSmvVar("projectFile", Utility.GetSmvVar("projectFileArg")); + } + + bool buildResult = false; + bool analysisResult = false; + double buildTime = 0, analysisTime = 0; + int localThreadCount = defaultLocalThreadCount; + + if(Utility.GetSmvVar("localThreads") != null) + { + localThreadCount = int.Parse(Utility.GetSmvVar("localThreads")); + } + Log.LogInfo(String.Format("Running local scheduler with {0} threads", localThreadCount)); + + // Load the cloud config from an XML file. + + SMVCloudConfig cloudConfig = GetSMVCloudConfig(); + + // Set up the schedulers. + using (Utility.scheduler = new MasterSMVActionScheduler()) + using (var localScheduler = new LocalSMVActionScheduler(localThreadCount)) + using (var cloudScheduler = new CloudSMVActionScheduler(cloudConfig)) + { + Utility.scheduler.AddScheduler("local", localScheduler); + Utility.scheduler.AddScheduler("cloud", cloudScheduler); + // Do build if specified in the configuration file + if (smvConfig.Build != null) + { + Stopwatch sw = Stopwatch.StartNew(); + + // Populate the actions dictionary that will be used by the schedulers. + Utility.PopulateActionsDictionary(smvConfig.Build); + + if (string.IsNullOrEmpty(Utility.GetSmvVar("projectFile"))) + { + Log.LogFatalError("Project file not set"); + } + + List buildActionsResult = Utility.ExecuteActions(Utility.GetRootActions(smvConfig.Build)); + buildResult = Utility.IsExecuteActionsSuccessful(buildActionsResult); + + if (Utility.plugin != null) + { + Utility.plugin.PostBuild(smvConfig.Build); + } + sw.Stop(); + buildTime = sw.Elapsed.TotalSeconds; + } + + // If build succeeded or it was not specified, do analysis (if specified and called) + if (smvConfig.Build == null || buildResult) + { + if (smvConfig.Analysis != null) + { + if (doAnalysis) + { + Stopwatch sw = Stopwatch.StartNew(); + Utility.PopulateActionsDictionary(smvConfig.Analysis); + + if (Utility.plugin != null) + { + Log.LogInfo("Using plugin " + Utility.plugin + " for analysis."); + analysisResult = Utility.plugin.DoPluginAnalysis(smvConfig.Analysis); + + Utility.plugin.PostAnalysis(smvConfig.Analysis); + } + else + { + List analysisActionsResult = Utility.ExecuteActions(Utility.GetRootActions(smvConfig.Analysis)); + analysisResult = Utility.IsExecuteActionsSuccessful(analysisActionsResult); + } + + if (!analysisResult) + { + Log.LogFatalError("Analysis failed."); + } + + sw.Stop(); + analysisTime = sw.Elapsed.TotalSeconds; + } + } + } + else + { + Log.LogFatalError("Build failed, skipping Analysis."); + } + } + + Utility.PrintResult(Utility.result, buildTime, analysisTime); + Log.LogInfo(String.Format("DONE. Total time taken {0} seconds", (buildTime + analysisTime))); + } + + /// + /// Load the cloud configuration from an XML file and store it in an SMVCloudConfig object. + /// + /// The SMVCloudConfig object containing the cloud configuration. + static SMVCloudConfig GetSMVCloudConfig() + { + string cloudConfigXmlPath = Path.Combine(Utility.GetSmvVar("assemblyDir"), cloudConfigXmlFileName); + string contents = Utility.ReadFile(cloudConfigXmlPath); + if (!String.IsNullOrEmpty(contents)) + { + bool isXMLValid = false; + string schemaPath = Path.Combine(Utility.GetSmvVar("assemblyDir"), cloudConfigXsdFileName); + + using (StringReader configContent = new StringReader(contents)) + { + isXMLValid = Utility.ValidateXmlFile(schemaPath, configContent); + } + + if (!isXMLValid) + { + Log.LogError("Could not load and validate XML file: " + Utility.GetSmvVar("configFilePath")); + return null; + } + + XmlSerializer serializer = new XmlSerializer(typeof(SMVCloudConfig)); + SMVCloudConfig config = null; + using (TextReader reader = new StringReader(contents)) + { + config = (SMVCloudConfig)serializer.Deserialize(reader); + } + + return config; + } + else + { + return null; + } + } + + /// + /// Load the configuration from the config file and store it in an SMVConfig object. + /// + /// The configuration as an SMVConfig object. + static SMVConfig GetSMVConfig() + { + string configFileContent = Utility.ReadFile(Utility.GetSmvVar("configFilePath")); + + if (!String.IsNullOrEmpty(configFileContent)) + { + bool isXMLValid = false; + string schemaPath = Path.Combine(Utility.GetSmvVar("assemblyDir"), configXsdFileName); + + using (StringReader configContent = new StringReader(configFileContent)) + { + isXMLValid = Utility.ValidateXmlFile(schemaPath, configContent); + } + + if (!isXMLValid) + { + Log.LogError("Could not load and validate XML file: " + Utility.GetSmvVar("configFilePath")); + return null; + } + + XmlSerializer serializer = new XmlSerializer(typeof(SMVConfig)); + using (TextReader reader = new StringReader(configFileContent)) + { + smvConfig = (SMVConfig)serializer.Deserialize(reader); + } + + return smvConfig; + } + else + { + return null; + } + } + + /// + /// Sets the global variables, defined in the Config file, in the SmvVar dictionary + /// + /// The variables defined in the config file. + static void LoadGlobalVariables(SetVar[] globalVars) + { + if (globalVars != null) + { + foreach (SetVar smvVar in globalVars) + { + string value = Environment.ExpandEnvironmentVariables(smvVar.value); + Utility.SetSmvVar(smvVar.key, Utility.ExpandSmvVariables(value)); + } + } + } + + } +} diff --git a/SmvSkeleton/Properties/AssemblyInfo.cs b/SmvSkeleton/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..e3bc305 --- /dev/null +++ b/SmvSkeleton/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("SmvSkeleton")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("SmvSkeleton")] +[assembly: AssemblyCopyright("Copyright © 2014")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("4d41175b-88cc-4b76-a920-bb5a577b4041")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/SmvSkeleton/Properties/Resources.Designer.cs b/SmvSkeleton/Properties/Resources.Designer.cs new file mode 100644 index 0000000..c18d805 --- /dev/null +++ b/SmvSkeleton/Properties/Resources.Designer.cs @@ -0,0 +1,81 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version:4.0.30319.34014 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace SmvSkeleton.Properties { + using System; + + + /// + /// A strongly-typed resource class, for looking up localized strings, etc. + /// + // This class was auto-generated by the StronglyTypedResourceBuilder + // class via a tool like ResGen or Visual Studio. + // To add or remove a member, edit your .ResX file then rerun ResGen + // with the /str option, or rebuild your VS project. + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "4.0.0.0")] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + internal class Resources { + + private static global::System.Resources.ResourceManager resourceMan; + + private static global::System.Globalization.CultureInfo resourceCulture; + + [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal Resources() { + } + + /// + /// Returns the cached ResourceManager instance used by this class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Resources.ResourceManager ResourceManager { + get { + if (object.ReferenceEquals(resourceMan, null)) { + global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("SmvSkeleton.Properties.Resources", typeof(Resources).Assembly); + resourceMan = temp; + } + return resourceMan; + } + } + + /// + /// Overrides the current thread's CurrentUICulture property for all + /// resource lookups using this strongly typed resource class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Globalization.CultureInfo Culture { + get { + return resourceCulture; + } + set { + resourceCulture = value; + } + } + + /// + /// Looks up a localized string similar to . + /// + internal static string HelpTextWithoutUsageString { + get { + return ResourceManager.GetString("HelpTextWithoutUsageString", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to smvskeleton.exe [/Module:<ModuleName>[@<LastModified>]] [/Rules|Check:*|Rule1,Rule2,Rule3...] [/Log:<Path to log file>] [/Config:<Path to config file>] [/GetAvailableModules] [/SearchModules:<SearchString>] [/Help|/?]. + /// + internal static string UsageString { + get { + return ResourceManager.GetString("UsageString", resourceCulture); + } + } + } +} diff --git a/SmvSkeleton/Properties/Resources.resx b/SmvSkeleton/Properties/Resources.resx new file mode 100644 index 0000000..e557d78 --- /dev/null +++ b/SmvSkeleton/Properties/Resources.resx @@ -0,0 +1,127 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + + ..\Resources\HelpTextWithoutUsageString.txt;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;Windows-1252 + + + ..\Resources\UsageString.txt;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;Windows-1252 + + \ No newline at end of file diff --git a/SmvSkeleton/Properties/Settings.Designer.cs b/SmvSkeleton/Properties/Settings.Designer.cs new file mode 100644 index 0000000..a5202fa --- /dev/null +++ b/SmvSkeleton/Properties/Settings.Designer.cs @@ -0,0 +1,26 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version:4.0.30319.42000 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace SmvSkeleton.Properties { + + + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.Editors.SettingsDesigner.SettingsSingleFileGenerator", "14.0.0.0")] + internal sealed partial class Settings : global::System.Configuration.ApplicationSettingsBase { + + private static Settings defaultInstance = ((Settings)(global::System.Configuration.ApplicationSettingsBase.Synchronized(new Settings()))); + + public static Settings Default { + get { + return defaultInstance; + } + } + } +} diff --git a/SmvSkeleton/Properties/Settings.settings b/SmvSkeleton/Properties/Settings.settings new file mode 100644 index 0000000..049245f --- /dev/null +++ b/SmvSkeleton/Properties/Settings.settings @@ -0,0 +1,6 @@ + + + + + + diff --git a/SmvSkeleton/Resources/HelpTextWithoutUsageString.txt b/SmvSkeleton/Resources/HelpTextWithoutUsageString.txt new file mode 100644 index 0000000..0a859dd --- /dev/null +++ b/SmvSkeleton/Resources/HelpTextWithoutUsageString.txt @@ -0,0 +1,40 @@ +/Debug + To write all the output to the console. + +/ProjectFile + Name of the project file. It can also be set in the config file. + +/Analyze + To run analysis. + +/Module:[@] + Specify a module for processing. is the name of the module. If there + are multiple modules with the same name, the version that was most recently + modified is used. Specify which version of the module to use by passing the + value in yyyyMMddHHmmss format. Use /SearchModules to find + all versions of a particular module. If the analysis requires multiple + modules, pass this parameter once for each module. + Eg. For an analysis that requires three modules: + smvskeleton.exe /Module:module1 /Module:module2 /Module:mod3@20140604221756 + +/Rules|Check:*|Rule1,Rule2,Rule3... + Specify the rules (comma separated or * for all the rules), you want to verify. + +/Log: + Specify the path for the log file, if you want to persist the log. + +/Plugin: + Specify the path to the DLL, if you want to use a plugin + +/Config: + Specify the path for the Config file. + +/GetAvailableModules + Print the entire list of modules in the system to the console. + +/SearchModules: + Print the list of all the modules in the data source whose names or paths + contain . + +/Help | /? + Print this help text. \ No newline at end of file diff --git a/SmvSkeleton/Resources/UsageString.txt b/SmvSkeleton/Resources/UsageString.txt new file mode 100644 index 0000000..e568e4e --- /dev/null +++ b/SmvSkeleton/Resources/UsageString.txt @@ -0,0 +1 @@ +smvskeleton.exe [/ProjectFile:] [/Analyze] [/Debug] [/Module:[@]] [/Rules|Check:*|Rule1,Rule2,Rule3...] [/Plugin:] [/Log:] [/Config:] [/GetAvailableModules] [/SearchModules:] [/Help|/?] \ No newline at end of file diff --git a/SmvSkeleton/Resources/sdv-pre-results.h b/SmvSkeleton/Resources/sdv-pre-results.h new file mode 100644 index 0000000..5acb62a --- /dev/null +++ b/SmvSkeleton/Resources/sdv-pre-results.h @@ -0,0 +1,8 @@ +#define SDV_NOTDONE 0 +#define SDV_PASSED 1 +#define SDV_FAILED 2 +#define SDV_NA 3 +#define SDV_TIMEOUT 4 +#define SDV_SPACEOUT 5 +#define SDV_GIVEUP 6 +#define SDV_TOOLERROR 7 diff --git a/SmvSkeleton/Samples/CloudBuild-Build.xml b/SmvSkeleton/Samples/CloudBuild-Build.xml new file mode 100644 index 0000000..7c6975e --- /dev/null +++ b/SmvSkeleton/Samples/CloudBuild-Build.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/SmvSkeleton/Samples/CoreXT-Build.xml b/SmvSkeleton/Samples/CoreXT-Build.xml new file mode 100644 index 0000000..14f9d3d --- /dev/null +++ b/SmvSkeleton/Samples/CoreXT-Build.xml @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/SmvSkeleton/Samples/CoreXT.MSBuild-Build.xml b/SmvSkeleton/Samples/CoreXT.MSBuild-Build.xml new file mode 100644 index 0000000..f2aef06 --- /dev/null +++ b/SmvSkeleton/Samples/CoreXT.MSBuild-Build.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/SmvSkeleton/Samples/MSBuild-Build.xml b/SmvSkeleton/Samples/MSBuild-Build.xml new file mode 100644 index 0000000..5ee056e --- /dev/null +++ b/SmvSkeleton/Samples/MSBuild-Build.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/SmvSkeleton/Samples/NMake-OpenSSL.xml b/SmvSkeleton/Samples/NMake-OpenSSL.xml new file mode 100644 index 0000000..b27081d --- /dev/null +++ b/SmvSkeleton/Samples/NMake-OpenSSL.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/SmvSkeleton/Samples/NoBuild-SlamScan.xml b/SmvSkeleton/Samples/NoBuild-SlamScan.xml new file mode 100644 index 0000000..c25f068 --- /dev/null +++ b/SmvSkeleton/Samples/NoBuild-SlamScan.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/SmvSkeleton/Samples/NoBuild-UnstableCodeAnalysis.xml b/SmvSkeleton/Samples/NoBuild-UnstableCodeAnalysis.xml new file mode 100644 index 0000000..b8f6bd7 --- /dev/null +++ b/SmvSkeleton/Samples/NoBuild-UnstableCodeAnalysis.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/SmvSkeleton/Samples/Razzle-Build.xml b/SmvSkeleton/Samples/Razzle-Build.xml new file mode 100644 index 0000000..a0fb75b --- /dev/null +++ b/SmvSkeleton/Samples/Razzle-Build.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/SmvSkeleton/Samples/SymDiff.xml b/SmvSkeleton/Samples/SymDiff.xml new file mode 100644 index 0000000..b5734fa --- /dev/null +++ b/SmvSkeleton/Samples/SymDiff.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + diff --git a/SmvSkeleton/SmvSkeleton.csproj b/SmvSkeleton/SmvSkeleton.csproj new file mode 100644 index 0000000..8639ce3 --- /dev/null +++ b/SmvSkeleton/SmvSkeleton.csproj @@ -0,0 +1,144 @@ + + + + + Debug + AnyCPU + {E016F57D-68E9-48F9-ABCE-B0C3D7714C84} + Exe + Properties + SmvSkeleton + SmvSkeleton + v4.5 + 512 + false + publish\ + true + Disk + false + Foreground + 7 + Days + false + false + true + 0 + 1.0.0.%2a + false + true + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + AllRules.ruleset + true + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + AllRules.ruleset + true + + + + + + + + + + + + + + + + + + True + True + Resources.resx + + + True + True + Settings.settings + + + + + + + + {5a923bec-71fc-48e3-b076-2a2aa81ebd03} + SmvAccessor + + + {e03dc0df-84cc-420c-91b1-2a937e88ff31} + SmvLibrary + + + + + False + Microsoft .NET Framework 4.5 %28x86 and x64%29 + true + + + False + .NET Framework 3.5 SP1 Client Profile + false + + + False + .NET Framework 3.5 SP1 + false + + + + + SettingsSingleFileGenerator + Settings.Designer.cs + + + + + + + + Designer + + + + + + + + ResXFileCodeGenerator + Resources.Designer.cs + + + + + copy $(ProjectDir)\Resources\sdv-pre-results.h $(TargetDir) + + + + \ No newline at end of file diff --git a/SmvSkeleton/SmvSkeleton.csproj.user b/SmvSkeleton/SmvSkeleton.csproj.user new file mode 100644 index 0000000..bc69bee --- /dev/null +++ b/SmvSkeleton/SmvSkeleton.csproj.user @@ -0,0 +1,18 @@ + + + + /plugin:"d:\slam1\src\t-apoup\smv\smvsdv\bin\debug\sdvplugin.dll" /config:"d:\slam1\smvcloud\analysisplugins\sdv\configurations\msbuild-build-verify.xml" /analyze + D:\slam1\WDK\src_5043\fail_drivers\wdm\fail_driver1\ + Project + + + publish\ + + + + + + en-US + false + + \ No newline at end of file diff --git a/dpks/SmvCloud.dpk b/dpks/SmvCloud.dpk new file mode 100644 index 0000000..5704823 Binary files /dev/null and b/dpks/SmvCloud.dpk differ diff --git a/msbuild.proj b/msbuild.proj new file mode 100644 index 0000000..6006058 --- /dev/null +++ b/msbuild.proj @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/smv.sln b/smv.sln new file mode 100644 index 0000000..b56db6b --- /dev/null +++ b/smv.sln @@ -0,0 +1,104 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 14 +VisualStudioVersion = 14.0.25123.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{00D1A9C2-B5F0-4AF3-8072-F6C62B433612}") = "SmvDb", "SmvDb\SmvDb.sqlproj", "{824F4A4A-89DA-47BB-95E0-8B3F3375505F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Smv2Sql", "Smv2Sql\Smv2Sql.csproj", "{B70DBC28-ED30-4306-8433-2E7B9C383CE7}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SmvInterceptorWrapper", "SmvInterceptorWrapper\SmvInterceptorWrapper.csproj", "{8F9EA492-E86B-4F1C-ADC5-BF407466F448}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SmvAccessor", "SmvAccessor\SmvAccessor.csproj", "{5A923BEC-71FC-48E3-B076-2A2AA81EBD03}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SmvInterceptor", "SmvInterceptor\SmvInterceptor.csproj", "{2422E37E-3DC7-4B6C-8246-317AB656D788}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SmvAccessorDemo", "SmvAccessorDemo\SmvAccessorDemo.csproj", "{BDC040C9-7918-419B-A6CA-046F1AAA3813}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SmvSkeleton", "SmvSkeleton\SmvSkeleton.csproj", "{E016F57D-68E9-48F9-ABCE-B0C3D7714C84}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SmvLibrary", "SMVLibrary\SmvLibrary.csproj", "{E03DC0DF-84CC-420C-91B1-2A937E88FF31}" + ProjectSection(ProjectDependencies) = postProject + {6F655D35-3867-4351-8929-23B4578E5190} = {6F655D35-3867-4351-8929-23B4578E5190} + {5A923BEC-71FC-48E3-B076-2A2AA81EBD03} = {5A923BEC-71FC-48E3-B076-2A2AA81EBD03} + EndProjectSection +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SmvSdv", "SmvSdv\SmvSdv.csproj", "{332F53A7-E9F4-4A7E-BDAC-E490F3A6935A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SmvTest", "SmvTest\SmvTest.csproj", "{55EE08BA-C5E5-40C5-A667-B91D88DABD0D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SmvLineCounter", "SmvLineCounter\SmvLineCounter.csproj", "{417057CF-9FBD-4732-BE9D-42875EBA61C9}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SMVActionsTable", "SMVActionsTable\SMVActionsTable.csproj", "{6F655D35-3867-4351-8929-23B4578E5190}" +EndProject +Project("{CC5FD16D-436D-48AD-A40C-5A424C6E3E79}") = "SmvCloud", "SmvCloud\SmvCloud.ccproj", "{3BBC4FAD-85A1-4451-8975-BC00EAF81E63}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SmvCloudWorker", "SmvCloudWorker\SmvCloudWorker.csproj", "{00D32586-D675-4EF4-B4F3-06DC45BBF079}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {824F4A4A-89DA-47BB-95E0-8B3F3375505F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {824F4A4A-89DA-47BB-95E0-8B3F3375505F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {824F4A4A-89DA-47BB-95E0-8B3F3375505F}.Debug|Any CPU.Deploy.0 = Debug|Any CPU + {824F4A4A-89DA-47BB-95E0-8B3F3375505F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {824F4A4A-89DA-47BB-95E0-8B3F3375505F}.Release|Any CPU.Build.0 = Release|Any CPU + {824F4A4A-89DA-47BB-95E0-8B3F3375505F}.Release|Any CPU.Deploy.0 = Release|Any CPU + {B70DBC28-ED30-4306-8433-2E7B9C383CE7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B70DBC28-ED30-4306-8433-2E7B9C383CE7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B70DBC28-ED30-4306-8433-2E7B9C383CE7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B70DBC28-ED30-4306-8433-2E7B9C383CE7}.Release|Any CPU.Build.0 = Release|Any CPU + {8F9EA492-E86B-4F1C-ADC5-BF407466F448}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8F9EA492-E86B-4F1C-ADC5-BF407466F448}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8F9EA492-E86B-4F1C-ADC5-BF407466F448}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8F9EA492-E86B-4F1C-ADC5-BF407466F448}.Release|Any CPU.Build.0 = Release|Any CPU + {5A923BEC-71FC-48E3-B076-2A2AA81EBD03}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5A923BEC-71FC-48E3-B076-2A2AA81EBD03}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5A923BEC-71FC-48E3-B076-2A2AA81EBD03}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5A923BEC-71FC-48E3-B076-2A2AA81EBD03}.Release|Any CPU.Build.0 = Release|Any CPU + {2422E37E-3DC7-4B6C-8246-317AB656D788}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2422E37E-3DC7-4B6C-8246-317AB656D788}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2422E37E-3DC7-4B6C-8246-317AB656D788}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2422E37E-3DC7-4B6C-8246-317AB656D788}.Release|Any CPU.Build.0 = Release|Any CPU + {BDC040C9-7918-419B-A6CA-046F1AAA3813}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BDC040C9-7918-419B-A6CA-046F1AAA3813}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E016F57D-68E9-48F9-ABCE-B0C3D7714C84}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E016F57D-68E9-48F9-ABCE-B0C3D7714C84}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E016F57D-68E9-48F9-ABCE-B0C3D7714C84}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E016F57D-68E9-48F9-ABCE-B0C3D7714C84}.Release|Any CPU.Build.0 = Release|Any CPU + {E03DC0DF-84CC-420C-91B1-2A937E88FF31}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E03DC0DF-84CC-420C-91B1-2A937E88FF31}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E03DC0DF-84CC-420C-91B1-2A937E88FF31}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E03DC0DF-84CC-420C-91B1-2A937E88FF31}.Release|Any CPU.Build.0 = Release|Any CPU + {332F53A7-E9F4-4A7E-BDAC-E490F3A6935A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {332F53A7-E9F4-4A7E-BDAC-E490F3A6935A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {332F53A7-E9F4-4A7E-BDAC-E490F3A6935A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {332F53A7-E9F4-4A7E-BDAC-E490F3A6935A}.Release|Any CPU.Build.0 = Release|Any CPU + {55EE08BA-C5E5-40C5-A667-B91D88DABD0D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {55EE08BA-C5E5-40C5-A667-B91D88DABD0D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {55EE08BA-C5E5-40C5-A667-B91D88DABD0D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {55EE08BA-C5E5-40C5-A667-B91D88DABD0D}.Release|Any CPU.Build.0 = Release|Any CPU + {417057CF-9FBD-4732-BE9D-42875EBA61C9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {417057CF-9FBD-4732-BE9D-42875EBA61C9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {417057CF-9FBD-4732-BE9D-42875EBA61C9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {417057CF-9FBD-4732-BE9D-42875EBA61C9}.Release|Any CPU.Build.0 = Release|Any CPU + {6F655D35-3867-4351-8929-23B4578E5190}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6F655D35-3867-4351-8929-23B4578E5190}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6F655D35-3867-4351-8929-23B4578E5190}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6F655D35-3867-4351-8929-23B4578E5190}.Release|Any CPU.Build.0 = Release|Any CPU + {3BBC4FAD-85A1-4451-8975-BC00EAF81E63}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3BBC4FAD-85A1-4451-8975-BC00EAF81E63}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3BBC4FAD-85A1-4451-8975-BC00EAF81E63}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3BBC4FAD-85A1-4451-8975-BC00EAF81E63}.Release|Any CPU.Build.0 = Release|Any CPU + {00D32586-D675-4EF4-B4F3-06DC45BBF079}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {00D32586-D675-4EF4-B4F3-06DC45BBF079}.Debug|Any CPU.Build.0 = Debug|Any CPU + {00D32586-D675-4EF4-B4F3-06DC45BBF079}.Release|Any CPU.ActiveCfg = Release|Any CPU + {00D32586-D675-4EF4-B4F3-06DC45BBF079}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal