From 942b2d55938d12f2d591c831ae476afa64d38c40 Mon Sep 17 00:00:00 2001 From: Avani Patel <32693971+avanikp@users.noreply.github.com> Date: Wed, 16 Jan 2019 10:59:21 -0800 Subject: [PATCH] Replay file APIs (#333) * Replay file APIs * update settings * PR comments * update owners * typo * Update DeviceModelScriptsTest.cs --- Services.Test/DeviceModelScriptsTest.cs | 30 +-- Services.Test/ReplayFileServiceTest.cs | 133 +++++++++++++ Services/DeviceModelScripts.cs | 24 +-- .../{DeviceModelScript.cs => DataFile.cs} | 13 +- Services/Models/Simulation.cs | 6 +- Services/ReplayFileService.cs | 174 ++++++++++++++++++ Services/Runtime/ServicesConfig.cs | 3 + Services/Services.csproj | 1 + Services/Simulation/JavascriptInterpreter.cs | 2 +- WebService.Test/WebService.Test.csproj | 1 + .../DeviceModelScriptsControllerTest.cs | 20 +- WebService/Runtime/Config.cs | 2 + WebService/Startup.cs | 1 + WebService/WebService.csproj | 1 + WebService/appsettings.ini | 16 ++ .../DeviceModelScriptsController.cs | 6 +- .../v1/Controllers/ReplayFileController.cs | 105 +++++++++++ .../DeviceModelScriptApiModel.cs | 8 +- .../v1/Models/DeviceModelScriptListModel.cs | 2 +- WebService/v1/Models/ReplayFileApiModel.cs | 87 +++++++++ docs/CODEOWNERS | 2 +- 21 files changed, 583 insertions(+), 54 deletions(-) create mode 100644 Services.Test/ReplayFileServiceTest.cs rename Services/Models/{DeviceModelScript.cs => DataFile.cs} (73%) create mode 100644 Services/ReplayFileService.cs create mode 100644 WebService/v1/Controllers/ReplayFileController.cs create mode 100644 WebService/v1/Models/ReplayFileApiModel.cs diff --git a/Services.Test/DeviceModelScriptsTest.cs b/Services.Test/DeviceModelScriptsTest.cs index 8a17af72..f4dce873 100644 --- a/Services.Test/DeviceModelScriptsTest.cs +++ b/Services.Test/DeviceModelScriptsTest.cs @@ -49,14 +49,14 @@ namespace Services.Test // Arrange var id = Guid.NewGuid().ToString(); var eTag = Guid.NewGuid().ToString(); - var deviceModelScript = new DeviceModelScript { Id = id, ETag = eTag }; + var deviceModelScript = new DataFile { Id = id, ETag = eTag }; this.storage .Setup(x => x.UpdateAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(this.BuildValueApiModel(deviceModelScript)); // Act - DeviceModelScript result = this.target.InsertAsync(deviceModelScript).Result; + DataFile result = this.target.InsertAsync(deviceModelScript).Result; // Assert Assert.NotNull(result); @@ -73,10 +73,10 @@ namespace Services.Test // Arrange var id = Guid.NewGuid().ToString(); - var deviceModelScript = new DeviceModelScript { Id = id, ETag = "oldEtag" }; + var deviceModelScript = new DataFile { Id = id, ETag = "oldEtag" }; this.TheScriptExists(id, deviceModelScript); - var updatedSimulationScript = new DeviceModelScript { Id = id, ETag = "newETag" }; + var updatedSimulationScript = new DataFile { Id = id, ETag = "newETag" }; this.storage .Setup(x => x.UpdateAsync( STORAGE_COLLECTION, @@ -94,7 +94,7 @@ namespace Services.Test this.storage.Verify(x => x.UpdateAsync( STORAGE_COLLECTION, id, - It.Is(json => JsonConvert.DeserializeObject(json).Id == id && !json.Contains("ETag")), + It.Is(json => JsonConvert.DeserializeObject(json).Id == id && !json.Contains("ETag")), "oldEtag"), Times.Once()); Assert.Equal(updatedSimulationScript.Id, deviceModelScript.Id); @@ -107,7 +107,7 @@ namespace Services.Test { // Arrange var id = Guid.NewGuid().ToString(); - var deviceModelScript = new DeviceModelScript { Id = id, ETag = "Etag" }; + var deviceModelScript = new DataFile { Id = id, ETag = "Etag" }; this.TheScriptDoesntExist(id); this.storage .Setup(x => x.UpdateAsync( @@ -133,11 +133,11 @@ namespace Services.Test { // Arrange var id = Guid.NewGuid().ToString(); - var deviceModelScriptInStorage = new DeviceModelScript { Id = id, ETag = "ETag" }; + var deviceModelScriptInStorage = new DataFile { Id = id, ETag = "ETag" }; this.TheScriptExists(id, deviceModelScriptInStorage); // Act & Assert - var deviceModelScript = new DeviceModelScript { Id = id, ETag = "not-matching-Etag" }; + var deviceModelScript = new DataFile { Id = id, ETag = "not-matching-Etag" }; Assert.ThrowsAsync( async () => await this.target.UpsertAsync(deviceModelScript)) .Wait(Constants.TEST_TIMEOUT); @@ -147,7 +147,7 @@ namespace Services.Test public void ItThrowsExceptionWhenInsertDeviceModelScriptFailed() { // Arrange - var deviceModelScript = new DeviceModelScript { Id = "id", ETag = "Etag" }; + var deviceModelScript = new DataFile { Id = "id", ETag = "Etag" }; this.storage .Setup(x => x.UpdateAsync( It.IsAny(), @@ -166,7 +166,7 @@ namespace Services.Test public void ItFailsToUpsertWhenUnableToFetchScriptFromStorage() { // Arrange - var deviceModelScript = new DeviceModelScript { Id = "id", ETag = "Etag" }; + var deviceModelScript = new DataFile { Id = "id", ETag = "Etag" }; this.storage .Setup(x => x.GetAsync(It.IsAny(), It.IsAny())) .ThrowsAsync(new SomeException()); @@ -182,7 +182,7 @@ namespace Services.Test { // Arrange var id = Guid.NewGuid().ToString(); - var deviceModelScript = new DeviceModelScript { Id = id, ETag = "Etag" }; + var deviceModelScript = new DataFile { Id = id, ETag = "Etag" }; this.TheScriptExists(id, deviceModelScript); this.storage @@ -203,7 +203,7 @@ namespace Services.Test public void ItThrowsExternalDependencyExceptionWhenFailedFetchingDeviceModelScriptInStorage() { // Arrange - var deviceModelScript = new DeviceModelScript { Id = "id", ETag = "Etag" }; + var deviceModelScript = new DataFile { Id = "id", ETag = "Etag" }; // Act var ex = Record.Exception(() => this.target.UpsertAsync(deviceModelScript).Result); @@ -268,7 +268,7 @@ namespace Services.Test var list = new ValueListApiModel(); var value = new ValueApiModel { - Key = "key", + Key = "key1", Data = "{ 'invalid': json", ETag = "etag" }; @@ -286,14 +286,14 @@ namespace Services.Test .Throws(); } - private void TheScriptExists(string id, DeviceModelScript deviceModelScript) + private void TheScriptExists(string id, DataFile deviceModelScript) { this.storage .Setup(x => x.GetAsync(STORAGE_COLLECTION, id)) .ReturnsAsync(this.BuildValueApiModel(deviceModelScript)); } - private ValueApiModel BuildValueApiModel(DeviceModelScript deviceModelScript) + private ValueApiModel BuildValueApiModel(DataFile deviceModelScript) { return new ValueApiModel { diff --git a/Services.Test/ReplayFileServiceTest.cs b/Services.Test/ReplayFileServiceTest.cs new file mode 100644 index 00000000..44df5ca0 --- /dev/null +++ b/Services.Test/ReplayFileServiceTest.cs @@ -0,0 +1,133 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services; +using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Diagnostics; +using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Exceptions; +using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Models; +using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Runtime; +using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Storage; +using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Storage.CosmosDbSql; +using Moq; +using Newtonsoft.Json; +using Services.Test.helpers; +using System; +using Xunit; + +namespace Services.Test +{ + public class ReplayFileServiceTest + { + private const string REPLAY_FILES = "replayFiles"; + + private readonly Mock log; + private readonly Mock config; + private readonly Mock enginesFactory; + private readonly Mock replayFilesStorage; + private readonly Mock logger; + private readonly ReplayFileService target; + + public ReplayFileServiceTest() + { + this.log = new Mock(); + this.config = new Mock(); + this.enginesFactory = new Mock(); + + this.replayFilesStorage = new Mock(); + this.replayFilesStorage.Setup(x => x.BuildRecord(It.IsAny(), It.IsAny())) + .Returns((string id, string json) => new DataRecord { Id = id, Data = json }); + this.replayFilesStorage.Setup(x => x.BuildRecord(It.IsAny())) + .Returns((string id) => new DataRecord { Id = id }); + + this.config.SetupGet(x => x.ReplayFilesStorage) + .Returns(new Config { CosmosDbSqlCollection = REPLAY_FILES }); + + this.enginesFactory + .Setup(x => x.Build(It.Is(c => c.CosmosDbSqlCollection == REPLAY_FILES))) + .Returns(this.replayFilesStorage.Object); + + this.target = new ReplayFileService( + this.config.Object, + this.enginesFactory.Object, + this.replayFilesStorage.Object, + this.log.Object); + } + + [Fact, Trait(Constants.TYPE, Constants.UNIT_TEST)] + public void ItCreatesReplayFileInStorage() + { + // Arrange + var id = Guid.NewGuid().ToString(); + var replayFile = new DataFile { Id = id, Content = "1, 2, 3", ETag = "tag" }; + + this.replayFilesStorage + .Setup(x => x.CreateAsync(It.IsAny())) + .ReturnsAsync(new DataRecord { Id = id, Data = JsonConvert.SerializeObject(replayFile) }); + + // Act + DataFile result = this.target.InsertAsync(replayFile).Result; + + // Assert + Assert.NotNull(result); + Assert.Equal(replayFile.Id, result.Id); + Assert.Equal(replayFile.Content, result.Content); + + this.replayFilesStorage.Verify( + x => x.CreateAsync(It.Is(n => n.GetId() == id)), Times.Once()); + } + + [Fact, Trait(Constants.TYPE, Constants.UNIT_TEST)] + public void ItThrowsExceptionWhenCreateReplayFileFails() + { + // Arrange + DataFile replayFile = new DataFile(); + replayFile.Content = "1, 2, 3, 4, 5"; + + this.replayFilesStorage + .Setup(x => x.CreateAsync(It.IsAny())) + .ThrowsAsync(new SomeException()); + + // Act & Assert + Assert.ThrowsAsync( + async () => await this.target.InsertAsync(replayFile)) + .Wait(Constants.TEST_TIMEOUT); + } + + + [Fact, Trait(Constants.TYPE, Constants.UNIT_TEST)] + public void ItThrowsExceptionWhenDeleteReplayFileFailes() + { + // Arrange + this.replayFilesStorage + .Setup(x => x.DeleteAsync(It.IsAny())) + .ThrowsAsync(new SomeException()); + + // Act & Assert + Assert.ThrowsAsync( + async () => await this.target.DeleteAsync("Id")) + .Wait(Constants.TEST_TIMEOUT); + } + + [Fact, Trait(Constants.TYPE, Constants.UNIT_TEST)] + public void ItFailsToGetReplayFileWhenGetAsyncFails() + { + // Arrange + this.replayFilesStorage + .Setup(x => x.GetAsync(It.IsAny())) + .ThrowsAsync(new SomeException()); + + // Act & Assert + Assert.ThrowsAsync( + async () => await this.target.GetAsync("Id")) + .Wait(Constants.TEST_TIMEOUT); + } + + [Fact, Trait(Constants.TYPE, Constants.UNIT_TEST)] + public void ItThrowsExceptionForInvalidId() + { + // Act & Assert + Assert.ThrowsAsync( + async () => await this.target.GetAsync(string.Empty)) + .Wait(Constants.TEST_TIMEOUT); + } + } +} diff --git a/Services/DeviceModelScripts.cs b/Services/DeviceModelScripts.cs index fcdac5dd..d3ffcae6 100644 --- a/Services/DeviceModelScripts.cs +++ b/Services/DeviceModelScripts.cs @@ -17,22 +17,22 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services /// /// Get list of device model scripts. /// - Task> GetListAsync(); + Task> GetListAsync(); /// /// Get a device model script. /// - Task GetAsync(string id); + Task GetAsync(string id); /// /// Create a device model script. /// - Task InsertAsync(DeviceModelScript deviceModelScript); + Task InsertAsync(DataFile deviceModelScript); /// /// Create or replace a device model script. /// - Task UpsertAsync(DeviceModelScript deviceModelScript); + Task UpsertAsync(DataFile deviceModelScript); /// /// Delete a device model script. @@ -74,7 +74,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services /// /// Get a device model script. /// - public async Task GetAsync(string id) + public async Task GetAsync(string id) { if (string.IsNullOrEmpty(id)) { @@ -99,7 +99,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services try { - var deviceModelScript = JsonConvert.DeserializeObject(item.Data); + var deviceModelScript = JsonConvert.DeserializeObject(item.Data); deviceModelScript.ETag = item.ETag; return deviceModelScript; } @@ -113,7 +113,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services /// /// Get list of device model scripts. /// - public async Task> GetListAsync() + public async Task> GetListAsync() { ValueListApiModel data; @@ -129,13 +129,13 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services try { - var results = new List(); + var results = new List(); foreach (var item in data.Items) { - var deviceModelScript = JsonConvert.DeserializeObject(item.Data); + var deviceModelScript = JsonConvert.DeserializeObject(item.Data); deviceModelScript.ETag = item.ETag; deviceModelScript.Type = ScriptInterpreter.JAVASCRIPT_SCRIPT; - deviceModelScript.Path = DeviceModelScript.DeviceModelScriptPath.Storage; + deviceModelScript.Path = DataFile.FilePath.Storage; results.Add(deviceModelScript); } @@ -151,7 +151,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services /// /// Create a device model script. /// - public async Task InsertAsync(DeviceModelScript deviceModelScript) + public async Task InsertAsync(DataFile deviceModelScript) { deviceModelScript.Created = DateTimeOffset.UtcNow; deviceModelScript.Modified = deviceModelScript.Created; @@ -188,7 +188,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services /// /// Create or replace a device model script. /// - public async Task UpsertAsync(DeviceModelScript deviceModelScript) + public async Task UpsertAsync(DataFile deviceModelScript) { var id = deviceModelScript.Id; var eTag = deviceModelScript.ETag; diff --git a/Services/Models/DeviceModelScript.cs b/Services/Models/DataFile.cs similarity index 73% rename from Services/Models/DeviceModelScript.cs rename to Services/Models/DataFile.cs index b2baa7b0..a6ab00a6 100644 --- a/Services/Models/DeviceModelScript.cs +++ b/Services/Models/DataFile.cs @@ -2,13 +2,12 @@ using System; using System.Runtime.Serialization; -using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Simulation; using Newtonsoft.Json; using Newtonsoft.Json.Converters; namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Models { - public class DeviceModelScript + public class DataFile { [JsonIgnore] public string ETag { get; set; } @@ -16,22 +15,22 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Models public string Name { get; set; } public string Type { get; set; } public string Content { get; set; } - public DeviceModelScriptPath Path { get; set; } + public FilePath Path { get; set; } public DateTimeOffset Created { get; set; } public DateTimeOffset Modified { get; set; } - public DeviceModelScript() + public DataFile() { this.ETag = string.Empty; this.Id = string.Empty; - this.Type = ScriptInterpreter.JAVASCRIPT_SCRIPT; + this.Type = string.Empty; this.Content = string.Empty; - this.Path = DeviceModelScriptPath.Storage; + this.Path = FilePath.Storage; this.Name = string.Empty; } [JsonConverter(typeof(StringEnumConverter))] - public enum DeviceModelScriptPath + public enum FilePath { [EnumMember(Value = "Undefined")] Undefined = 0, diff --git a/Services/Models/Simulation.cs b/Services/Models/Simulation.cs index 01c8e239..54a8d219 100644 --- a/Services/Models/Simulation.cs +++ b/Services/Models/Simulation.cs @@ -133,9 +133,13 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Models public DateTimeOffset Modified { get; set; } // ActualStartTime is the time when Simulation was started - [JsonProperty(Order = 140)] + [JsonProperty(Order = 150)] public DateTimeOffset? ActualStartTime { get; set; } + // ReplayFileId is the id of the replay file in storage + [JsonProperty(Order = 160)] + public string ReplayFileId { get; set; } + public Simulation() { // When unspecified, a simulation is enabled diff --git a/Services/ReplayFileService.cs b/Services/ReplayFileService.cs new file mode 100644 index 00000000..9aa84295 --- /dev/null +++ b/Services/ReplayFileService.cs @@ -0,0 +1,174 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.IO; +using System.Threading.Tasks; +using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Diagnostics; +using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Exceptions; +using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Models; +using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Runtime; +using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Storage; +using Newtonsoft.Json; +using Microsoft.VisualBasic.FileIO; +using FieldType = Microsoft.VisualBasic.FileIO.FieldType; + +namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services +{ + public interface IReplayFileService + { + /// + /// Get a replay file. + /// + Task GetAsync(string id); + + /// + /// Create a replay file. + /// + Task InsertAsync(DataFile replayFile); + + /// + /// Delete a replay file. + /// + Task DeleteAsync(string id); + + /// + /// Validate replay file. + /// + string ValidateFile(Stream stream); + } + + public class ReplayFileService : IReplayFileService + { + private readonly IEngine replayFilesStorage; + private readonly ILogger log; + + public ReplayFileService( + IServicesConfig config, + IEngines engines, + IEngine storage, + ILogger logger) + { + this.replayFilesStorage = engines.Build(config.ReplayFilesStorage); + this.log = logger; + } + + /// + /// Delete a device model script. + /// + public async Task DeleteAsync(string id) + { + try + { + await this.replayFilesStorage.DeleteAsync(id); + } + catch (Exception e) + { + this.log.Error("Something went wrong while deleting the replay file.", () => new { id, e }); + throw new ExternalDependencyException("Failed to delete the replay file", e); + } + } + + /// + /// Get a device model script. + /// + public async Task GetAsync(string id) + { + if (string.IsNullOrEmpty(id)) + { + this.log.Error("Simulation script id cannot be empty!"); + throw new InvalidInputException("Simulation script id cannot be empty! "); + } + + IDataRecord item; + try + { + item = await this.replayFilesStorage.GetAsync(id); + } + catch (ResourceNotFoundException) + { + throw; + } + catch (Exception e) + { + this.log.Error("Unable to load replay file from storage", () => new { id, e }); + throw new ExternalDependencyException("Unable to load device model script from storage", e); + } + + try + { + var deviceModelScript = JsonConvert.DeserializeObject(item.GetData()); + deviceModelScript.ETag = item.GetETag(); + return deviceModelScript; + } + catch (Exception e) + { + this.log.Error("Unable to parse device model script loaded from storage", () => new { id, e }); + throw new ExternalDependencyException("Unable to parse device model script loaded from storage", e); + } + } + + /// + /// Create a device model script. + /// + public async Task InsertAsync(DataFile replayFile) + { + replayFile.Created = DateTimeOffset.UtcNow; + replayFile.Modified = replayFile.Created; + + if (string.IsNullOrEmpty(replayFile.Id)) + { + replayFile.Id = Guid.NewGuid().ToString(); + } + + this.log.Debug("Creating a new replay file.", () => new { replayFile }); + + try + { + IDataRecord record = this.replayFilesStorage.BuildRecord(replayFile.Id, + JsonConvert.SerializeObject(replayFile)); + + var result = await this.replayFilesStorage.CreateAsync(record); + + replayFile.ETag = result.GetETag(); + } + catch (Exception e) + { + this.log.Error("Failed to insert new replay file into storage", + () => new { replayFile, e }); + throw new ExternalDependencyException( + "Failed to insert new replay file into storage", e); + } + + return replayFile; + } + + /// + /// Validate replay file + /// + public string ValidateFile(Stream stream) + { + var reader = new StreamReader(stream); + var file = reader.ReadToEnd(); + + using (TextFieldParser parser = new TextFieldParser(file)) + { + parser.TextFieldType = FieldType.Delimited; + parser.SetDelimiters(","); + while (!parser.EndOfData) + { + try + { + string[] lines = parser.ReadFields(); + } + catch (MalformedLineException ex) + { + this.log.Error("Replay file has invalid csv format", () => new { ex }); + throw new InvalidInputException("Replay file has invalid csv format", ex); + } + } + } + + return file; + } + } +} diff --git a/Services/Runtime/ServicesConfig.cs b/Services/Runtime/ServicesConfig.cs index dd02e367..9618d0f8 100644 --- a/Services/Runtime/ServicesConfig.cs +++ b/Services/Runtime/ServicesConfig.cs @@ -27,6 +27,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Runtime Config DevicesStorage { get; set; } Config PartitionsStorage { get; set; } Config StatisticsStorage { get; set; } + Config ReplayFilesStorage { get; set; } string DiagnosticsEndpointUrl { get; } string UserAgent { get; } bool DevelopmentMode { get; } @@ -116,6 +117,8 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Runtime public Config PartitionsStorage { get; set; } + public Config ReplayFilesStorage { get; set; } + public string UserAgent { get; set; } public bool DevelopmentMode { get; set; } diff --git a/Services/Services.csproj b/Services/Services.csproj index 062d60bb..3d02ba6b 100644 --- a/Services/Services.csproj +++ b/Services/Services.csproj @@ -19,6 +19,7 @@ + diff --git a/Services/Simulation/JavascriptInterpreter.cs b/Services/Simulation/JavascriptInterpreter.cs index 4179a115..899672ba 100644 --- a/Services/Simulation/JavascriptInterpreter.cs +++ b/Services/Simulation/JavascriptInterpreter.cs @@ -85,7 +85,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Simulation { Program program; bool isInStorage = string.Equals(script.Path.Trim(), - DeviceModelScript.DeviceModelScriptPath.Storage.ToString(), + DataFile.FilePath.Storage.ToString(), StringComparison.OrdinalIgnoreCase); string filename = isInStorage ? script.Id : script.Path; diff --git a/WebService.Test/WebService.Test.csproj b/WebService.Test/WebService.Test.csproj index b334d7c1..201cdc07 100755 --- a/WebService.Test/WebService.Test.csproj +++ b/WebService.Test/WebService.Test.csproj @@ -7,6 +7,7 @@ + diff --git a/WebService.Test/v1/Controllers/DeviceModelScriptsControllerTest.cs b/WebService.Test/v1/Controllers/DeviceModelScriptsControllerTest.cs index 893f2c4b..7956aa74 100644 --- a/WebService.Test/v1/Controllers/DeviceModelScriptsControllerTest.cs +++ b/WebService.Test/v1/Controllers/DeviceModelScriptsControllerTest.cs @@ -94,7 +94,7 @@ namespace WebService.Test.v1.Controllers IFormFile file = this.SetupFileMock(); this.deviceModelScriptsService - .Setup(x => x.InsertAsync(It.IsAny())) + .Setup(x => x.InsertAsync(It.IsAny())) .ReturnsAsync(deviceModelScript); // Act @@ -123,7 +123,7 @@ namespace WebService.Test.v1.Controllers IFormFile file = this.SetupFileMock(); this.deviceModelScriptsService - .Setup(x => x.UpsertAsync(It.IsAny())) + .Setup(x => x.UpsertAsync(It.IsAny())) .ReturnsAsync(deviceModelScript); // Act @@ -199,23 +199,23 @@ namespace WebService.Test.v1.Controllers return fileMock.Object; } - private DeviceModelScript GetDeviceModelScriptById(string id) + private DataFile GetDeviceModelScriptById(string id) { - return new DeviceModelScript + return new DataFile { Id = id, ETag = "etag", - Path = DeviceModelScript.DeviceModelScriptPath.Storage + Path = DataFile.FilePath.Storage }; } - private List GetDeviceModelScripts() + private List GetDeviceModelScripts() { - return new List + return new List { - new DeviceModelScript { Id = "Id_1", ETag = "Etag_1" }, - new DeviceModelScript { Id = "Id_2", ETag = "Etag_2" }, - new DeviceModelScript { Id = "Id_3", ETag = "Etag_3" } + new DataFile { Id = "Id_1", ETag = "Etag_1" }, + new DataFile { Id = "Id_2", ETag = "Etag_2" }, + new DataFile { Id = "Id_3", ETag = "Etag_3" } }; } } diff --git a/WebService/Runtime/Config.cs b/WebService/Runtime/Config.cs index 78425ed3..0e40eeb7 100644 --- a/WebService/Runtime/Config.cs +++ b/WebService/Runtime/Config.cs @@ -99,6 +99,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.Runtime private const string DEVICES_STORAGE_KEY = APPLICATION_KEY + "Storage:Devices:"; private const string PARTITIONS_STORAGE_KEY = APPLICATION_KEY + "Storage:Partitions:"; private const string STATISTICS_STORAGE_KEY = APPLICATION_KEY + "Storage:Statistics:"; + private const string REPLAY_FILES_STORAGE_KEY = APPLICATION_KEY + "Storage:ReplayFiles:"; private const string STORAGE_TYPE_KEY = "type"; private const string STORAGE_MAX_PENDING_OPERATIONS = "max_pending_storage_tasks"; @@ -285,6 +286,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.Runtime PartitionsStorage = GetStorageConfig(configData, PARTITIONS_STORAGE_KEY), UserAgent = configData.GetString(USER_AGENT_KEY, DEFAULT_USER_AGENT_STRING), StatisticsStorage = GetStorageConfig(configData, STATISTICS_STORAGE_KEY), + ReplayFilesStorage = GetStorageConfig(configData, REPLAY_FILES_STORAGE_KEY), DiagnosticsEndpointUrl = configData.GetString(LOGGING_DIAGNOSTICS_URL_KEY), DevelopmentMode = configData.GetBool(DEBUGGING_DEVELOPMENT_MODE_KEY, false), DisableSimulationAgent = configData.GetBool(DEBUGGING_DISABLE_SIMULATION_AGENT_KEY, false), diff --git a/WebService/Startup.cs b/WebService/Startup.cs index 72cb5935..8ea782a4 100755 --- a/WebService/Startup.cs +++ b/WebService/Startup.cs @@ -230,6 +230,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService log.Write("Main storage: " + config.ServicesConfig.MainStorage.StorageType); log.Write("Simulations storage: " + config.ServicesConfig.SimulationsStorage.StorageType); log.Write("Statistics storage: " + config.ServicesConfig.StatisticsStorage.StorageType); + log.Write("Replay files storage:" + config.ServicesConfig.ReplayFilesStorage.StorageType); log.Write("Devices storage: " + config.ServicesConfig.DevicesStorage.StorageType); log.Write("Partitions storage: " + config.ServicesConfig.PartitionsStorage.StorageType); log.Write("Nodes storage: " + config.ServicesConfig.NodesStorage.StorageType); diff --git a/WebService/WebService.csproj b/WebService/WebService.csproj index 53b5db53..3fc74628 100644 --- a/WebService/WebService.csproj +++ b/WebService/WebService.csproj @@ -42,6 +42,7 @@ + diff --git a/WebService/appsettings.ini b/WebService/appsettings.ini index 4d1b9e6e..d1da41db 100644 --- a/WebService/appsettings.ini +++ b/WebService/appsettings.ini @@ -197,6 +197,22 @@ cosmosdbsql_collection_throughput = 2500 max_pending_storage_tasks = 25 +[DeviceSimulationService:Storage:ReplayFiles] +# Type of storage used to store this data +# Possible values (not case sensitive): CosmosDbSql, TableStorage +type = "CosmosDbSql" +# Cosmos DB SQL (prev. known as DocumentDb) connection string, +# Format: AccountEndpoint=https://_____.documents.azure.com:443/;AccountKey=_____; +cosmosdbsql_connstring = "${PCS_STORAGEADAPTER_DOCUMENTDB_CONNSTRING}" +cosmosdbsql_database = "devicesimulation" +cosmosdbsql_collection = "replayFiles" +# CosmosDb throughput, see https://docs.microsoft.com/azure/cosmos-db/request-units +# Default: 400, Recommended: 2500 +cosmosdbsql_collection_throughput = 2500 +# Max number of concurrent requests when running tasks in parallel +# Default: 25 +max_pending_storage_tasks = 25 + [DeviceSimulationService:Deployment] # AAD Domain of the Azure subscription where the Azure IoT Hub is deployed. # The value is optional because the service can be deployed without a hub. diff --git a/WebService/v1/Controllers/DeviceModelScriptsController.cs b/WebService/v1/Controllers/DeviceModelScriptsController.cs index 911fe7eb..2dc210f3 100644 --- a/WebService/v1/Controllers/DeviceModelScriptsController.cs +++ b/WebService/v1/Controllers/DeviceModelScriptsController.cs @@ -99,13 +99,14 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.v1.Controller throw new BadRequestException("Wrong content type provided."); } - var deviceModelScript = new DeviceModelScript(); + var deviceModelScript = new DataFile(); try { var content = this.javascriptInterpreter.Validate(file.OpenReadStream()); deviceModelScript.Content = content; deviceModelScript.Name = file.FileName; + deviceModelScript.Type = ScriptInterpreter.JAVASCRIPT_SCRIPT; } catch (Exception e) { @@ -139,7 +140,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.v1.Controller throw new BadRequestException("No ETag provided."); } - var simulationScript = new DeviceModelScript + var simulationScript = new DataFile { ETag = eTag, Id = id @@ -150,6 +151,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.v1.Controller var reader = new StreamReader(file.OpenReadStream()); simulationScript.Content = reader.ReadToEnd(); simulationScript.Name = file.FileName; + simulationScript.Type = ScriptInterpreter.JAVASCRIPT_SCRIPT; } catch (Exception e) { diff --git a/WebService/v1/Controllers/ReplayFileController.cs b/WebService/v1/Controllers/ReplayFileController.cs new file mode 100644 index 00000000..7b90ab3d --- /dev/null +++ b/WebService/v1/Controllers/ReplayFileController.cs @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services; +using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Diagnostics; +using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Models; +using Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.v1.Exceptions; +using Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.v1.Filters; +using Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.v1.Models.Helpers; +using Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.v1.Models.ReplayFileApiModel; + +namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.v1.Controllers +{ + [ExceptionsFilter] + public class ReplayFileController : Controller + { + private const string TEXT_CSV = "text/csv"; + private const string TYPE_CSV = "csv"; + private readonly ILogger log; + private readonly IReplayFileService replayFileService; + + public ReplayFileController( + IReplayFileService replayFileService, + ILogger logger) + { + this.replayFileService = replayFileService; + this.log = logger; + } + + [HttpGet(Version.PATH + "/[controller]/{id}")] + public async Task GetAsync(string id) + { + return ReplayFileApiModel.FromServiceModel(await this.replayFileService.GetAsync(id)); + } + + [HttpPost(Version.PATH + "/[controller]!validate")] + public ActionResult Validate(IFormFile file) + { + try + { + var content = this.replayFileService.ValidateFile(file.OpenReadStream()); + } + catch (Exception e) + { + return new JsonResult(new ValidationApiModel + { + IsValid = false, + Messages = new List + { + e.Message + } + }) + { StatusCode = (int) HttpStatusCode.BadRequest }; + } + + return new JsonResult(new ValidationApiModel()) { StatusCode = (int) HttpStatusCode.OK }; + } + + [HttpPost(Version.PATH + "/[controller]")] + public async Task PostAsync(IFormFile file) + { + var replayFile = new DataFile(); + + try + { + var content = this.replayFileService.ValidateFile(file.OpenReadStream()); + replayFile.Content = content; + replayFile.Name = file.FileName; + } + catch (Exception e) + { + throw new BadRequestException(e.Message); + } + + return ReplayFileApiModel.FromServiceModel(await this.replayFileService.InsertAsync(replayFile)); + } + + [HttpDelete(Version.PATH + "/[controller]/{id}")] + public async Task DeleteAsync(string id) + { + await this.replayFileService.DeleteAsync(id); + } + + private void ValidateInput(IFormFile file) + { + if (file == null) + { + this.log.Warn("No replay data provided"); + throw new BadRequestException("No replay data provided."); + } + + if (file.ContentType != TEXT_CSV && !file.FileName.EndsWith(".csv")) + { + this.log.Warn("Wrong content type provided. Expected csv file format.", () => new { file.ContentType }); + throw new BadRequestException("Wrong content type provided. Expected csv file format."); + } + } + } +} diff --git a/WebService/v1/Models/DeviceModelScriptApiModel/DeviceModelScriptApiModel.cs b/WebService/v1/Models/DeviceModelScriptApiModel/DeviceModelScriptApiModel.cs index b1e3d86f..454144db 100644 --- a/WebService/v1/Models/DeviceModelScriptApiModel/DeviceModelScriptApiModel.cs +++ b/WebService/v1/Models/DeviceModelScriptApiModel/DeviceModelScriptApiModel.cs @@ -51,21 +51,21 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.v1.Models.Dev } // Map API model to service model - public Services.Models.DeviceModelScript ToServiceModel() + public Services.Models.DataFile ToServiceModel() { - return new Services.Models.DeviceModelScript + return new Services.Models.DataFile { ETag = this.ETag, Id = this.Id, Type = this.Type, - Path = (Services.Models.DeviceModelScript.DeviceModelScriptPath)Enum.Parse(typeof(Services.Models.DeviceModelScript.DeviceModelScriptPath), this.Path, true), + Path = (Services.Models.DataFile.FilePath)Enum.Parse(typeof(Services.Models.DataFile.FilePath), this.Path, true), Content = this.Content, Name = this.Name }; } // Map service model to API model - public static DeviceModelScriptApiModel FromServiceModel(Services.Models.DeviceModelScript value) + public static DeviceModelScriptApiModel FromServiceModel(Services.Models.DataFile value) { if (value == null) return null; diff --git a/WebService/v1/Models/DeviceModelScriptListModel.cs b/WebService/v1/Models/DeviceModelScriptListModel.cs index 7b71e3b6..643857c1 100644 --- a/WebService/v1/Models/DeviceModelScriptListModel.cs +++ b/WebService/v1/Models/DeviceModelScriptListModel.cs @@ -25,7 +25,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.v1.Models } // Map service model to API model - public static DeviceModelScriptListModel FromServiceModel(IEnumerable value) + public static DeviceModelScriptListModel FromServiceModel(IEnumerable value) { if (value == null) return null; diff --git a/WebService/v1/Models/ReplayFileApiModel.cs b/WebService/v1/Models/ReplayFileApiModel.cs new file mode 100644 index 00000000..e56d3ea8 --- /dev/null +++ b/WebService/v1/Models/ReplayFileApiModel.cs @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.v1.Models.ReplayFileApiModel +{ + public class ReplayFileApiModel + { + private const string DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:sszzz"; + + private DateTimeOffset created; + private DateTimeOffset modified; + + [JsonProperty(PropertyName = "ETag")] + public string ETag { get; set; } + + [JsonProperty(PropertyName = "Id")] + public string Id { get; set; } + + [JsonProperty(PropertyName = "Type")] + public string Type { get; set; } + + [JsonProperty(PropertyName = "Name")] + public string Name { get; set; } + + [JsonProperty(PropertyName = "Content")] + public string Content { get; set; } + + [JsonProperty(PropertyName = "Path")] + public string Path { get; set; } + + [JsonProperty(PropertyName = "$metadata", Order = 1000)] + public IDictionary Metadata => new Dictionary + { + { "$type", "ReplayFile;" + v1.Version.NUMBER }, + { "$uri", "/" + v1.Version.PATH + "/replayfile/" + this.Id }, + { "$created", this.created.ToString(DATE_FORMAT) }, + { "$modified", this.modified.ToString(DATE_FORMAT) } + }; + + public ReplayFileApiModel() + { + this.ETag = string.Empty; + this.Id = string.Empty; + this.Type = string.Empty; + this.Content = string.Empty; + this.Path = string.Empty; + this.Name = string.Empty; + } + + // Map API model to service model + public Services.Models.DataFile ToServiceModel() + { + return new Services.Models.DataFile + { + ETag = this.ETag, + Id = this.Id, + Type = this.Type, + Path = (Services.Models.DataFile.FilePath)Enum.Parse(typeof(Services.Models.DataFile.FilePath), this.Path, true), + Content = this.Content, + Name = this.Name, + }; + } + + // Map service model to API model + public static ReplayFileApiModel FromServiceModel(Services.Models.DataFile value) + { + if (value == null) return null; + + var result = new ReplayFileApiModel + { + ETag = value.ETag, + Id = value.Id, + Type = value.Type, + created = value.Created, + modified = value.Modified, + Path = value.Path.ToString(), + Content = value.Content, + Name = value.Name + }; + + return result; + } + } +} diff --git a/docs/CODEOWNERS b/docs/CODEOWNERS index 03bc0aa7..5798e69a 100644 --- a/docs/CODEOWNERS +++ b/docs/CODEOWNERS @@ -9,4 +9,4 @@ LICENSE @miwolfms @tommo-ms @saixiaohui scripts/ @miwolfms @tommo-ms @saixiaohui WebService/Auth/ @miwolfms @tommo-ms @saixiaohui -Services/Runtime/ @miwolfms @tommo-ms @saixiaohui +Services/Runtime/ @miwolfms @tommo-ms @saixiaohui \ No newline at end of file