Replay file APIs (#333)

* Replay file  APIs

* update settings

* PR comments

* update owners

* typo

* Update DeviceModelScriptsTest.cs

Services.Test/DeviceModelScriptsTest.cs

@@ -49,14 +49,14 @@ namespace Services.Test
             // Arrange
             var id = Guid.NewGuid().ToString();
             var eTag = Guid.NewGuid().ToString();
-            var deviceModelScript = new DeviceModelScript { Id = id, ETag = eTag };
+            var deviceModelScript = new DataFile { Id = id, ETag = eTag };
 
             this.storage
                 .Setup(x => x.UpdateAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>()))
                 .ReturnsAsync(this.BuildValueApiModel(deviceModelScript));
 
             // Act
-            DeviceModelScript result = this.target.InsertAsync(deviceModelScript).Result;
+            DataFile result = this.target.InsertAsync(deviceModelScript).Result;
 
             // Assert
             Assert.NotNull(result);
@@ -73,10 +73,10 @@ namespace Services.Test
             // Arrange
             var id = Guid.NewGuid().ToString();
 
-            var deviceModelScript = new DeviceModelScript { Id = id, ETag = "oldEtag" };
+            var deviceModelScript = new DataFile { Id = id, ETag = "oldEtag" };
             this.TheScriptExists(id, deviceModelScript);
 
-            var updatedSimulationScript = new DeviceModelScript { Id = id, ETag = "newETag" };
+            var updatedSimulationScript = new DataFile { Id = id, ETag = "newETag" };
             this.storage
                 .Setup(x => x.UpdateAsync(
                     STORAGE_COLLECTION,
@@ -94,7 +94,7 @@ namespace Services.Test
             this.storage.Verify(x => x.UpdateAsync(
                 STORAGE_COLLECTION,
                 id,
-                It.Is<string>(json => JsonConvert.DeserializeObject<DeviceModelScript>(json).Id == id && !json.Contains("ETag")),
+                It.Is<string>(json => JsonConvert.DeserializeObject<DataFile>(json).Id == id && !json.Contains("ETag")),
                 "oldEtag"), Times.Once());
 
             Assert.Equal(updatedSimulationScript.Id, deviceModelScript.Id);
@@ -107,7 +107,7 @@ namespace Services.Test
         {
             // Arrange
             var id = Guid.NewGuid().ToString();
-            var deviceModelScript = new DeviceModelScript { Id = id, ETag = "Etag" };
+            var deviceModelScript = new DataFile { Id = id, ETag = "Etag" };
             this.TheScriptDoesntExist(id);
             this.storage
                 .Setup(x => x.UpdateAsync(
@@ -133,11 +133,11 @@ namespace Services.Test
         {
             // Arrange
             var id = Guid.NewGuid().ToString();
-            var deviceModelScriptInStorage = new DeviceModelScript { Id = id, ETag = "ETag" };
+            var deviceModelScriptInStorage = new DataFile { Id = id, ETag = "ETag" };
             this.TheScriptExists(id, deviceModelScriptInStorage);
 
             // Act & Assert
-            var deviceModelScript = new DeviceModelScript { Id = id, ETag = "not-matching-Etag" };
+            var deviceModelScript = new DataFile { Id = id, ETag = "not-matching-Etag" };
             Assert.ThrowsAsync<ConflictingResourceException>(
                     async () => await this.target.UpsertAsync(deviceModelScript))
                 .Wait(Constants.TEST_TIMEOUT);
@@ -147,7 +147,7 @@ namespace Services.Test
         public void ItThrowsExceptionWhenInsertDeviceModelScriptFailed()
         {
             // Arrange
-            var deviceModelScript = new DeviceModelScript { Id = "id", ETag = "Etag" };
+            var deviceModelScript = new DataFile { Id = "id", ETag = "Etag" };
             this.storage
                 .Setup(x => x.UpdateAsync(
                     It.IsAny<string>(),
@@ -166,7 +166,7 @@ namespace Services.Test
         public void ItFailsToUpsertWhenUnableToFetchScriptFromStorage()
         {
             // Arrange
-            var deviceModelScript = new DeviceModelScript { Id = "id", ETag = "Etag" };
+            var deviceModelScript = new DataFile { Id = "id", ETag = "Etag" };
             this.storage
                 .Setup(x => x.GetAsync(It.IsAny<string>(), It.IsAny<string>()))
                 .ThrowsAsync(new SomeException());
@@ -182,7 +182,7 @@ namespace Services.Test
         {
             // Arrange
             var id = Guid.NewGuid().ToString();
-            var deviceModelScript = new DeviceModelScript { Id = id, ETag = "Etag" };
+            var deviceModelScript = new DataFile { Id = id, ETag = "Etag" };
             this.TheScriptExists(id, deviceModelScript);
 
             this.storage
@@ -203,7 +203,7 @@ namespace Services.Test
         public void ItThrowsExternalDependencyExceptionWhenFailedFetchingDeviceModelScriptInStorage()
         {
             // Arrange
-            var deviceModelScript = new DeviceModelScript { Id = "id", ETag = "Etag" };
+            var deviceModelScript = new DataFile { Id = "id", ETag = "Etag" };
 
             // Act
             var ex = Record.Exception(() => this.target.UpsertAsync(deviceModelScript).Result);
@@ -268,7 +268,7 @@ namespace Services.Test
             var list = new ValueListApiModel();
             var value = new ValueApiModel
             {
-                Key = "key",
+                Key = "key1",
                 Data = "{ 'invalid': json",
                 ETag = "etag"
             };
@@ -286,14 +286,14 @@ namespace Services.Test
                 .Throws<ResourceNotFoundException>();
         }
 
-        private void TheScriptExists(string id, DeviceModelScript deviceModelScript)
+        private void TheScriptExists(string id, DataFile deviceModelScript)
         {
             this.storage
                 .Setup(x => x.GetAsync(STORAGE_COLLECTION, id))
                 .ReturnsAsync(this.BuildValueApiModel(deviceModelScript));
         }
 
-        private ValueApiModel BuildValueApiModel(DeviceModelScript deviceModelScript)
+        private ValueApiModel BuildValueApiModel(DataFile deviceModelScript)
         {
             return new ValueApiModel
             {

Services.Test/ReplayFileServiceTest.cs

@@ -0,0 +1,133 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services;
+using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Diagnostics;
+using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Exceptions;
+using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Models;
+using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Runtime;
+using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Storage;
+using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Storage.CosmosDbSql;
+using Moq;
+using Newtonsoft.Json;
+using Services.Test.helpers;
+using System;
+using Xunit;
+
+namespace Services.Test
+{
+    public class ReplayFileServiceTest
+    {
+        private const string REPLAY_FILES = "replayFiles";
+        
+        private readonly Mock<ILogger> log;
+        private readonly Mock<IServicesConfig> config;
+        private readonly Mock<IEngines> enginesFactory;
+        private readonly Mock<IEngine> replayFilesStorage;
+        private readonly Mock<ILogger> logger;
+        private readonly ReplayFileService target;
+
+        public ReplayFileServiceTest()
+        {
+            this.log = new Mock<ILogger>();
+            this.config = new Mock<IServicesConfig>();
+            this.enginesFactory = new Mock<IEngines>();
+
+            this.replayFilesStorage = new Mock<IEngine>();
+            this.replayFilesStorage.Setup(x => x.BuildRecord(It.IsAny<string>(), It.IsAny<string>()))
+                .Returns((string id, string json) => new DataRecord { Id = id, Data = json });
+            this.replayFilesStorage.Setup(x => x.BuildRecord(It.IsAny<string>()))
+                .Returns((string id) => new DataRecord { Id = id });
+
+            this.config.SetupGet(x => x.ReplayFilesStorage)
+                .Returns(new Config { CosmosDbSqlCollection = REPLAY_FILES });
+
+            this.enginesFactory
+              .Setup(x => x.Build(It.Is<Config>(c => c.CosmosDbSqlCollection == REPLAY_FILES)))
+              .Returns(this.replayFilesStorage.Object);
+
+            this.target = new ReplayFileService(
+                this.config.Object,
+                this.enginesFactory.Object,
+                this.replayFilesStorage.Object,
+                this.log.Object);
+        }
+
+        [Fact, Trait(Constants.TYPE, Constants.UNIT_TEST)]
+        public void ItCreatesReplayFileInStorage()
+        {
+            // Arrange
+            var id = Guid.NewGuid().ToString();
+            var replayFile = new DataFile { Id = id, Content = "1, 2, 3", ETag = "tag" };
+
+            this.replayFilesStorage
+                .Setup(x => x.CreateAsync(It.IsAny<IDataRecord>()))
+                .ReturnsAsync(new DataRecord { Id = id, Data = JsonConvert.SerializeObject(replayFile) });
+
+            // Act
+            DataFile result = this.target.InsertAsync(replayFile).Result;
+
+            // Assert
+            Assert.NotNull(result);
+            Assert.Equal(replayFile.Id, result.Id);
+            Assert.Equal(replayFile.Content, result.Content);
+
+            this.replayFilesStorage.Verify(
+                x => x.CreateAsync(It.Is<IDataRecord>(n => n.GetId() == id)), Times.Once());
+        }
+
+        [Fact, Trait(Constants.TYPE, Constants.UNIT_TEST)]
+        public void ItThrowsExceptionWhenCreateReplayFileFails()
+        {
+            // Arrange
+            DataFile replayFile = new DataFile();
+            replayFile.Content = "1, 2, 3, 4, 5";
+
+            this.replayFilesStorage
+                .Setup(x => x.CreateAsync(It.IsAny<IDataRecord>()))
+                .ThrowsAsync(new SomeException());
+
+            // Act & Assert
+            Assert.ThrowsAsync<ExternalDependencyException>(
+                    async () => await this.target.InsertAsync(replayFile))
+                .Wait(Constants.TEST_TIMEOUT);
+        }
+
+
+        [Fact, Trait(Constants.TYPE, Constants.UNIT_TEST)]
+        public void ItThrowsExceptionWhenDeleteReplayFileFailes()
+        {
+            // Arrange
+            this.replayFilesStorage
+                .Setup(x => x.DeleteAsync(It.IsAny<string>()))
+                .ThrowsAsync(new SomeException());
+
+            // Act & Assert
+            Assert.ThrowsAsync<ExternalDependencyException>(
+                    async () => await this.target.DeleteAsync("Id"))
+                .Wait(Constants.TEST_TIMEOUT);
+        }
+
+        [Fact, Trait(Constants.TYPE, Constants.UNIT_TEST)]
+        public void ItFailsToGetReplayFileWhenGetAsyncFails()
+        {
+            // Arrange
+            this.replayFilesStorage
+                .Setup(x => x.GetAsync(It.IsAny<string>()))
+                .ThrowsAsync(new SomeException());
+
+            // Act & Assert
+            Assert.ThrowsAsync<ExternalDependencyException>(
+                    async () => await this.target.GetAsync("Id"))
+                .Wait(Constants.TEST_TIMEOUT);
+        }
+
+        [Fact, Trait(Constants.TYPE, Constants.UNIT_TEST)]
+        public void ItThrowsExceptionForInvalidId()
+        {
+            // Act & Assert
+            Assert.ThrowsAsync<InvalidInputException>(
+                    async () => await this.target.GetAsync(string.Empty))
+                .Wait(Constants.TEST_TIMEOUT);
+        }
+    }
+}

Services/DeviceModelScripts.cs

@@ -17,22 +17,22 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services
         /// <summary>
         /// Get list of device model scripts.
         /// </summary>
-        Task<IEnumerable<DeviceModelScript>> GetListAsync();
+        Task<IEnumerable<DataFile>> GetListAsync();
 
         /// <summary>
         /// Get a device model script.
         /// </summary>
-        Task<DeviceModelScript> GetAsync(string id);
+        Task<DataFile> GetAsync(string id);
 
         /// <summary>
         /// Create a device model script.
         /// </summary>
-        Task<DeviceModelScript> InsertAsync(DeviceModelScript deviceModelScript);
+        Task<DataFile> InsertAsync(DataFile deviceModelScript);
 
         /// <summary>
         /// Create or replace a device model script.
         /// </summary>
-        Task<DeviceModelScript> UpsertAsync(DeviceModelScript deviceModelScript);
+        Task<DataFile> UpsertAsync(DataFile deviceModelScript);
 
         /// <summary>
         /// Delete a device model script.
@@ -74,7 +74,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services
         /// <summary>
         /// Get a device model script.
         /// </summary>
-        public async Task<DeviceModelScript> GetAsync(string id)
+        public async Task<DataFile> GetAsync(string id)
         {
             if (string.IsNullOrEmpty(id))
             {
@@ -99,7 +99,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services
 
             try
             {
-                var deviceModelScript = JsonConvert.DeserializeObject<DeviceModelScript>(item.Data);
+                var deviceModelScript = JsonConvert.DeserializeObject<DataFile>(item.Data);
                 deviceModelScript.ETag = item.ETag;
                 return deviceModelScript;
             }
@@ -113,7 +113,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services
         /// <summary>
         /// Get list of device model scripts.
         /// </summary>
-        public async Task<IEnumerable<DeviceModelScript>> GetListAsync()
+        public async Task<IEnumerable<DataFile>> GetListAsync()
         {
             ValueListApiModel data;
 
@@ -129,13 +129,13 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services
 
             try
             {
-                var results = new List<DeviceModelScript>();
+                var results = new List<DataFile>();
                 foreach (var item in data.Items)
                 {
-                    var deviceModelScript = JsonConvert.DeserializeObject<DeviceModelScript>(item.Data);
+                    var deviceModelScript = JsonConvert.DeserializeObject<DataFile>(item.Data);
                     deviceModelScript.ETag = item.ETag;
                     deviceModelScript.Type = ScriptInterpreter.JAVASCRIPT_SCRIPT;
-                    deviceModelScript.Path = DeviceModelScript.DeviceModelScriptPath.Storage;
+                    deviceModelScript.Path = DataFile.FilePath.Storage;
                     results.Add(deviceModelScript);
                 }
 
@@ -151,7 +151,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services
         /// <summary>
         /// Create a device model script.
         /// </summary>
-        public async Task<DeviceModelScript> InsertAsync(DeviceModelScript deviceModelScript)
+        public async Task<DataFile> InsertAsync(DataFile deviceModelScript)
         {
             deviceModelScript.Created = DateTimeOffset.UtcNow;
             deviceModelScript.Modified = deviceModelScript.Created;
@@ -188,7 +188,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services
         /// <summary>
         /// Create or replace a device model script.
         /// </summary>
-        public async Task<DeviceModelScript> UpsertAsync(DeviceModelScript deviceModelScript)
+        public async Task<DataFile> UpsertAsync(DataFile deviceModelScript)
         {
             var id = deviceModelScript.Id;
             var eTag = deviceModelScript.ETag;

Services/Models/DataFile.cs

@@ -2,13 +2,12 @@
 
 using System;
 using System.Runtime.Serialization;
-using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Simulation;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Converters;
 
 namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Models
 {
-    public class DeviceModelScript
+    public class DataFile
     {
         [JsonIgnore]
         public string ETag { get; set; }
@@ -16,22 +15,22 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Models
         public string Name { get; set; }
         public string Type { get; set; }
         public string Content { get; set; }
-        public DeviceModelScriptPath Path { get; set; }
+        public FilePath Path { get; set; }
         public DateTimeOffset Created { get; set; }
         public DateTimeOffset Modified { get; set; }
 
-        public DeviceModelScript()
+        public DataFile()
         {
             this.ETag = string.Empty;
             this.Id = string.Empty;
-            this.Type = ScriptInterpreter.JAVASCRIPT_SCRIPT;
+            this.Type = string.Empty;
             this.Content = string.Empty;
-            this.Path = DeviceModelScriptPath.Storage;
+            this.Path = FilePath.Storage;
             this.Name = string.Empty;
         }
 
         [JsonConverter(typeof(StringEnumConverter))]
-        public enum DeviceModelScriptPath
+        public enum FilePath
         {
             [EnumMember(Value = "Undefined")]
             Undefined = 0,

Services/Models/Simulation.cs

@@ -133,9 +133,13 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Models
         public DateTimeOffset Modified { get; set; }
 
         // ActualStartTime is the time when Simulation was started
-        [JsonProperty(Order = 140)]
+        [JsonProperty(Order = 150)]
         public DateTimeOffset? ActualStartTime { get; set; }
 
+        // ReplayFileId is the id of the replay file in storage
+        [JsonProperty(Order = 160)]
+        public string ReplayFileId { get; set; }
+
         public Simulation()
         {
             // When unspecified, a simulation is enabled

Services/ReplayFileService.cs

@@ -0,0 +1,174 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.IO;
+using System.Threading.Tasks;
+using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Diagnostics;
+using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Exceptions;
+using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Models;
+using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Runtime;
+using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Storage;
+using Newtonsoft.Json;
+using Microsoft.VisualBasic.FileIO;
+using FieldType = Microsoft.VisualBasic.FileIO.FieldType;
+
+namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services
+{
+    public interface IReplayFileService
+    {
+        /// <summary>
+        /// Get a replay file.
+        /// </summary>
+        Task<DataFile> GetAsync(string id);
+
+        /// <summary>
+        /// Create a replay file.
+        /// </summary>
+        Task<DataFile> InsertAsync(DataFile replayFile);
+
+        /// <summary>
+        /// Delete a replay file.
+        /// </summary>
+        Task DeleteAsync(string id);
+
+        /// <summary>
+        /// Validate replay file.
+        /// </summary>
+        string ValidateFile(Stream stream);
+    }
+
+    public class ReplayFileService : IReplayFileService
+    {
+        private readonly IEngine replayFilesStorage;
+        private readonly ILogger log;
+
+        public ReplayFileService(
+            IServicesConfig config,
+            IEngines engines,
+            IEngine storage,
+            ILogger logger)
+        {
+            this.replayFilesStorage = engines.Build(config.ReplayFilesStorage);
+            this.log = logger;
+        }
+
+        /// <summary>
+        /// Delete a device model script.
+        /// </summary>
+        public async Task DeleteAsync(string id)
+        {
+            try
+            {
+                await this.replayFilesStorage.DeleteAsync(id);
+            }
+            catch (Exception e)
+            {
+                this.log.Error("Something went wrong while deleting the replay file.", () => new { id, e });
+                throw new ExternalDependencyException("Failed to delete the replay file", e);
+            }
+        }
+
+        /// <summary>
+        /// Get a device model script.
+        /// </summary>
+        public async Task<DataFile> GetAsync(string id)
+        {
+            if (string.IsNullOrEmpty(id))
+            {
+                this.log.Error("Simulation script id cannot be empty!");
+                throw new InvalidInputException("Simulation script id cannot be empty! ");
+            }
+
+            IDataRecord item;
+            try
+            {
+                item = await this.replayFilesStorage.GetAsync(id);
+            }
+            catch (ResourceNotFoundException)
+            {
+                throw;
+            }
+            catch (Exception e)
+            {
+                this.log.Error("Unable to load replay file from storage", () => new { id, e });
+                throw new ExternalDependencyException("Unable to load device model script from storage", e);
+            }
+
+            try
+            {
+                var deviceModelScript = JsonConvert.DeserializeObject<DataFile>(item.GetData());
+                deviceModelScript.ETag = item.GetETag();
+                return deviceModelScript;
+            }
+            catch (Exception e)
+            {
+                this.log.Error("Unable to parse device model script loaded from storage", () => new { id, e });
+                throw new ExternalDependencyException("Unable to parse device model script loaded from storage", e);
+            }
+        }
+
+        /// <summary>
+        /// Create a device model script.
+        /// </summary>
+        public async Task<DataFile> InsertAsync(DataFile replayFile)
+        {
+            replayFile.Created = DateTimeOffset.UtcNow;
+            replayFile.Modified = replayFile.Created;
+
+            if (string.IsNullOrEmpty(replayFile.Id))
+            {
+                replayFile.Id = Guid.NewGuid().ToString();
+            }
+
+            this.log.Debug("Creating a new replay file.", () => new { replayFile });
+
+            try
+            {
+                IDataRecord record = this.replayFilesStorage.BuildRecord(replayFile.Id,
+                    JsonConvert.SerializeObject(replayFile));
+
+                var result = await this.replayFilesStorage.CreateAsync(record);
+
+                replayFile.ETag = result.GetETag();
+            }
+            catch (Exception e)
+            {
+                this.log.Error("Failed to insert new replay file into storage",
+                    () => new { replayFile, e });
+                throw new ExternalDependencyException(
+                    "Failed to insert new replay file into storage", e);
+            }
+
+            return replayFile;
+        }
+
+        /// <summary>
+        /// Validate replay file
+        /// </summary>
+        public string ValidateFile(Stream stream)
+        {
+            var reader = new StreamReader(stream);
+            var file = reader.ReadToEnd();
+
+            using (TextFieldParser parser = new TextFieldParser(file))
+            {
+                parser.TextFieldType = FieldType.Delimited;
+                parser.SetDelimiters(",");
+                while (!parser.EndOfData)
+                {
+                    try
+                    {
+                        string[] lines = parser.ReadFields();
+                    }
+                    catch (MalformedLineException ex)
+                    {
+                        this.log.Error("Replay file has invalid csv format", () => new { ex });
+                        throw new InvalidInputException("Replay file has invalid csv format", ex);
+                    }
+                }
+            }
+
+            return file;
+        }
+    }
+}

Services/Runtime/ServicesConfig.cs

@@ -27,6 +27,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Runtime
         Config DevicesStorage { get; set; }
         Config PartitionsStorage { get; set; }
         Config StatisticsStorage { get; set; }
+        Config ReplayFilesStorage { get; set; }
         string DiagnosticsEndpointUrl { get; }
         string UserAgent { get; }
         bool DevelopmentMode { get; }
@@ -116,6 +117,8 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Runtime
 
         public Config PartitionsStorage { get; set; }
 
+        public Config ReplayFilesStorage { get; set; }
+
         public string UserAgent { get; set; }
 
         public bool DevelopmentMode { get; set; }

Services/Services.csproj

@@ -19,6 +19,7 @@
         <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="2.1.1" />
         <PackageReference Include="Microsoft.Extensions.Configuration.Ini" Version="2.1.1" />
         <PackageReference Include="Newtonsoft.Json" Version="12.0.1" />
+        <PackageReference Include="Upn_Components_Dependencies" Version="1.0.0" />
         <PackageReference Include="WindowsAzure.Storage" Version="9.3.3" />
     </ItemGroup>
     <ItemGroup>

Services/Simulation/JavascriptInterpreter.cs

@@ -85,7 +85,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Simulation
             {
                 Program program;
                 bool isInStorage = string.Equals(script.Path.Trim(),
-                    DeviceModelScript.DeviceModelScriptPath.Storage.ToString(),
+                    DataFile.FilePath.Storage.ToString(),
                     StringComparison.OrdinalIgnoreCase);
                 string filename = isInStorage ? script.Id : script.Path;
 

WebService.Test/WebService.Test.csproj

@@ -7,6 +7,7 @@
     <ItemGroup>
         <PackageReference Include="Microsoft.AspNetCore.TestHost" Version="2.1.1" />
         <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.9.0" />
+        <PackageReference Include="Microsoft.VisualBasic" Version="10.3.0" />
         <PackageReference Include="Moq" Version="4.10.1" />
         <PackageReference Include="xunit" Version="2.4.1" />
         <PackageReference Include="xunit.assert" Version="2.4.1" />

WebService.Test/v1/Controllers/DeviceModelScriptsControllerTest.cs

@@ -94,7 +94,7 @@ namespace WebService.Test.v1.Controllers
             IFormFile file = this.SetupFileMock();
 
             this.deviceModelScriptsService
-                .Setup(x => x.InsertAsync(It.IsAny<DeviceModelScript>()))
+                .Setup(x => x.InsertAsync(It.IsAny<DataFile>()))
                 .ReturnsAsync(deviceModelScript);
 
             // Act
@@ -123,7 +123,7 @@ namespace WebService.Test.v1.Controllers
             IFormFile file = this.SetupFileMock();
 
             this.deviceModelScriptsService
-                .Setup(x => x.UpsertAsync(It.IsAny<DeviceModelScript>()))
+                .Setup(x => x.UpsertAsync(It.IsAny<DataFile>()))
                 .ReturnsAsync(deviceModelScript);
 
             // Act
@@ -199,23 +199,23 @@ namespace WebService.Test.v1.Controllers
             return fileMock.Object;
         }
 
-        private DeviceModelScript GetDeviceModelScriptById(string id)
+        private DataFile GetDeviceModelScriptById(string id)
         {
-            return new DeviceModelScript
+            return new DataFile
             {
                 Id = id,
                 ETag = "etag",
-                Path = DeviceModelScript.DeviceModelScriptPath.Storage
+                Path = DataFile.FilePath.Storage
             };
         }
 
-        private List<DeviceModelScript> GetDeviceModelScripts()
+        private List<DataFile> GetDeviceModelScripts()
         {
-            return new List<DeviceModelScript>
+            return new List<DataFile>
             {
-                new DeviceModelScript { Id = "Id_1", ETag = "Etag_1" },
+                new DataFile { Id = "Id_1", ETag = "Etag_1" },
-                new DeviceModelScript { Id = "Id_2", ETag = "Etag_2" },
+                new DataFile { Id = "Id_2", ETag = "Etag_2" },
-                new DeviceModelScript { Id = "Id_3", ETag = "Etag_3" }
+                new DataFile { Id = "Id_3", ETag = "Etag_3" }
             };
         }
     }

WebService/Runtime/Config.cs

@@ -99,6 +99,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.Runtime
         private const string DEVICES_STORAGE_KEY = APPLICATION_KEY + "Storage:Devices:";
         private const string PARTITIONS_STORAGE_KEY = APPLICATION_KEY + "Storage:Partitions:";
         private const string STATISTICS_STORAGE_KEY = APPLICATION_KEY + "Storage:Statistics:";
+        private const string REPLAY_FILES_STORAGE_KEY = APPLICATION_KEY + "Storage:ReplayFiles:";
 
         private const string STORAGE_TYPE_KEY = "type";
         private const string STORAGE_MAX_PENDING_OPERATIONS = "max_pending_storage_tasks";
@@ -285,6 +286,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.Runtime
                 PartitionsStorage = GetStorageConfig(configData, PARTITIONS_STORAGE_KEY),
                 UserAgent = configData.GetString(USER_AGENT_KEY, DEFAULT_USER_AGENT_STRING),
                 StatisticsStorage = GetStorageConfig(configData, STATISTICS_STORAGE_KEY),
+                ReplayFilesStorage = GetStorageConfig(configData, REPLAY_FILES_STORAGE_KEY),
                 DiagnosticsEndpointUrl = configData.GetString(LOGGING_DIAGNOSTICS_URL_KEY),
                 DevelopmentMode = configData.GetBool(DEBUGGING_DEVELOPMENT_MODE_KEY, false),
                 DisableSimulationAgent = configData.GetBool(DEBUGGING_DISABLE_SIMULATION_AGENT_KEY, false),

WebService/Startup.cs

@@ -230,6 +230,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService
             log.Write("Main storage:        " + config.ServicesConfig.MainStorage.StorageType);
             log.Write("Simulations storage: " + config.ServicesConfig.SimulationsStorage.StorageType);
             log.Write("Statistics storage:  " + config.ServicesConfig.StatisticsStorage.StorageType);
+            log.Write("Replay files storage:" + config.ServicesConfig.ReplayFilesStorage.StorageType);
             log.Write("Devices storage:     " + config.ServicesConfig.DevicesStorage.StorageType);
             log.Write("Partitions storage:  " + config.ServicesConfig.PartitionsStorage.StorageType);
             log.Write("Nodes storage:       " + config.ServicesConfig.NodesStorage.StorageType);

WebService/WebService.csproj

@@ -42,6 +42,7 @@
         <PackageReference Include="Microsoft.AspNetCore.Mvc" Version="2.1.3" />
         <PackageReference Include="Microsoft.Extensions.Configuration.Ini" Version="2.1.1" />
         <PackageReference Include="Microsoft.IdentityModel.Protocols.OpenIdConnect" Version="2.1.5" />
+        <PackageReference Include="Microsoft.VisualBasic" Version="10.3.0" />
     </ItemGroup>
     <ItemGroup>
         <ProjectReference Include="..\PartitioningAgent\PartitioningAgent.csproj" />

WebService/appsettings.ini

@@ -197,6 +197,22 @@ cosmosdbsql_collection_throughput = 2500
 max_pending_storage_tasks = 25
 
 
+[DeviceSimulationService:Storage:ReplayFiles]
+# Type of storage used to store this data
+# Possible values (not case sensitive): CosmosDbSql, TableStorage
+type = "CosmosDbSql"
+# Cosmos DB SQL (prev. known as DocumentDb) connection string,
+# Format: AccountEndpoint=https://_____.documents.azure.com:443/;AccountKey=_____;
+cosmosdbsql_connstring = "${PCS_STORAGEADAPTER_DOCUMENTDB_CONNSTRING}"
+cosmosdbsql_database = "devicesimulation"
+cosmosdbsql_collection = "replayFiles"
+# CosmosDb throughput, see https://docs.microsoft.com/azure/cosmos-db/request-units
+# Default: 400, Recommended: 2500
+cosmosdbsql_collection_throughput = 2500
+# Max number of concurrent requests when running tasks in parallel
+# Default: 25
+max_pending_storage_tasks = 25
+
 [DeviceSimulationService:Deployment]
 # AAD Domain of the Azure subscription where the Azure IoT Hub is deployed.
 # The value is optional because the service can be deployed without a hub.

WebService/v1/Controllers/DeviceModelScriptsController.cs

@@ -99,13 +99,14 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.v1.Controller
                 throw new BadRequestException("Wrong content type provided.");
             }
 
-            var deviceModelScript = new DeviceModelScript();
+            var deviceModelScript = new DataFile();
 
             try
             {
                 var content = this.javascriptInterpreter.Validate(file.OpenReadStream());
                 deviceModelScript.Content = content;
                 deviceModelScript.Name = file.FileName;
+                deviceModelScript.Type = ScriptInterpreter.JAVASCRIPT_SCRIPT;
             }
             catch (Exception e)
             {
@@ -139,7 +140,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.v1.Controller
                 throw new BadRequestException("No ETag provided.");
             }
 
-            var simulationScript = new DeviceModelScript
+            var simulationScript = new DataFile
             {
                 ETag = eTag,
                 Id = id
@@ -150,6 +151,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.v1.Controller
                 var reader = new StreamReader(file.OpenReadStream());
                 simulationScript.Content = reader.ReadToEnd();
                 simulationScript.Name = file.FileName;
+                simulationScript.Type = ScriptInterpreter.JAVASCRIPT_SCRIPT;
             }
             catch (Exception e)
             {

WebService/v1/Controllers/ReplayFileController.cs

@@ -0,0 +1,105 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Net;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services;
+using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Diagnostics;
+using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Models;
+using Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.v1.Exceptions;
+using Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.v1.Filters;
+using Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.v1.Models.Helpers;
+using Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.v1.Models.ReplayFileApiModel;
+
+namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.v1.Controllers
+{
+    [ExceptionsFilter]
+    public class ReplayFileController : Controller
+    {
+        private const string TEXT_CSV = "text/csv";
+        private const string TYPE_CSV = "csv";
+        private readonly ILogger log;
+        private readonly IReplayFileService replayFileService;
+
+        public ReplayFileController(
+            IReplayFileService replayFileService,
+            ILogger logger)
+        {
+            this.replayFileService = replayFileService;
+            this.log = logger;
+        }
+
+        [HttpGet(Version.PATH + "/[controller]/{id}")]
+        public async Task<ReplayFileApiModel> GetAsync(string id)
+        {
+            return ReplayFileApiModel.FromServiceModel(await this.replayFileService.GetAsync(id));
+        }
+
+        [HttpPost(Version.PATH + "/[controller]!validate")]
+        public ActionResult Validate(IFormFile file)
+        {
+            try
+            {
+                var content = this.replayFileService.ValidateFile(file.OpenReadStream());
+            }
+            catch (Exception e)
+            {
+                return new JsonResult(new ValidationApiModel
+                    { 
+                        IsValid = false,
+                        Messages = new List<string>
+                        {
+                            e.Message
+                        }
+                    })
+                    { StatusCode = (int) HttpStatusCode.BadRequest };
+            }
+
+            return new JsonResult(new ValidationApiModel()) { StatusCode = (int) HttpStatusCode.OK };
+        }
+
+        [HttpPost(Version.PATH + "/[controller]")]
+        public async Task<ReplayFileApiModel> PostAsync(IFormFile file)
+        {
+            var replayFile = new DataFile();
+
+            try
+            {
+                var content = this.replayFileService.ValidateFile(file.OpenReadStream());
+                replayFile.Content = content;
+                replayFile.Name = file.FileName;
+            }
+            catch (Exception e)
+            {
+                throw new BadRequestException(e.Message);
+            }
+
+            return ReplayFileApiModel.FromServiceModel(await this.replayFileService.InsertAsync(replayFile));
+        }
+
+        [HttpDelete(Version.PATH + "/[controller]/{id}")]
+        public async Task DeleteAsync(string id)
+        {
+            await this.replayFileService.DeleteAsync(id);
+        }
+
+        private void ValidateInput(IFormFile file)
+        {
+            if (file == null)
+            {
+                this.log.Warn("No replay data provided");
+                throw new BadRequestException("No replay data provided.");
+            }
+
+            if (file.ContentType != TEXT_CSV && !file.FileName.EndsWith(".csv"))
+            {
+                this.log.Warn("Wrong content type provided. Expected csv file format.", () => new { file.ContentType });
+                throw new BadRequestException("Wrong content type provided. Expected csv file format.");
+            }
+        }
+    }
+}

WebService/v1/Models/DeviceModelScriptApiModel/DeviceModelScriptApiModel.cs

@@ -51,21 +51,21 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.v1.Models.Dev
         }
 
         // Map API model to service model
-        public Services.Models.DeviceModelScript ToServiceModel()
+        public Services.Models.DataFile ToServiceModel()
         {
-            return new Services.Models.DeviceModelScript
+            return new Services.Models.DataFile
             {
                 ETag = this.ETag,
                 Id = this.Id,
                 Type = this.Type,
-                Path = (Services.Models.DeviceModelScript.DeviceModelScriptPath)Enum.Parse(typeof(Services.Models.DeviceModelScript.DeviceModelScriptPath), this.Path, true),
+                Path = (Services.Models.DataFile.FilePath)Enum.Parse(typeof(Services.Models.DataFile.FilePath), this.Path, true),
                 Content = this.Content,
                 Name = this.Name
             };
         }
 
         // Map service model to API model
-        public static DeviceModelScriptApiModel FromServiceModel(Services.Models.DeviceModelScript value)
+        public static DeviceModelScriptApiModel FromServiceModel(Services.Models.DataFile value)
         {
             if (value == null) return null;
 

WebService/v1/Models/DeviceModelScriptListModel.cs

@@ -25,7 +25,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.v1.Models
         }
 
         // Map service model to API model
-        public static DeviceModelScriptListModel FromServiceModel(IEnumerable<DeviceModelScript> value)
+        public static DeviceModelScriptListModel FromServiceModel(IEnumerable<DataFile> value)
         {
             if (value == null) return null;
 

WebService/v1/Models/ReplayFileApiModel.cs

@@ -0,0 +1,87 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.v1.Models.ReplayFileApiModel
+{
+    public class ReplayFileApiModel
+    {
+        private const string DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:sszzz";
+
+        private DateTimeOffset created;
+        private DateTimeOffset modified;
+
+        [JsonProperty(PropertyName = "ETag")]
+        public string ETag { get; set; }
+
+        [JsonProperty(PropertyName = "Id")]
+        public string Id { get; set; }
+
+        [JsonProperty(PropertyName = "Type")]
+        public string Type { get; set; }
+
+        [JsonProperty(PropertyName = "Name")]
+        public string Name { get; set; }
+
+        [JsonProperty(PropertyName = "Content")]
+        public string Content { get; set; }
+
+        [JsonProperty(PropertyName = "Path")]
+        public string Path { get; set; }
+
+        [JsonProperty(PropertyName = "$metadata", Order = 1000)]
+        public IDictionary<string, string> Metadata => new Dictionary<string, string>
+        {
+            { "$type", "ReplayFile;" + v1.Version.NUMBER },
+            { "$uri", "/" + v1.Version.PATH + "/replayfile/" + this.Id },
+            { "$created", this.created.ToString(DATE_FORMAT) },
+            { "$modified", this.modified.ToString(DATE_FORMAT) }
+        };
+
+        public ReplayFileApiModel()
+        {
+            this.ETag = string.Empty;
+            this.Id = string.Empty;
+            this.Type = string.Empty;
+            this.Content = string.Empty;
+            this.Path = string.Empty;
+            this.Name = string.Empty;
+        }
+
+        // Map API model to service model
+        public Services.Models.DataFile ToServiceModel()
+        {
+            return new Services.Models.DataFile
+            {
+                ETag = this.ETag,
+                Id = this.Id,
+                Type = this.Type,
+                Path = (Services.Models.DataFile.FilePath)Enum.Parse(typeof(Services.Models.DataFile.FilePath), this.Path, true),
+                Content = this.Content,
+                Name = this.Name,
+            };
+        }
+
+        // Map service model to API model
+        public static ReplayFileApiModel FromServiceModel(Services.Models.DataFile value)
+        {
+            if (value == null) return null;
+
+            var result = new ReplayFileApiModel
+            {
+                ETag = value.ETag,
+                Id = value.Id,
+                Type = value.Type,
+                created = value.Created,
+                modified = value.Modified,
+                Path = value.Path.ToString(),
+                Content = value.Content,
+                Name = value.Name
+            };
+
+            return result;
+        }
+    }
+}