bugfix: use correct Device IDs (#274)

This commit is contained in:
Devis Lucato 2018-10-09 17:59:19 -07:00 коммит произвёл Harleen Thind
Родитель 8f66a67290
Коммит d1a986f1dd
19 изменённых файлов: 223 добавлений и 41 удалений

4
.gitignore поставляемый
Просмотреть файл

@ -2,6 +2,10 @@
### Custom
appsettings.local.ini
appsettings.development.ini
appsettings.dev.ini
appsettings.tmp.ini
_dev/
*.crt
*._gistcs

Просмотреть файл

@ -324,7 +324,7 @@ namespace Services.Test
{
// Arrange
this.mockStorageRecords.Setup(x => x.GetAsync(It.IsAny<string>()))
.ReturnsAsync((StorageRecord)null);
.ReturnsAsync((StorageRecord) null);
var sim = new SimulationModel
{
Id = "1",
@ -445,6 +445,11 @@ namespace Services.Test
new SimulationModel.DeviceModelRef { Id = modelId2, Count = 2 }
}
};
this.devices.Setup(x => x.GenerateId(simulationId, modelId1, 1)).Returns($"{simulationId}.{modelId1}.1");
this.devices.Setup(x => x.GenerateId(simulationId, modelId1, 2)).Returns($"{simulationId}.{modelId1}.2");
this.devices.Setup(x => x.GenerateId(simulationId, modelId1, 3)).Returns($"{simulationId}.{modelId1}.3");
this.devices.Setup(x => x.GenerateId(simulationId, modelId2, 1)).Returns($"{simulationId}.{modelId2}.1");
this.devices.Setup(x => x.GenerateId(simulationId, modelId2, 2)).Returns($"{simulationId}.{modelId2}.2");
// Act
Dictionary<string, List<string>> result = this.target.GetDeviceIdsByModel(sim);
@ -452,6 +457,9 @@ namespace Services.Test
// Assert
Assert.True(result.ContainsKey(modelId1));
Assert.True(result.ContainsKey(modelId2));
this.devices.Verify(
x => x.GenerateId(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<int>()),
Times.Exactly(sim.DeviceModels[0].Count + sim.DeviceModels[1].Count));
Assert.Contains($"{simulationId}.{modelId1}.1", result[modelId1]);
Assert.Contains($"{simulationId}.{modelId1}.2", result[modelId1]);
@ -514,6 +522,11 @@ namespace Services.Test
}
}
};
this.devices.Setup(x => x.GenerateId(simulationId, modelId1, 1)).Returns($"{simulationId}.{modelId1}.1");
this.devices.Setup(x => x.GenerateId(simulationId, modelId1, 2)).Returns($"{simulationId}.{modelId1}.2");
this.devices.Setup(x => x.GenerateId(simulationId, modelId1, 3)).Returns($"{simulationId}.{modelId1}.3");
this.devices.Setup(x => x.GenerateId(simulationId, modelId2, 1)).Returns($"{simulationId}.{modelId2}.1");
this.devices.Setup(x => x.GenerateId(simulationId, modelId2, 2)).Returns($"{simulationId}.{modelId2}.2");
// Act
Dictionary<string, List<string>> result = this.target.GetDeviceIdsByModel(sim);

Просмотреть файл

@ -87,7 +87,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Clustering
"The key to lock the master role doesn't exist yet, will create",
() => new { currentProcessNodeId, MASTER_NODE_KEY });
var record = new StorageRecord { Id = MASTER_NODE_KEY, Data = currentProcessNodeId };
var record = new StorageRecord { Id = MASTER_NODE_KEY, Data = "Record locked by the master node" };
await this.mainStorage.CreateAsync(record);
}

Просмотреть файл

@ -30,7 +30,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Clustering
public class ClusteringConfig : IClusteringConfig
{
private const int DEFAULT_CHECK_INTERVAL_MSECS = 15000;
private const int DEFAULT_CHECK_INTERVAL_MSECS = 10000;
private const int MIN_CHECK_INTERVAL_MSECS = 1000;
private const int MAX_CHECK_INTERVAL_MSECS = 300000;

Просмотреть файл

@ -61,7 +61,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services
/// <summary>
/// Generate a device Id
/// </summary>
string GenerateId(string deviceModelId, int position);
string GenerateId(string simulationId, string deviceModelId, int position);
/// <summary>
/// Create a list of devices using bulk import via storage account
@ -399,9 +399,9 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services
/// <summary>
/// Generate a device Id
/// </summary>
public string GenerateId(string deviceModelId, int position)
public string GenerateId(string simulationId, string deviceModelId, int position)
{
return deviceModelId + "." + position;
return simulationId + "." + deviceModelId + "." + position;
}
// Create a list of devices using bulk import via storage account

Просмотреть файл

@ -63,7 +63,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Models
[JsonProperty(Order = 30)]
public string Description { get; set; }
[JsonProperty(Order = 40)]
[JsonProperty(Order = 13)]
public bool PartitioningComplete { get; set; }
[JsonProperty(Order = 50)]

Просмотреть файл

@ -388,7 +388,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services
{
for (var i = 0; i < model.Count; i++)
{
deviceIds.Add(this.devices.GenerateId(model.Id, i));
deviceIds.Add(this.devices.GenerateId(simulation.Id, model.Id, i));
}
}
@ -409,7 +409,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services
var deviceIds = new List<string>();
for (var i = 1; i <= model.Count; i++)
{
deviceIds.Add(this.GenerateId(simulation.Id, model.Id, i));
deviceIds.Add(this.devices.GenerateId(simulation.Id, model.Id, i));
deviceCount++;
}
@ -466,11 +466,5 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services
return simulation;
}
// Generate a device Id
private string GenerateId(string simulationId, string deviceModelId, int position)
{
return simulationId + "." + deviceModelId + "." + position;
}
}
}

Просмотреть файл

@ -152,7 +152,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Storage.Documen
var client = new DocumentClient(docDbEndpoint, docDbKey);
await this.CreateDatabaseIfNotExistsAsync(client, docDbOptions, cfg.DocumentDbDatabase);
await this.EnsureCollectionExistsAsync(client, docDbOptions, cfg.DocumentDbDatabase, cfg.DocumentDbCollection);
await this.CreateCollectionIfNotExistsAsync(client, docDbOptions, cfg.DocumentDbDatabase, cfg.DocumentDbCollection);
return client;
}
@ -165,6 +165,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Storage.Documen
try
{
var uri = "/dbs/" + db;
this.log.Info("Checking if the database exists", () => new { uri });
await client.ReadDatabaseAsync(uri, options);
}
catch (DocumentClientException e) when (e.StatusCode == HttpStatusCode.NotFound)
@ -198,7 +199,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Storage.Documen
}
}
private async Task EnsureCollectionExistsAsync(
private async Task CreateCollectionIfNotExistsAsync(
IDocumentClient client,
RequestOptions options,
string dbName,
@ -207,11 +208,12 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Storage.Documen
try
{
var uri = $"/dbs/{dbName}/colls/{collName}";
this.log.Info("Checking if the collection exists", () => new { dbName, collName });
await client.ReadDocumentCollectionAsync(uri, options);
}
catch (DocumentClientException e) when (e.StatusCode == HttpStatusCode.NotFound)
{
await this.CreateCollectionIfNotExistsAsync(client, dbName, collName, options);
await this.CreateCollectionAsync(client, dbName, collName, options);
}
catch (Exception e)
{
@ -220,7 +222,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Storage.Documen
}
}
private async Task CreateCollectionIfNotExistsAsync(
private async Task CreateCollectionAsync(
IDocumentClient client,
string dbName,
string collName,

Просмотреть файл

@ -367,6 +367,7 @@ namespace SimulationAgent.Test
{
this.devices
.Setup(x => x.GenerateId(
It.IsAny<string>(),
It.IsAny<string>(),
It.IsAny<int>()))
.Returns("Simulate-01");

Просмотреть файл

@ -135,7 +135,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.SimulationAgent
if (this.running)
{
this.log.Info("Add device to running simulation");
await this.runner.AddDeviceAsync(deviceId, modelId);
await this.runner.AddDeviceAsync(this.simulation.Id, deviceId, modelId);
}
else
{

Просмотреть файл

@ -23,7 +23,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.SimulationAgent
{
Task StartAsync(Simulation simulation);
void Stop();
Task AddDeviceAsync(string deviceId, string modelId);
Task AddDeviceAsync(string simulationId, string deviceId, string modelId);
void DeleteDevices(List<string> ids);
long ActiveDevicesCount { get; }
long TotalMessagesCount { get; }
@ -180,9 +180,9 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.SimulationAgent
// Load device model and merge with overrides
var deviceModel = this.GetDeviceModel(model.Id, model.Override);
for (var i = 0; i < model.Count; i++)
for (var i = 1; i <= model.Count; i++)
{
await this.CreateActorsForDeviceAsync(deviceModel, i, total);
await this.CreateActorsForDeviceAsync(simulation.Id, deviceModel, i, total);
}
}
catch (ResourceNotFoundException)
@ -203,7 +203,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.SimulationAgent
foreach (var customDevice in simulation.CustomDevices)
{
await this.AddCustomDeviceAsync(customDevice.DeviceId, customDevice.DeviceModel.Id);
await this.AddCustomDeviceAsync(simulation.Id, customDevice.DeviceId, customDevice.DeviceModel.Id);
}
// Use `running` to avoid starting the simulation more than once
@ -266,9 +266,9 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.SimulationAgent
}
}
public async Task AddDeviceAsync(string deviceId, string modelId)
public async Task AddDeviceAsync(string simulationId, string deviceId, string modelId)
{
await this.AddCustomDeviceAsync(deviceId, modelId);
await this.AddCustomDeviceAsync(simulationId, deviceId, modelId);
}
/// <summary>
@ -503,9 +503,9 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.SimulationAgent
* one actor to manage the connection to the hub, and one actor for each
* telemetry message to send.
*/
private async Task CreateActorsForDeviceAsync(DeviceModel deviceModel, int position, int total, string deviceId = null)
private async Task CreateActorsForDeviceAsync(string simulationId, DeviceModel deviceModel, int position, int total, string deviceId = null)
{
var id = deviceId ?? this.devices.GenerateId(deviceModel.Id, position);
var id = deviceId ?? this.devices.GenerateId(simulationId, deviceModel.Id, position);
var key = deviceId ?? deviceModel.Id + "#" + position;
this.log.Debug("Creating device actors...",
@ -705,10 +705,10 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.SimulationAgent
Interlocked.Increment(ref this.simulationErrors);
}
private async Task AddCustomDeviceAsync(string deviceId, string modelId)
private async Task AddCustomDeviceAsync(string simulationId, string deviceId, string modelId)
{
DeviceModel model = this.GetDeviceModel(modelId, null);
await this.CreateActorsForDeviceAsync(model, 0, 1, deviceId);
await this.CreateActorsForDeviceAsync(simulationId, model, 0, 1, deviceId);
}
}
}

Просмотреть файл

@ -59,15 +59,15 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService
var assembly = Assembly.GetEntryAssembly();
builder.RegisterAssemblyTypes(assembly).AsImplementedInterfaces();
// Auto-wire Services.DLL
// Autowire Services.DLL
assembly = typeof(IServicesConfig).GetTypeInfo().Assembly;
builder.RegisterAssemblyTypes(assembly).AsImplementedInterfaces();
// Auto-wire SimulationAgent.DLL
// Autowire SimulationAgent.DLL
assembly = typeof(ISimulationAgent).GetTypeInfo().Assembly;
builder.RegisterAssemblyTypes(assembly).AsImplementedInterfaces();
// Auto-wire PartitioningAgent.DLL
// Autowire PartitioningAgent.DLL
assembly = typeof(IPartitioningAgent).GetTypeInfo().Assembly;
builder.RegisterAssemblyTypes(assembly).AsImplementedInterfaces();
}
@ -81,7 +81,13 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService
// More info about configuration at
// https://docs.microsoft.com/aspnet/core/fundamentals/configuration
var configurationBuilder = new ConfigurationBuilder();
configurationBuilder.AddIniFile("appsettings.ini", optional: true, reloadOnChange: false);
configurationBuilder
.AddIniFile(ConfigFile.DEFAULT, optional: false, reloadOnChange: false);
if (ConfigFile.GetDevOnlyConfigFile() != null)
{
configurationBuilder.AddIniFile(ConfigFile.GetDevOnlyConfigFile(), optional: true, reloadOnChange: true);
}
// Parse file and ensure the file is parsed only once
configuration = configurationBuilder.Build();

Просмотреть файл

@ -50,7 +50,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.Runtime
private const string DEVICE_MODELS_SCRIPTS_FOLDER_KEY = APPLICATION_KEY + "device_models_scripts_folder";
private const string IOTHUB_DATA_FOLDER_KEY = APPLICATION_KEY + "iothub_data_folder";
private const string IOTHUB_CONNSTRING_KEY = APPLICATION_KEY + "iothub_connstring";
private const string IOTHUB_IMPORT_STORAGE_CONNSTRING_KEY = APPLICATION_KEY + "iothub_import_storage_account";
private const string IOTHUB_IMPORT_STORAGE_CONNSTRING_KEY = APPLICATION_KEY + "iothub_import_storage_account_connstring";
private const string IOTHUB_SDK_DEVICE_CLIENT_TIMEOUT_KEY = APPLICATION_KEY + "iothub_sdk_device_client_timeout";
private const string TWIN_READ_WRITE_ENABLED_KEY = APPLICATION_KEY + "twin_read_write_enabled";

Просмотреть файл

@ -0,0 +1,37 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.IO;
namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.Runtime
{
public static class ConfigFile
{
public const string DEFAULT = "appsettings.ini";
private const string DEV_ENV_VAR = "PCS_DEV_APPSETTINGS";
// If the system has a "PCS_DEV_APPSETTINGS" environment variable, and the
// value references an existing file, then the app will use this file to pull in settings,
// before looking into "appsettings.ini". Use this mechanism to override the default
// configuration with your values during development. Do not use this approach
// in production. Also, make sure your dev config settings are not checked into
// the repository or the Docker image.
public static string GetDevOnlyConfigFile()
{
try
{
string devConfigFile = Environment.GetEnvironmentVariable(DEV_ENV_VAR);
if (!string.IsNullOrEmpty(devConfigFile) && File.Exists(devConfigFile))
{
return devConfigFile;
}
}
catch (Exception)
{
// no op
}
return null;
}
}
}

Просмотреть файл

@ -36,7 +36,13 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService
{
var builder = new ConfigurationBuilder()
.SetBasePath(env.ContentRootPath)
.AddIniFile("appsettings.ini", optional: false, reloadOnChange: true);
.AddIniFile(ConfigFile.DEFAULT, optional: false, reloadOnChange: true);
if (ConfigFile.GetDevOnlyConfigFile() != null)
{
builder.AddIniFile(ConfigFile.GetDevOnlyConfigFile(), optional: true, reloadOnChange: true);
}
this.Configuration = builder.Build();
}

Просмотреть файл

@ -1,8 +1,26 @@
#
# DEVELOPMENT NOTE:
#
# While developing the solution, you can override the settings below using a temporary/secondary file,
# so you don't need to modify the default configuration stored here.
#
# To do so, create an INI file, anywhere in your system, preferably outside the project to avoid
# the risk of checking the file into the repository. Name the file anything you like, e.g.
# "my-settings.ini", "simulation-dev-settings.ini" etc.
#
# Add to the file only the settings you need to override, i.e. you don't need to enter all the settings.
# Just make sure that each setting is under the right section, e.g. "[DeviceSimulationService:Logging]"
#
# Then, create an environment variable with name "PCS_DEV_APPSETTINGS" and put the complete file path in it.
# For instance "d:\temp\my_settings.ini" or "/tmp/dev-settings.ini".
#
[DeviceSimulationService]
# TCP port where to listen for web service requests. 9003 by default.
webservice_port = 9003
# Azure IoT Hub connection string, format: HostName=....azure-devices.net;SharedAccessKeyName=...;SharedAccessKey=...
# Azure IoT Hub connection string
# Format: HostName=_____.azure-devices.net;SharedAccessKeyName=_____;SharedAccessKey=_____
iothub_connstring = "${PCS_IOTHUB_CONNSTRING}"
# Timeout for the SDK client. By default the SDK uses 4 minutes (240000 msecs).
@ -23,8 +41,9 @@ iothub_data_folder = ./data/iothub/
# and open connections.
twin_read_write_enabled = "${?PCS_TWIN_READ_WRITE_ENABLED}"
# The Azure storage account used to create and delete devices in bulk
iothub_import_storage_account = "${PCS_AZURE_STORAGE_ACCOUNT}"
# Connection string for the Azure storage account used to create and delete devices in bulk
# Format: DefaultEndpointsProtocol=https;AccountName=_____;AccountKey=_____;EndpointSuffix=core.windows.net
iothub_import_storage_account_connstring = "${PCS_AZURE_STORAGE_ACCOUNT}"
[StorageAdapterService]
@ -279,8 +298,8 @@ min_device_properties_loop_duration = 2000
# - (as a master node) remove stale nodes from the cluster
# - (as a master node) check for new simulations and devices to create
# - (as a master node) check for new simulations and partitions to create
# Value in milliseconds - Default: 15000, Min: 1000, Max: 300000
check_interval = 15000
# Value in milliseconds - Default: 10000, Min: 1000, Max: 300000
check_interval = 10000
# When a node record is older than this value, the node is considered dead and removed
# from the list of known nodes. The value should be at least twice that of `check_interval`.

Просмотреть файл

@ -95,6 +95,11 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PartitioningAgent", "Partit
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PartitioningAgent.Test", "PartitioningAgent.Test\PartitioningAgent.Test.csproj", "{E1351585-2B99-44EA-8474-09C091652FE4}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "development", "development", "{D97D0D98-A424-4FBA-9E18-FE4EEC538893}"
ProjectSection(SolutionItems) = preProject
scripts\development\postman_collection.json = scripts\development\postman_collection.json
EndProjectSection
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -145,6 +150,7 @@ Global
{4C88469C-C551-435D-AE2C-36C58EE686A5} = {2B58D756-04D5-427F-9161-E1B45D35682A}
{39EA50D7-B4CD-41C4-85EA-E53E89E9BC98} = {DEF8CFF7-B332-4BA3-AB0B-8F7AC3275054}
{0680130C-7156-4118-BE65-4734BA92638B} = {71787873-D1A0-4D9E-BC84-942E0FB3841E}
{D97D0D98-A424-4FBA-9E18-FE4EEC538893} = {71787873-D1A0-4D9E-BC84-942E0FB3841E}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {25EBA0C1-8A8C-4116-948B-626F1638B7DA}

Просмотреть файл

@ -133,6 +133,11 @@ public void It$SOMENAME$()
// Assert
}</s:String>
<s:Boolean x:Key="/Default/UserDictionary/Words/=appsettings/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Autofac/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Autowire/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=autowired/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=autowiring/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=CONNSTRING/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=DOCUMENTDB/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=IOTHUB/@EntryIndexedValue">True</s:Boolean>

Просмотреть файл

@ -0,0 +1,89 @@
{
"info": {
"_postman_id": "4a75fbb6-cc7c-46a8-83d1-25f86dbd18db",
"name": "Azure IoT Simulation",
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
},
"item": [
{
"name": "Service status",
"request": {
"auth": {
"type": "noauth"
},
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "http://localhost:9003/v1/status",
"protocol": "http",
"host": [
"localhost"
],
"port": "9003",
"path": [
"v1",
"status"
]
}
},
"response": []
},
{
"name": "Add device to simulation",
"request": {
"auth": {
"type": "noauth"
},
"method": "POST",
"header": [],
"body": {},
"url": {
"raw": "http://localhost:9003/v1/simulations/1/Devices!create",
"protocol": "http",
"host": [
"localhost"
],
"port": "9003",
"path": [
"v1",
"simulations",
"1",
"Devices!create"
]
}
},
"response": []
},
{
"name": "Fetch simulation details",
"request": {
"auth": {
"type": "noauth"
},
"method": "GET",
"header": [
{
"key": "Content-Type",
"value": "application/json"
}
],
"body": {},
"url": {
"raw": "http://localhost:9003/v1/simulations/1",
"protocol": "http",
"host": [
"localhost"
],
"port": "9003",
"path": [
"v1",
"simulations",
"1"
]
}
},
"response": []
}
]
}