diff --git a/LibraryStoreMigration/LibraryStoreMigration.csproj b/LibraryStoreMigration/LibraryStoreMigration.csproj new file mode 100644 index 0000000..4e84645 --- /dev/null +++ b/LibraryStoreMigration/LibraryStoreMigration.csproj @@ -0,0 +1,25 @@ + + + + Exe + netcoreapp2.1 + + + + + + + + + + + + + + + + + + + + diff --git a/LibraryStoreMigration/Program.cs b/LibraryStoreMigration/Program.cs new file mode 100644 index 0000000..0c2515c --- /dev/null +++ b/LibraryStoreMigration/Program.cs @@ -0,0 +1,135 @@ +using System; +using System.Reflection; +using System.IO; +using ServerlessLibrary.Models; +using System.Collections.Generic; +using Newtonsoft.Json; +using ServerlessLibrary; +using Microsoft.WindowsAzure.Storage.Table; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.RetryPolicies; +using System.Threading.Tasks; +using System.Linq; + +namespace LibraryStoreMigration +{ + class Program + { + const string tableName = "slitemstats"; + static void Main(string[] args) + { + if (args.Length < 1) + { + Console.WriteLine("Please enter specify operation to be performed (cosmosdb|stats)"); + Console.WriteLine("Please note that connection informations need to be provided as environment variables."); + return; + } + + if (args[0].Equals("cosmosdb", StringComparison.OrdinalIgnoreCase)) + { + MigrateToCosmosDB(); + } + + if (args[0].Equals("stats", StringComparison.OrdinalIgnoreCase)) + { + AddNewStatsColumns(); + } + } + + public static void AddNewStatsColumns() + { + TableRequestOptions tableRequestRetry = new TableRequestOptions { RetryPolicy = new LinearRetry(TimeSpan.FromSeconds(2), 3) }; + TableQuery query = new TableQuery(); + TableContinuationToken continuationToken = null; + List entities = new List(); + CloudStorageAccount storageAccount = CloudStorageAccount.Parse(ServerlessLibrarySettings.SLStorageString); + + CloudTableClient tableClient = storageAccount.CreateCloudTableClient(); + CloudTable table = tableClient.GetTableReference(tableName); + var opContext = new OperationContext(); + do + { + TableQuerySegment queryResults = (table).ExecuteQuerySegmentedAsync(query, continuationToken, tableRequestRetry, opContext).Result; + continuationToken = queryResults.ContinuationToken; + entities.AddRange(queryResults.Results); + } while (continuationToken != null); + + Console.WriteLine(entities.Count); + List libraryItems = GetAllLibraryItemsFromFile(); + foreach (var entity in entities) + { + entity.id = libraryItems.FirstOrDefault(l => l.Template == entity.template).Id; + entity.likes = 0; + entity.dislikes = 0; + TableOperation operation = TableOperation.InsertOrMerge(entity); + Task r = table.ExecuteAsync(operation); + TableResult a = r.Result; + } + } + + public static void MigrateToCosmosDB() + { + var libraryItems = GetAllLibraryItemsFromFile(); + Console.WriteLine("Number of samples to be migrated from file to cosmos db: {0}", libraryItems.Count); + CosmosLibraryStore libraryStore = new CosmosLibraryStore(); + + IList libraryItemsInCosmos = libraryStore.GetAllItems().Result; + Console.WriteLine("Number of samples already present in cosmos db: {0}", libraryItemsInCosmos.Count); + + if (libraryItemsInCosmos.Count != libraryItems.Count) + { + foreach (LibraryItem libraryItem in libraryItems) + { + if (!libraryItemsInCosmos.Any(c => c.Id == libraryItem.Id)) + { + Console.WriteLine("Item {0} not present in cosmos db. will be migrated" + libraryItem.Id); + try + { + libraryStore.Add(libraryItem).Wait(); + Console.WriteLine("Migrated sample with id {0}" + libraryItem.Id); + } + catch (Exception ex) + { + Console.WriteLine("Got exception {0}", ex); + throw; + } + } + } + + Console.WriteLine("Samples are successfully migrated to cosmos db"); + } + else + { + Console.WriteLine("Samples are already migrated to cosmos db"); + } + } + + public static List GetAllLibraryItemsFromFile() + { + var assembly = Assembly.GetExecutingAssembly(); + using (Stream stream = assembly.GetManifestResourceStream("LibraryStoreMigration.items.json")) + using (StreamReader reader = new StreamReader(stream)) + { + string result = reader.ReadToEnd(); + return JsonConvert.DeserializeObject>(result); + } + } + } + + public class OldSLItemStats : TableEntity + { + public string template { get; set; } + public int totalDownloads { get; set; } + public int downloadsToday { get; set; } + public int downloadsThisWeek { get; set; } + public int downloadsThisMonth { get; set; } + public DateTime lastUpdated { get; set; } + } + + public class NewSLItemStats : OldSLItemStats + { + public string id { get; set; } + public int likes { get; set; } + public int dislikes { get; set; } + } +} diff --git a/ServerlessLibraryAPI/CacheService.cs b/ServerlessLibraryAPI/CacheService.cs index 4f3d49f..535d935 100644 --- a/ServerlessLibraryAPI/CacheService.cs +++ b/ServerlessLibraryAPI/CacheService.cs @@ -73,34 +73,7 @@ namespace ServerlessLibrary { IList libraryItems; IList libraryItemsWithStats = new List(); - - if (cosmosDBInitialized) - { - libraryItems = await this.libraryStore.GetAllItems(); - } - else - { - libraryItems = await new FileLibraryStore(_env).GetAllItems(); - - if (!string.IsNullOrWhiteSpace(ServerlessLibrarySettings.CosmosEndpoint)) - { - IList libraryItemsInCosmos = await this.libraryStore.GetAllItems(); - if (libraryItemsInCosmos.Count == 0) - { - foreach (LibraryItem libraryItem in libraryItems) - { - this.libraryStore.Add(libraryItem); - } - } - else - { - libraryItems = libraryItemsInCosmos; - } - - cosmosDBInitialized = true; - } - } - + libraryItems = await this.libraryStore.GetAllItems(); var stats = await StorageHelper.getSLItemRecordsAsync(); foreach (var storeItem in libraryItems) { diff --git a/ServerlessLibraryAPI/Controllers/LibraryController.cs b/ServerlessLibraryAPI/Controllers/LibraryController.cs index 346a8a4..356c239 100644 --- a/ServerlessLibraryAPI/Controllers/LibraryController.cs +++ b/ServerlessLibraryAPI/Controllers/LibraryController.cs @@ -5,6 +5,7 @@ using Microsoft.AspNetCore.Mvc; using System.Text.RegularExpressions; using ServerlessLibrary.Models; using Newtonsoft.Json; +using System.Threading.Tasks; // For more information on enabling Web API for empty projects, visit https://go.microsoft.com/fwlink/?LinkID=397860 @@ -53,7 +54,7 @@ namespace ServerlessLibrary.Controllers [HttpPut] [ProducesResponseType(typeof(LibraryItem), 200)] - public IActionResult Put([FromBody]LibraryItem libraryItem) + public async Task Put([FromBody]LibraryItem libraryItem) { if (!User.Identity.IsAuthenticated) { @@ -74,7 +75,7 @@ namespace ServerlessLibrary.Controllers GitHubUser user = new GitHubUser(User); libraryItem.Author = user.UserName; - StorageHelper.submitContributionForApproval(JsonConvert.SerializeObject(libraryItem)); + await StorageHelper.submitContributionForApproval(JsonConvert.SerializeObject(libraryItem)); return new JsonResult(libraryItem); } diff --git a/ServerlessLibraryAPI/Controllers/MetricsController.cs b/ServerlessLibraryAPI/Controllers/MetricsController.cs index c7a9ae5..b0a681d 100644 --- a/ServerlessLibraryAPI/Controllers/MetricsController.cs +++ b/ServerlessLibraryAPI/Controllers/MetricsController.cs @@ -1,6 +1,8 @@ using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; using Newtonsoft.Json; using ServerlessLibrary.Models; +using System; namespace ServerlessLibrary.Controllers { @@ -8,12 +10,26 @@ namespace ServerlessLibrary.Controllers [ApiController] public class MetricsController : ControllerBase { + private ILogger logger; + public MetricsController(ILogger logger) + { + this.logger = logger; + } + // PUT api//downloads [ProducesResponseType(typeof(bool), 200)] [HttpPut] public JsonResult Downloads([FromBody]string id) { - StorageHelper.updateUserStats(JsonConvert.SerializeObject(new { id, userAction = "download" })); + try + { + StorageHelper.updateUserStats(JsonConvert.SerializeObject(new { id, userAction = "download" })).Wait(); + } + catch (Exception ex) + { + logger.LogError("Unable to update download count. Exception: {0}", ex.ToString()); + } + return new JsonResult(true); } // PUT api//sentiment @@ -21,20 +37,28 @@ namespace ServerlessLibrary.Controllers [HttpPut] public IActionResult Sentiment([FromBody]SentimentPayload sentimentPayload) { - if (sentimentPayload.LikeChanges < -1 - || sentimentPayload.LikeChanges > 1 + if (sentimentPayload.LikeChanges < -1 + || sentimentPayload.LikeChanges > 1 || sentimentPayload.LikeChanges == sentimentPayload.DislikeChanges) { return BadRequest("Invalid values for like or dislike count"); } - StorageHelper.updateUserStats(JsonConvert.SerializeObject(new - { - id = sentimentPayload.Id, - userAction = "Sentiment", - likeChanges = sentimentPayload.LikeChanges, - dislikeChanges = sentimentPayload.DislikeChanges - })); + try + { + StorageHelper.updateUserStats(JsonConvert.SerializeObject(new + { + id = sentimentPayload.Id, + userAction = "Sentiment", + likeChanges = sentimentPayload.LikeChanges, + dislikeChanges = sentimentPayload.DislikeChanges + })).Wait(); + } + catch (Exception ex) + { + logger.LogError("Unable to update Sentiments. Exception: {0}", ex.ToString()); + } + return new JsonResult(true); } } diff --git a/ServerlessLibraryAPI/CosmosLibraryStore.cs b/ServerlessLibraryAPI/CosmosLibraryStore.cs index da5216a..fde4609 100644 --- a/ServerlessLibraryAPI/CosmosLibraryStore.cs +++ b/ServerlessLibraryAPI/CosmosLibraryStore.cs @@ -17,13 +17,10 @@ namespace ServerlessLibrary { public CosmosLibraryStore() { - if (!string.IsNullOrWhiteSpace(ServerlessLibrarySettings.CosmosEndpoint)) - { - CosmosDBRepository.Initialize(); - } + CosmosDBRepository.Initialize(); } - async public void Add(LibraryItem libraryItem) + public async Task Add(LibraryItem libraryItem) { await CosmosDBRepository.CreateItemAsync(libraryItem); } @@ -34,8 +31,7 @@ namespace ServerlessLibrary return libraryItems.ToList(); } } - - + /// /// Cosmos db APIs /// diff --git a/ServerlessLibraryAPI/FileLibraryStore.cs b/ServerlessLibraryAPI/FileLibraryStore.cs index 36d044e..0ee8441 100644 --- a/ServerlessLibraryAPI/FileLibraryStore.cs +++ b/ServerlessLibraryAPI/FileLibraryStore.cs @@ -20,7 +20,7 @@ namespace ServerlessLibrary _env = env; } - public void Add(LibraryItem libraryItem) + public Task Add(LibraryItem libraryItem) { throw new NotImplementedException(); } diff --git a/ServerlessLibraryAPI/ILibraryStore.cs b/ServerlessLibraryAPI/ILibraryStore.cs index d2be4a7..04f6a5e 100644 --- a/ServerlessLibraryAPI/ILibraryStore.cs +++ b/ServerlessLibraryAPI/ILibraryStore.cs @@ -13,7 +13,7 @@ namespace ServerlessLibrary /// Add an item to library /// /// Library item - void Add(LibraryItem libraryItem); + Task Add(LibraryItem libraryItem); /// /// Get all items from library diff --git a/ServerlessLibraryAPI/ServerlessLibrarySettings.cs b/ServerlessLibraryAPI/ServerlessLibrarySettings.cs index 34ebb8e..6d8ea20 100644 --- a/ServerlessLibraryAPI/ServerlessLibrarySettings.cs +++ b/ServerlessLibraryAPI/ServerlessLibrarySettings.cs @@ -25,7 +25,6 @@ namespace ServerlessLibrary public static string CosmosAuthkey { get { return config(); } } public static string Database { get { return "serverlesslibrary"; } } public static string Collection { get { return "contributions"; } } - public static bool ApiOnly { get { return Boolean.Parse(config("false")); } } } } diff --git a/ServerlessLibraryAPI/Startup.cs b/ServerlessLibraryAPI/Startup.cs index 88cf3ef..f9b92ed 100644 --- a/ServerlessLibraryAPI/Startup.cs +++ b/ServerlessLibraryAPI/Startup.cs @@ -1,12 +1,15 @@ using Microsoft.AspNetCore.Authentication.Cookies; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.SpaServices.ReactDevelopmentServer; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using ServerlessLibrary.OAuth.GitHub; using Swashbuckle.AspNetCore.Swagger; +using System.Threading.Tasks; namespace ServerlessLibrary { @@ -14,7 +17,7 @@ namespace ServerlessLibrary { public Startup(IConfiguration configuration) { - Configuration = configuration; + Configuration = configuration; } public IConfiguration Configuration { get; } @@ -62,13 +65,10 @@ namespace ServerlessLibrary // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. public void Configure(IApplicationBuilder app, IHostingEnvironment env) { - app.UseExceptionHandler("/Error"); app.UseHsts(); - app.UseDefaultFiles(); app.UseStaticFiles(); app.UseSpaStaticFiles(); - app.UseSwaggerUI(c => { c.SwaggerEndpoint("/swagger/v1/swagger.json", "Serverless library API v1"); @@ -84,18 +84,15 @@ namespace ServerlessLibrary template: "{controller}/{action=Index}/{id?}"); }); - if (!ServerlessLibrarySettings.ApiOnly) + app.UseSpa(spa => { - app.UseSpa(spa => - { - spa.Options.SourcePath = "ClientApp"; + spa.Options.SourcePath = "ClientApp"; - if (env.IsDevelopment()) - { - spa.UseReactDevelopmentServer(npmScript: "start"); - } - }); - } + if (env.IsDevelopment()) + { + spa.UseReactDevelopmentServer(npmScript: "start"); + } + }); app.Use(async (context, next) => { diff --git a/ServerlessLibraryAPI/StorageHelper.cs b/ServerlessLibraryAPI/StorageHelper.cs index 7c2825a..588fb8b 100644 --- a/ServerlessLibraryAPI/StorageHelper.cs +++ b/ServerlessLibraryAPI/StorageHelper.cs @@ -52,13 +52,13 @@ namespace ServerlessLibrary return queue; } - public static async void submitContributionForApproval(string contributionPayload) + public static async Task submitContributionForApproval(string contributionPayload) { var message = new CloudQueueMessage(contributionPayload); await (await getQueueReference(slContributionRequests)).AddMessageAsync(message); } - public static async void updateUserStats(string statsPayload) + public static async Task updateUserStats(string statsPayload) { var message = new CloudQueueMessage(statsPayload); await (await getQueueReference(slItemTableName)).AddMessageAsync(message);