diff --git a/README.md b/README.md index 5d24c7c..54c564e 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,14 @@ docker run --rm -p 8080:80 --name wwtcoredata aasworldwidetelescope/core-data:la ``` However, most API endpoints won't work since they need to be wired up to the -backing data storage. +backing data storage. To test locally with WWT production assets, launch the Docker +container with the following environment variables: + +``` +-e UseAzurePlateFiles=true -e AzurePlateFileStorageAccount=[secret] +``` + +where the secret can be obtained from the running app's configuration KeyVault. See comments in the `azure-pipelines.yml` file for descriptions of how deployment works. diff --git a/src/WWT.PlateFiles/StreamSlice.cs b/src/WWT.PlateFiles/StreamSlice.cs index 6d94ef6..e949655 100644 --- a/src/WWT.PlateFiles/StreamSlice.cs +++ b/src/WWT.PlateFiles/StreamSlice.cs @@ -13,11 +13,11 @@ namespace WWT.PlateFiles /// public class StreamSlice : Stream { - private Stream _baseStream; - private long _position; - + private readonly long _offset; private readonly long _length; + private Stream _baseStream; + /// /// Attempts to create a . Any exception is wrapped in /// a . @@ -38,6 +38,7 @@ namespace WWT.PlateFiles { _baseStream = baseStream ?? throw new ArgumentNullException(nameof(baseStream)); _length = length; + _offset = offset; if (!baseStream.CanRead) { @@ -79,7 +80,7 @@ namespace WWT.PlateFiles private async ValueTask ReadInternalAsync(byte[] buffer, int offset, int count, bool isAsync, CancellationToken cancellationToken) { CheckDisposed(); - var remaining = _length - _position; + var remaining = _length - Position; if (remaining <= 0) { @@ -95,8 +96,6 @@ namespace WWT.PlateFiles ? await _baseStream.ReadAsync(buffer, offset, count, cancellationToken) : _baseStream.Read(buffer, offset, count); - _position += read; - return read; } @@ -140,7 +139,7 @@ namespace WWT.PlateFiles get { CheckDisposed(); - return false; + return _baseStream.CanSeek; } } @@ -149,12 +148,27 @@ namespace WWT.PlateFiles get { CheckDisposed(); - return _position; + return _baseStream.Position - _offset; } - set => throw new NotSupportedException(); + set => Seek(value, SeekOrigin.Begin); } - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) + { + CheckDisposed(); + + var basePosition = origin switch + { + SeekOrigin.Begin => offset + _offset, + SeekOrigin.Current => _baseStream.Position + offset, + SeekOrigin.End => _offset + _length - offset, + _ => throw new ArgumentOutOfRangeException(nameof(origin)), + }; + + _baseStream.Seek(basePosition, SeekOrigin.Begin); + + return Position; + } public override void SetLength(long value) => throw new NotSupportedException(); diff --git a/src/WWT.Providers/IResponse.cs b/src/WWT.Providers/IResponse.cs index a07b82b..42e7d4d 100644 --- a/src/WWT.Providers/IResponse.cs +++ b/src/WWT.Providers/IResponse.cs @@ -24,6 +24,8 @@ namespace WWT.Providers Stream OutputStream { get; } + Task ServeStreamAsync(Stream stream, string contentType, string etag); + void Close(); void ClearHeaders(); diff --git a/src/WWT.Providers/OtherProviders/CatalogProvider.cs b/src/WWT.Providers/OtherProviders/CatalogProvider.cs index d4aa6fd..4ec86cd 100644 --- a/src/WWT.Providers/OtherProviders/CatalogProvider.cs +++ b/src/WWT.Providers/OtherProviders/CatalogProvider.cs @@ -1,5 +1,3 @@ -#nullable disable - using Microsoft.Extensions.Logging; using System.IO; using System.Threading; @@ -28,77 +26,49 @@ namespace WWT.Providers public override async Task RunAsync(IWwtContext context, CancellationToken token) { - string etag = context.Request.Headers["If-None-Match"]; - string filename = ""; + string query = ""; + string extension = ""; if (context.Request.Params["Q"] != null) { - string query = context.Request.Params["Q"]; - - query = query.Replace("..", ""); - query = query.Replace("\\", ""); - query = query.Replace("/", ""); - filename = Path.Combine(query + ".txt"); + query = context.Request.Params["Q"]; + extension = "txt"; } else if (context.Request.Params["X"] != null) { - string query = context.Request.Params["X"]; - - query = query.Replace("..", ""); - query = query.Replace("\\", ""); - query = query.Replace("/", ""); - filename = $"{query}.xml"; + query = context.Request.Params["X"]; + extension = "xml"; } else if (context.Request.Params["W"] != null) { - string query = context.Request.Params["W"]; - - query = query.Replace("..", ""); - query = query.Replace("\\", ""); - query = query.Replace("/", ""); - filename = $"{query}.wtml"; - } - - if (!await GetEntry(context, etag, filename, token)) - { - context.Response.StatusCode = 404; - } - } - - private async Task GetEntry(IWwtContext context, string etag, string filename, CancellationToken token) - { - if (string.IsNullOrEmpty(filename)) - { - return false; - } - - var catalogEntry = await _catalog.GetCatalogEntryAsync(filename, token); - - if (catalogEntry is null) - { - _logger.LogWarning("Requested catalog {Name} does not exist.", filename); - return false; - } - - string newEtag = catalogEntry.LastModified.ToUniversalTime().ToString(); - - if (newEtag != etag) - { - context.Response.AddHeader("etag", newEtag); - - using (var c = catalogEntry.Contents) - { - await c.CopyToAsync(context.Response.OutputStream, token); - context.Response.Flush(); - context.Response.End(); - } + query = context.Request.Params["W"]; + extension = "wtml"; } else { - context.Response.StatusCode = 304; + await Report400Async(context, "must pass Q or X or W query parameter", token); + return; } - return true; + query = query.Replace("..", "").Replace("\\", "").Replace("/", ""); + string filename = $"{query}.{extension}"; + + var catalogEntry = await _catalog.GetCatalogEntryAsync(filename, token); + if (catalogEntry is null) + { + string msg = $"Requested catalog item {filename} does not exist."; + _logger.LogWarning(msg); + await Report404Async(context, msg, token); + return; + } + + string mtime = catalogEntry.LastModified.ToUniversalTime().ToString(); + string etag = $"\"{mtime}\""; + + using (var c = catalogEntry.Contents) + { + await context.Response.ServeStreamAsync(c, "text/plain", etag); + } } } } diff --git a/src/WWT.Providers/RequestProvider.cs b/src/WWT.Providers/RequestProvider.cs index 5852b59..15ed097 100644 --- a/src/WWT.Providers/RequestProvider.cs +++ b/src/WWT.Providers/RequestProvider.cs @@ -33,10 +33,22 @@ namespace WWT.Providers public const string Zip = "application/zip"; } - protected Task Report404Async(IWwtContext context, string detail, CancellationToken token) { + // Not found. + protected async Task Report404Async(IWwtContext context, string detail, CancellationToken token) { context.Response.StatusCode = 404; context.Response.ContentType = ContentTypes.Text; - return context.Response.WriteAsync($"HTTP/404 Not Found\n\n{detail}", token); + await context.Response.WriteAsync($"HTTP/404 Not Found\n\n{detail}", token); + context.Response.Flush(); + context.Response.End(); + } + + // Bad request. + protected async Task Report400Async(IWwtContext context, string detail, CancellationToken token) { + context.Response.StatusCode = 400; + context.Response.ContentType = ContentTypes.Text; + await context.Response.WriteAsync($"HTTP/400 Bad Request\n\n{detail}", token); + context.Response.Flush(); + context.Response.End(); } // This function is async because it handles the case of reporting an diff --git a/src/WWT.Web/AspNetCoreWwtContext.cs b/src/WWT.Web/AspNetCoreWwtContext.cs index c285fe2..cc38d4c 100644 --- a/src/WWT.Web/AspNetCoreWwtContext.cs +++ b/src/WWT.Web/AspNetCoreWwtContext.cs @@ -2,6 +2,12 @@ using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Http.Extensions; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.Abstractions; +using Microsoft.AspNetCore.Mvc.Infrastructure; +using Microsoft.AspNetCore.Routing; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Net.Http.Headers; using System; using System.IO; using System.Threading; @@ -13,19 +19,18 @@ namespace WWT.Web public class AspNetCoreWwtContext : IWwtContext, IRequest, IResponse, IHeaders, IParameters { private readonly HttpContext _ctx; - private readonly ICache _cache; public AspNetCoreWwtContext(HttpContext ctx, ICache cache) { _ctx = ctx; - _cache = cache; + Cache = cache; } string IParameters.this[string p] => _ctx.Request.Query[p]; string IHeaders.this[string p] => _ctx.Request.Headers[p]; - public ICache Cache => _cache; + public ICache Cache { get; } public IRequest Request => this; @@ -86,5 +91,19 @@ namespace WWT.Web Task IResponse.WriteAsync(string message, CancellationToken token) => _ctx.Response.WriteAsync(message, token); void IResponse.Redirect(string redirectUri) => _ctx.Response.Redirect(redirectUri); + + Task IResponse.ServeStreamAsync(Stream stream, string contentType, string etag) + { + var e = _ctx.RequestServices.GetRequiredService>(); + var route = _ctx.GetRouteData(); + var actionContext = new ActionContext(_ctx, route, new ActionDescriptor()); + + var result = new FileStreamResult(stream, contentType) { + EnableRangeProcessing = true, + EntityTag = EntityTagHeaderValue.Parse(etag), + }; + + return e.ExecuteAsync(actionContext, result); + } } } diff --git a/src/WWT.Web/Startup.cs b/src/WWT.Web/Startup.cs index c4f0339..59447c1 100644 --- a/src/WWT.Web/Startup.cs +++ b/src/WWT.Web/Startup.cs @@ -11,7 +11,6 @@ using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Net.Http.Headers; using System; - using WWT.Azure; using WWT.Caching; using WWT.Imaging; @@ -43,6 +42,8 @@ namespace WWT.Web }); }); + services.AddMvcCore(); + services.AddApplicationInsightsTelemetry(options => { options.InstrumentationKey = _configuration["APPINSIGHTS_INSTRUMENTATIONKEY"]; diff --git a/src/WWTMVC5/SystemWebWwtContext.cs b/src/WWTMVC5/SystemWebWwtContext.cs index f922918..1dfc167 100644 --- a/src/WWTMVC5/SystemWebWwtContext.cs +++ b/src/WWTMVC5/SystemWebWwtContext.cs @@ -83,5 +83,14 @@ namespace WWTMVC5 _context.Response.Write(message); return Task.CompletedTask; } + + Task IResponse.ServeStreamAsync(Stream stream, string contentType, string etag) + { + // No known reason that we can't implement this; but this API is + // mainly intended for the data services, not the WWTMVC5 app, and + // right now I'm not in a position to easily build and test an + // implementation here. + throw new NotImplementedException(); + } } }