Merge pull request #280 from pkgw/ranged-support

Add support for byte-range requests in catalog.aspx endpoint

README.md

@@ -28,7 +28,14 @@ docker run --rm -p 8080:80 --name wwtcoredata aasworldwidetelescope/core-data:la
 ```
 
 However, most API endpoints won't work since they need to be wired up to the
-backing data storage.
+backing data storage. To test locally with WWT production assets, launch the Docker
+container with the following environment variables:
+
+```
+-e UseAzurePlateFiles=true -e AzurePlateFileStorageAccount=[secret]
+```
+
+where the secret can be obtained from the running app's configuration KeyVault.
 
 See comments in the `azure-pipelines.yml` file for descriptions of how deployment
 works.

src/WWT.PlateFiles/StreamSlice.cs

@@ -13,11 +13,11 @@ namespace WWT.PlateFiles
     /// </summary>
     public class StreamSlice : Stream
     {
-        private Stream _baseStream;
-        private long _position;
-
+        private readonly long _offset;
         private readonly long _length;
 
+        private Stream _baseStream;
+
         /// <summary>
         /// Attempts to create a <see cref="StreamSlice"/>. Any exception is wrapped in
         /// a <see cref=" PlateTileException"/>.
@@ -38,6 +38,7 @@ namespace WWT.PlateFiles
         {
             _baseStream = baseStream ?? throw new ArgumentNullException(nameof(baseStream));
             _length = length;
+            _offset = offset;
 
             if (!baseStream.CanRead)
             {
@@ -79,7 +80,7 @@ namespace WWT.PlateFiles
         private async ValueTask<int> ReadInternalAsync(byte[] buffer, int offset, int count, bool isAsync, CancellationToken cancellationToken)
         {
             CheckDisposed();
-            var remaining = _length - _position;
+            var remaining = _length - Position;
 
             if (remaining <= 0)
             {
@@ -95,8 +96,6 @@ namespace WWT.PlateFiles
                 ? await _baseStream.ReadAsync(buffer, offset, count, cancellationToken)
                 : _baseStream.Read(buffer, offset, count);
 
-            _position += read;
-
             return read;
         }
 
@@ -140,7 +139,7 @@ namespace WWT.PlateFiles
             get
             {
                 CheckDisposed();
-                return false;
+                return _baseStream.CanSeek;
             }
         }
 
@@ -149,12 +148,27 @@ namespace WWT.PlateFiles
             get
             {
                 CheckDisposed();
-                return _position;
+                return _baseStream.Position - _offset;
             }
-            set => throw new NotSupportedException();
+            set => Seek(value, SeekOrigin.Begin);
         }
 
-        public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
+        public override long Seek(long offset, SeekOrigin origin)
+        {
+            CheckDisposed();
+
+            var basePosition = origin switch
+            {
+                SeekOrigin.Begin => offset + _offset,
+                SeekOrigin.Current => _baseStream.Position + offset,
+                SeekOrigin.End => _offset + _length - offset,
+                _ => throw new ArgumentOutOfRangeException(nameof(origin)),
+            };
+
+            _baseStream.Seek(basePosition, SeekOrigin.Begin);
+
+            return Position;
+        }
 
         public override void SetLength(long value) => throw new NotSupportedException();
 

src/WWT.Providers/IResponse.cs

@@ -24,6 +24,8 @@ namespace WWT.Providers
 
         Stream OutputStream { get; }
 
+        Task ServeStreamAsync(Stream stream, string contentType, string etag);
+
         void Close();
 
         void ClearHeaders();

src/WWT.Providers/OtherProviders/CatalogProvider.cs

@@ -1,5 +1,3 @@
-#nullable disable
-
 using Microsoft.Extensions.Logging;
 using System.IO;
 using System.Threading;
@@ -28,77 +26,49 @@ namespace WWT.Providers
 
         public override async Task RunAsync(IWwtContext context, CancellationToken token)
         {
-            string etag = context.Request.Headers["If-None-Match"];
-            string filename = "";
+            string query = "";
+            string extension = "";
 
             if (context.Request.Params["Q"] != null)
             {
-                string query = context.Request.Params["Q"];
-
-                query = query.Replace("..", "");
-                query = query.Replace("\\", "");
-                query = query.Replace("/", "");
-                filename = Path.Combine(query + ".txt");
+                query = context.Request.Params["Q"];
+                extension = "txt";
             }
             else if (context.Request.Params["X"] != null)
             {
-                string query = context.Request.Params["X"];
-
-                query = query.Replace("..", "");
-                query = query.Replace("\\", "");
-                query = query.Replace("/", "");
-                filename = $"{query}.xml";
+                query = context.Request.Params["X"];
+                extension = "xml";
             }
             else if (context.Request.Params["W"] != null)
             {
-                string query = context.Request.Params["W"];
-
-                query = query.Replace("..", "");
-                query = query.Replace("\\", "");
-                query = query.Replace("/", "");
-                filename = $"{query}.wtml";
-            }
-
-            if (!await GetEntry(context, etag, filename, token))
-            {
-                context.Response.StatusCode = 404;
-            }
-        }
-
-        private async Task<bool> GetEntry(IWwtContext context, string etag, string filename, CancellationToken token)
-        {
-            if (string.IsNullOrEmpty(filename))
-            {
-                return false;
-            }
-
-            var catalogEntry = await _catalog.GetCatalogEntryAsync(filename, token);
-
-            if (catalogEntry is null)
-            {
-                _logger.LogWarning("Requested catalog {Name} does not exist.", filename);
-                return false;
-            }
-
-            string newEtag = catalogEntry.LastModified.ToUniversalTime().ToString();
-
-            if (newEtag != etag)
-            {
-                context.Response.AddHeader("etag", newEtag);
-
-                using (var c = catalogEntry.Contents)
-                {
-                    await c.CopyToAsync(context.Response.OutputStream, token);
-                    context.Response.Flush();
-                    context.Response.End();
-                }
+                query = context.Request.Params["W"];
+                extension = "wtml";
             }
             else
             {
-                context.Response.StatusCode = 304;
+                await Report400Async(context, "must pass Q or X or W query parameter", token);
+                return;
             }
 
-            return true;
+            query = query.Replace("..", "").Replace("\\", "").Replace("/", "");
+            string filename = $"{query}.{extension}";
+
+            var catalogEntry = await _catalog.GetCatalogEntryAsync(filename, token);
+            if (catalogEntry is null)
+            {
+                string msg = $"Requested catalog item {filename} does not exist.";
+                _logger.LogWarning(msg);
+                await Report404Async(context, msg, token);
+                return;
+            }
+
+            string mtime = catalogEntry.LastModified.ToUniversalTime().ToString();
+            string etag = $"\"{mtime}\"";
+
+            using (var c = catalogEntry.Contents)
+            {
+                await context.Response.ServeStreamAsync(c, "text/plain", etag);
+            }
         }
     }
 }

src/WWT.Providers/RequestProvider.cs

@@ -33,10 +33,22 @@ namespace WWT.Providers
             public const string Zip = "application/zip";
         }
 
-        protected Task Report404Async(IWwtContext context, string detail, CancellationToken token) {
+        // Not found.
+        protected async Task Report404Async(IWwtContext context, string detail, CancellationToken token) {
             context.Response.StatusCode = 404;
             context.Response.ContentType = ContentTypes.Text;
-            return context.Response.WriteAsync($"HTTP/404 Not Found\n\n{detail}", token);
+            await context.Response.WriteAsync($"HTTP/404 Not Found\n\n{detail}", token);
+            context.Response.Flush();
+            context.Response.End();
+        }
+
+        // Bad request.
+        protected async Task Report400Async(IWwtContext context, string detail, CancellationToken token) {
+            context.Response.StatusCode = 400;
+            context.Response.ContentType = ContentTypes.Text;
+            await context.Response.WriteAsync($"HTTP/400 Bad Request\n\n{detail}", token);
+            context.Response.Flush();
+            context.Response.End();
         }
 
         // This function is async because it handles the case of reporting an

src/WWT.Web/AspNetCoreWwtContext.cs

@@ -2,6 +2,12 @@
 
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Http.Extensions;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Mvc.Abstractions;
+using Microsoft.AspNetCore.Mvc.Infrastructure;
+using Microsoft.AspNetCore.Routing;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Net.Http.Headers;
 using System;
 using System.IO;
 using System.Threading;
@@ -13,19 +19,18 @@ namespace WWT.Web
     public class AspNetCoreWwtContext : IWwtContext, IRequest, IResponse, IHeaders, IParameters
     {
         private readonly HttpContext _ctx;
-        private readonly ICache _cache;
 
         public AspNetCoreWwtContext(HttpContext ctx, ICache cache)
         {
             _ctx = ctx;
-            _cache = cache;
+            Cache = cache;
         }
 
         string IParameters.this[string p] => _ctx.Request.Query[p];
 
         string IHeaders.this[string p] => _ctx.Request.Headers[p];
 
-        public ICache Cache => _cache;
+        public ICache Cache { get; }
 
         public IRequest Request => this;
 
@@ -86,5 +91,19 @@ namespace WWT.Web
         Task IResponse.WriteAsync(string message, CancellationToken token) => _ctx.Response.WriteAsync(message, token);
 
         void IResponse.Redirect(string redirectUri) => _ctx.Response.Redirect(redirectUri);
+
+        Task IResponse.ServeStreamAsync(Stream stream, string contentType, string etag)
+        {
+            var e = _ctx.RequestServices.GetRequiredService<IActionResultExecutor<FileStreamResult>>();
+            var route = _ctx.GetRouteData();
+            var actionContext = new ActionContext(_ctx, route, new ActionDescriptor());
+
+            var result = new FileStreamResult(stream, contentType) {
+                EnableRangeProcessing = true,
+                EntityTag = EntityTagHeaderValue.Parse(etag),
+            };
+
+            return e.ExecuteAsync(actionContext, result);
+        }
     }
 }

src/WWT.Web/Startup.cs

@@ -11,7 +11,6 @@ using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Net.Http.Headers;
 using System;
-
 using WWT.Azure;
 using WWT.Caching;
 using WWT.Imaging;
@@ -43,6 +42,8 @@ namespace WWT.Web
                 });
             });
 
+            services.AddMvcCore();
+
             services.AddApplicationInsightsTelemetry(options =>
             {
                 options.InstrumentationKey = _configuration["APPINSIGHTS_INSTRUMENTATIONKEY"];

src/WWTMVC5/SystemWebWwtContext.cs

@@ -83,5 +83,14 @@ namespace WWTMVC5
             _context.Response.Write(message);
             return Task.CompletedTask;
         }
+
+        Task IResponse.ServeStreamAsync(Stream stream, string contentType, string etag)
+        {
+            // No known reason that we can't implement this; but this API is
+            // mainly intended for the data services, not the WWTMVC5 app, and
+            // right now I'm not in a position to easily build and test an
+            // implementation here.
+            throw new NotImplementedException();
+        }
     }
 }