Tylake/configuration (#51)

* Adding a configurationsource for use in ASP

* Adding dependencies

* Fixing name

* Modifying for new settings and optional SF

* Added Configuration parameter to StartUpUtil, not being used yet

* Moved service fabric configuration to servicehost; added settings constants; modified startuputil for DataXSettings

* Reducing surface area of SF config calls

* Adding SF to config builder

* Correcting errorneous default value

* Runs and apis are hittable

* Removing dev strings

* Adding gitignore entry

* Modifying dev default auth connection string

* Modifying launch api to simple get call

* Switching connection string

* Adding onebox

* Uppdating dev settings

* Modified the sf configuration source to better conform to customizable settings; began work on local scenario

* Using constant instead of literal

* Enabling more of the local scenario

* Modified mef extension to generate ILoggers for classes automatically

* Updating settings for onebox scenario

* Adding authentication

* Begin adding auth attribute; Not working yet

* Adding auth scenario; needs cleaning

* Disable RequireAuthenticatedUser if in OneBox mode

* Revert "Disable RequireAuthenticatedUser if in OneBox mode"

This reverts commit 547f49b0c6.

* Removing authentication requirement from OneBox scenario

* Temporary: modified for rerouteService scenario

* Minor code improvements

* Removing unnecessary project dependency

* Added a gateway policy for compatibility in SF auth scenario; Moved some auth classes to be better accessed; RoleCheck changed to consider SF environment

* Cleanup

* Reverting auto changes

* Renaming appinsights setting to name in SF

* Modified gateway policy for fix in SF

* Updating SF xmls with EnableOneBox

* Added varying documentation by request

* Adding onebox to settings

* Renaming based on feedback

DataProcessing/DataX.Utilities/.gitignore

@@ -0,0 +1 @@
+*/obj

Services/.gitignore

@@ -128,4 +128,7 @@ BundleArtifacts/
 /DeploymentCloud/Deployment.DataX/cachedVariables
 /DeploymentCloud/Deployment.DataX/Temp
 /DeploymentCloud/Deployment.DataX/Outputs
-/DeploymentCloud/Deployment.DataX/Packages
+/DeploymentCloud/Deployment.DataX/Packages
+
+# Development settings
+appsettings.Development.json

Services/AspnetCore/DataX.FlowManagement/DataX.FlowManagement/Startup.cs

@@ -27,8 +27,9 @@ namespace DataX.FlowManagement
     {
 
         private const string _MetricsHttpEndpointRelativeUri = "/api/data/upload";
+        private readonly ILoggerFactory _loggerFactory;
 
-        public Startup(IHostingEnvironment env)
+        public Startup(IHostingEnvironment env, ILoggerFactory loggerFactory)
         {
             var builder = new ConfigurationBuilder()
                 .SetBasePath(env.ContentRootPath)
@@ -37,6 +38,8 @@ namespace DataX.FlowManagement
                 .AddEnvironmentVariables();
 
             Configuration = builder.Build();
+
+            _loggerFactory = loggerFactory;
         }
 
         public IConfiguration Configuration { get; }
@@ -51,7 +54,7 @@ namespace DataX.FlowManagement
 
             // Export the Config dependencies
             Type[] exportTypes = new Type[] { typeof(FlowOperation), typeof(RuntimeConfigGeneration), typeof(JobOperation) };
-            services.AddMefExportsFromAssemblies(ServiceLifetime.Scoped, GetOneBoxModeDependencyAssemblies(), exportTypes, new object[] { });
+            services.AddMefExportsFromAssemblies(ServiceLifetime.Scoped, GetOneBoxModeDependencyAssemblies(), exportTypes, null, _loggerFactory, true);
             var result = InitTemplatesForLocal(services);
             Ensure.IsSuccessResult(result);
         }

Services/DataX.Config/DataX.Config.Local/LocalSparkClientFactory.cs

@@ -17,7 +17,7 @@ namespace DataX.Config.Local
         private ILogger _logger { get; }
 
         [ImportingConstructor]
-        public LocalSparkClientFactory(ILogger logger)
+        public LocalSparkClientFactory(ILogger<LocalSparkClientFactory> logger)
         {
             this._logger = logger;
         }

Services/DataX.Config/DataX.Config/InternalService/SparkJobOperation.cs

@@ -21,7 +21,7 @@ namespace DataX.Config
     public class SparkJobOperation
     {
         [ImportingConstructor]
-        public SparkJobOperation(SparkJobData jobData, SparkClusterManager clusterManager, ILogger logger)
+        public SparkJobOperation(SparkJobData jobData, SparkClusterManager clusterManager, ILogger<SparkJobOperation> logger)
         {
             this.JobData= jobData;
             this.ClusterManager = clusterManager;

Services/DataX.Config/DataX.Config/PublicService/RuntimeConfigGeneration.cs

@@ -25,7 +25,7 @@ namespace DataX.Config
         /// </summary>
         [ImportingConstructor]
         public RuntimeConfigGeneration(
-            ILogger logger,
+            ILogger<RuntimeConfigGeneration> logger,
             FlowDataManager flows,
             JobDataManager jobData,
             [ImportMany] IEnumerable<IFlowDeploymentProcessor> flowProcessors

Services/DataX.Config/DataX.Config/Utility/LoggerExportDescriptorProvider.cs

@@ -0,0 +1,76 @@
+// *********************************************************************
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+// Licensed under the MIT License
+// *********************************************************************
+using Microsoft.Extensions.Logging;
+using System;
+using System.Collections.Generic;
+using System.Composition.Hosting.Core;
+using System.Linq;
+using System.Reflection;
+
+namespace DataX.Config.Utility
+{
+    /// <summary>
+    /// Using a LoggerFactory, dynamically creates Logger<T> to resolve for MEF
+    /// Falls back to InstanceExportDescriptorProvider implementation if not a Logger
+    /// </summary>
+    public class LoggerAndInstanceExportDescriptorProvider : InstanceExportDescriptorProvider
+    {
+        private static readonly Type _ILoggerType = typeof(ILogger);
+
+        private static readonly MethodInfo _CreateLogger =
+            typeof(Microsoft.Extensions.Logging.LoggerFactoryExtensions)
+            .GetMethods()
+            .Where(m => m.Name == nameof(LoggerFactoryExtensions.CreateLogger) && m.IsGenericMethod)
+            .FirstOrDefault();
+
+        private readonly ILoggerFactory _loggerFactory;
+        private readonly bool _hasInstances;
+
+        public LoggerAndInstanceExportDescriptorProvider(object[] instances, ILoggerFactory loggerFactory)
+            : base(instances)
+        {
+            _hasInstances = instances?.Length > 0;
+            _loggerFactory = loggerFactory;
+        }
+
+        ///<inheritdoc />
+        public override IEnumerable<ExportDescriptorPromise> GetExportDescriptors(CompositionContract contract, DependencyAccessor descriptorAccessor)
+        {
+            if (_loggerFactory != null && _ILoggerType.IsAssignableFrom(contract.ContractType))
+            {
+                ILogger logger;
+
+                if (contract.ContractType.GenericTypeArguments.Length > 0)
+                {
+                    logger = CreateLogger(contract.ContractType.GenericTypeArguments.FirstOrDefault());
+                }
+                else
+                {
+                    logger = _loggerFactory.CreateLogger(contract.ContractType);
+                }
+
+                yield return new ExportDescriptorPromise(
+                        contract,
+                        contract.ContractType.FullName,
+                        true,
+                        NoDependencies,
+                        dependencies => ExportDescriptor.Create((context, operation) => logger, NoMetadata));
+            }
+            else if(_hasInstances)
+            {
+                foreach (var descriptor in base.GetExportDescriptors(contract, descriptorAccessor))
+                {
+                    yield return descriptor;
+                }
+            }
+        }
+
+        private ILogger CreateLogger(Type t)
+        {
+            var genericMethod = _CreateLogger.MakeGenericMethod(t);
+            return genericMethod.Invoke(null, new object[] { _loggerFactory }) as ILogger;
+        }
+    }
+}

Services/DataX.Flow/DataX.Flow/ApplicationPackageRoot/ApplicationManifest.xml

@@ -22,6 +22,7 @@ Licensed under the MIT License
     <Parameter Name="MefStorageAccountName" DefaultValue="" />
     <Parameter Name="MefContainerName" DefaultValue="" />
     <Parameter Name="MefBlobDirectory" DefaultValue="" />
+    <Parameter Name="EnableOneBox" DefaultValue="false" />
   </Parameters>
   <!-- Import the ServiceManifest from the ServicePackage. The ServiceManifestName and ServiceManifestVersion 
        should match the Name and Version attributes of the ServiceManifest element defined in the 
@@ -37,6 +38,7 @@ Licensed under the MIT License
             <Parameter Name="cosmosDBConfigDatabaseName" Value="[cosmosDBConfigDatabaseName]" />
             <Parameter Name="cosmosDBConfigCollectionName" Value="[cosmosDBConfigCollectionName]" />
             <Parameter Name="AppInsightsIntrumentationKey" Value="[AppInsightsIntrumentationKey]" />
+            <Parameter Name="EnableOneBox" Value="[EnableOneBox]" />
           </Section>
         </Settings>
       </ConfigOverride>
@@ -57,6 +59,7 @@ Licensed under the MIT License
             <Parameter Name="cosmosDBConfigDatabaseName" Value="[cosmosDBConfigDatabaseName]" />
             <Parameter Name="cosmosDBConfigCollectionName" Value="[cosmosDBConfigCollectionName]" />
             <Parameter Name="AppInsightsIntrumentationKey" Value="[AppInsightsIntrumentationKey]" />
+            <Parameter Name="EnableOneBox" Value="[EnableOneBox]" />
           </Section>
         </Settings>
       </ConfigOverride>
@@ -77,6 +80,7 @@ Licensed under the MIT License
             <Parameter Name="cosmosDBConfigDatabaseName" Value="[cosmosDBConfigDatabaseName]" />
             <Parameter Name="cosmosDBConfigCollectionName" Value="[cosmosDBConfigCollectionName]" />
             <Parameter Name="AppInsightsIntrumentationKey" Value="[AppInsightsIntrumentationKey]" />
+            <Parameter Name="EnableOneBox" Value="[EnableOneBox]" />
           </Section>
         </Settings>
       </ConfigOverride>
@@ -100,6 +104,7 @@ Licensed under the MIT License
             <Parameter Name="MefStorageAccountName" Value="[MefStorageAccountName]" />
             <Parameter Name="MefContainerName" Value="[MefContainerName]" />
             <Parameter Name="MefBlobDirectory" Value="[MefBlobDirectory]" />
+            <Parameter Name="EnableOneBox" Value="[EnableOneBox]" />
           </Section>
         </Settings>
       </ConfigOverride>

Services/DataX.Flow/DataX.Flow/ApplicationParameters/Cloud.xml

@@ -18,5 +18,6 @@ Licensed under the MIT License
     <Parameter Name="MefStorageAccountName" Value="" />
     <Parameter Name="MefContainerName" Value="" />
     <Parameter Name="MefBlobDirectory" Value="" />
+    <Parameter Name="EnableOneBox" Value="" />
   </Parameters>
 </Application>

Services/DataX.Flow/DataX.Flow/ApplicationParameters/Local.1Node.xml

@@ -22,5 +22,6 @@ Licensed under the MIT License
     <Parameter Name="MefStorageAccountName" Value="" />
     <Parameter Name="MefContainerName" Value="" />
     <Parameter Name="MefBlobDirectory" Value="" />
+    <Parameter Name="EnableOneBox" Value="" />
   </Parameters>
 </Application>

Services/DataX.Flow/DataX.Flow/ApplicationParameters/Local.5Node.xml

@@ -22,5 +22,6 @@ Licensed under the MIT License
     <Parameter Name="MefStorageAccountName" Value="" />
     <Parameter Name="MefContainerName" Value="" />
     <Parameter Name="MefBlobDirectory" Value="" />
+    <Parameter Name="EnableOneBox" Value="" />
   </Parameters>
 </Application>

Services/DataX.Flow/DataX.Flow/DataX.Flow.sfproj

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" InitialTargets=";ValidateMSBuildFiles">
-  <Import Project="..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.props" Condition="Exists('..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.props')" />
+  <Import Project="..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.props" Condition="Exists('..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.props')" />
   <PropertyGroup Label="Globals">
     <ProjectGuid>34e85436-8f73-44fc-9c03-ca205077d68d</ProjectGuid>
     <ProjectVersion>2.3</ProjectVersion>
@@ -42,9 +42,9 @@
     <ApplicationProjectTargetsPath>$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\Service Fabric Tools\Microsoft.VisualStudio.Azure.Fabric.ApplicationProject.targets</ApplicationProjectTargetsPath>
   </PropertyGroup>
   <Import Project="$(ApplicationProjectTargetsPath)" Condition="Exists('$(ApplicationProjectTargetsPath)')" />
-  <Import Project="..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets" Condition="Exists('..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets')" />
-  <Target Name="ValidateMSBuildFiles">
-    <Error Condition="!Exists('..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.props')" Text="Unable to find the '..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.props' file. Please restore the 'Microsoft.VisualStudio.Azure.Fabric.MSBuild' Nuget package." />
-    <Error Condition="!Exists('..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets')" Text="Unable to find the '..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets' file. Please restore the 'Microsoft.VisualStudio.Azure.Fabric.MSBuild' Nuget package." />
+  <Import Project="..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets" Condition="Exists('..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets')" />
+  <Target Name="ValidateMSBuildFiles" BeforeTargets="PrepareForBuild">
+    <Error Condition="!Exists('..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.props')" Text="Unable to find the '..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.props' file. Please restore the 'Microsoft.VisualStudio.Azure.Fabric.MSBuild' Nuget package." />
+    <Error Condition="!Exists('..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets')" Text="Unable to find the '..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets' file. Please restore the 'Microsoft.VisualStudio.Azure.Fabric.MSBuild' Nuget package." />
   </Target>
 </Project>

Services/DataX.Flow/Flow.ManagementService/Controllers/FlowManagementController.cs

@@ -17,14 +17,17 @@ using System.Threading.Tasks;
 using System.Linq;
 using DataX.Utilities.Web;
 using System.Collections.Generic;
+using Microsoft.AspNetCore.Authorization;
+using DataX.ServiceHost.AspNetCore.Authorization.Roles;
 
 namespace Flow.Management.Controllers
 {
     [Route("api")]
+    [DataXReader]
     public partial class FlowManagementController : Controller
     {
         private readonly ILogger<FlowManagementController> _logger;
-        private FlowOperation _flowOperation;
+        private readonly FlowOperation _flowOperation;
         private JobOperation _jobOperation;
         private RuntimeConfigGeneration _runtimeConfigGenerator;
         private bool _isLocal = false;
@@ -44,6 +47,7 @@ namespace Flow.Management.Controllers
 
         [HttpPost]
         [Route("flow/save")] // save flow config
+        [DataXWriter]
         public async Task<ApiResult> SaveFlow([FromBody]JObject config)
         {
             try
@@ -75,6 +79,7 @@ namespace Flow.Management.Controllers
 
         [HttpPost]
         [Route("flow/generateconfigs")] // generate flow configs
+        [DataXWriter]
         public async Task<ApiResult> GenerateConfigs([FromBody] string flowName)
         {
             try
@@ -154,6 +159,7 @@ namespace Flow.Management.Controllers
 
         [HttpPost]
         [Route("flow/startjobs")]
+        [DataXWriter]
         public async Task<ApiResult> StartJobsForFlow([FromBody] string flowName)
         {
             try
@@ -174,6 +180,7 @@ namespace Flow.Management.Controllers
 
         [HttpPost]
         [Route("flow/restartjobs")]
+        [DataXWriter]
         public async Task<ApiResult> RestartJobsForFlow([FromBody] string flowName)
         {
             try
@@ -194,6 +201,7 @@ namespace Flow.Management.Controllers
 
         [HttpPost]
         [Route("flow/stopjobs")]
+        [DataXWriter]
         public async Task<ApiResult> StopJobsForFlow([FromBody] string flowName)
         {
             try
@@ -215,6 +223,7 @@ namespace Flow.Management.Controllers
 
         [HttpPost]
         [Route("userqueries/schema")] // generator (sqlparser)
+        [DataXWriter]
         public async Task<ApiResult> GetSchema([FromBody]JObject config)
         {
             try
@@ -242,6 +251,7 @@ namespace Flow.Management.Controllers
 
         [HttpPost]
         [Route("userqueries/codegen")] // generator
+        [DataXWriter]
         public async Task<ApiResult> GetCodeGen([FromBody]JObject config)
         {
             try
@@ -328,6 +338,7 @@ namespace Flow.Management.Controllers
 
         [HttpPost]
         [Route("job/start")] // start the job
+        [DataXWriter]
         public async Task<ApiResult> StartJob([FromBody] string jobName)
         {
             try
@@ -349,6 +360,7 @@ namespace Flow.Management.Controllers
 
         [HttpPost]
         [Route("job/stop")] // stop the job
+        [DataXWriter]
         public async Task<ApiResult> StopJob([FromBody] string jobName)
         {
             try
@@ -369,6 +381,7 @@ namespace Flow.Management.Controllers
 
         [HttpPost]
         [Route("job/restart")]
+        [DataXWriter]
         public async Task<ApiResult> RestartJob([FromBody] string jobName)
         {
             try
@@ -390,6 +403,7 @@ namespace Flow.Management.Controllers
 
         [HttpGet]
         [Route("job/syncall")]
+        [DataXWriter]
         public async Task<ApiResult> SyncAllJobs()
         {
             try

Services/DataX.Flow/Flow.ManagementService/Flow.ManagementService.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk.Web">
+<Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
     <TargetFramework>netcoreapp2.1</TargetFramework>
@@ -46,9 +46,11 @@
     <ProjectReference Include="..\..\DataX.Config\DataX.Config.Input.EventHub\DataX.Config.Input.EventHub.csproj" />
     <ProjectReference Include="..\..\DataX.Config\DataX.Config.KeyVault\DataX.Config.KeyVault.csproj" />
     <ProjectReference Include="..\..\DataX.Config\DataX.Config.LivyClient\DataX.Config.LivyClient.csproj" />
+    <ProjectReference Include="..\..\DataX.Config\DataX.Config.Local\DataX.Config.Local.csproj" />
     <ProjectReference Include="..\..\DataX.Config\DataX.Config.Storage\DataX.Config.Storage.csproj" />
     <ProjectReference Include="..\..\DataX.Config\DataX.Config\DataX.Config.csproj" />
     <ProjectReference Include="..\..\DataX.ServiceHost\DataX.ServiceHost.AspNetCore\DataX.ServiceHost.AspNetCore.csproj" />
+    <ProjectReference Include="..\..\DataX.ServiceHost\DataX.ServiceHost\DataX.ServiceHost.csproj" />
     <ProjectReference Include="..\..\DataX.Utilities\DataX.Utilities.KeyVault\DataX.Utilities.KeyVault.csproj" />
     <ProjectReference Include="..\..\DataX.Utilities\DataX.Utilities.Telemetry\DataX.Utilities.Telemetry.csproj" />
     <ProjectReference Include="..\..\DataX.Utilities\DataX.Utilities.Web\DataX.Utilities.Web.csproj" />

Services/DataX.Flow/Flow.ManagementService/FlowManagementService.cs

@@ -26,9 +26,13 @@ namespace Flow.Management
     /// </summary>
     internal sealed class FlowManagementService : StatelessService
     {
-        public FlowManagementService(StatelessServiceContext context)
+        private readonly IWebHostBuilder _webHostBuilder;
+
+        public FlowManagementService(StatelessServiceContext context, IWebHostBuilder webHostBuilder)
             : base(context)
-        { }
+        {
+            _webHostBuilder = webHostBuilder;
+        }
 
         /// <summary>
         /// Optional override to create listeners (like tcp, http) for this service instance.
@@ -43,15 +47,13 @@ namespace Flow.Management
                     {
                         ServiceEventSource.Current.ServiceMessage(serviceContext, $"Starting Kestrel on {url}");
 
-                        return new WebHostBuilder()
-                                    .UseKestrel()
+
+
+                        return _webHostBuilder
                                     .ConfigureServices(
                                         services => services
                                             .AddSingleton<StatelessServiceContext>(serviceContext)
                                              )
-                                    .UseContentRoot(Directory.GetCurrentDirectory())
-                                    .UseStartup<Startup>()
-                                    
                                     .UseServiceFabricIntegration(listener, ServiceFabricIntegrationOptions.None)
                                     .UseUrls(url)
                                     .Build();

Services/DataX.Flow/Flow.ManagementService/PackageRoot/Config/Settings.xml

@@ -14,5 +14,6 @@ Licensed under the MIT License
     <Parameter Name="MefStorageAccountName" Value="" />
     <Parameter Name="MefContainerName" Value="" />
     <Parameter Name="MefBlobDirectory" Value="" />
+    <Parameter Name="EnableOneBox" Value="" />
   </Section>
 </Settings>

Services/DataX.Flow/Flow.ManagementService/Program.cs

@@ -2,9 +2,12 @@
 // Copyright (c) Microsoft Corporation.  All rights reserved.
 // Licensed under the MIT License
 // *********************************************************************
+using DataX.ServiceHost;
+using Microsoft.AspNetCore.Hosting;
 using Microsoft.ServiceFabric.Services.Runtime;
 using System;
 using System.Diagnostics;
+using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -19,18 +22,28 @@ namespace Flow.Management
         {
             try
             {
-                // The ServiceManifest.XML file defines one or more service type names.
-                // Registering a service maps a service type name to a .NET type.
-                // When Service Fabric creates an instance of this service type,
-                // an instance of the class is created in this host process.
+                if(HostUtil.InServiceFabric)
+                {
+                    // The ServiceManifest.XML file defines one or more service type names.
+                    // Registering a service maps a service type name to a .NET type.
+                    // When Service Fabric creates an instance of this service type,
+                    // an instance of the class is created in this host process.
 
-                ServiceRuntime.RegisterServiceAsync("Flow.ManagementServiceType",
-                    context => new FlowManagementService(context)).GetAwaiter().GetResult();
+                    ServiceRuntime.RegisterServiceAsync("Flow.ManagementServiceType",
+                        context => new FlowManagementService(context, WebHostBuilder)).GetAwaiter().GetResult();
 
-                ServiceEventSource.Current.ServiceTypeRegistered(Process.GetCurrentProcess().Id, typeof(FlowManagementService).Name);
+                    ServiceEventSource.Current.ServiceTypeRegistered(Process.GetCurrentProcess().Id, typeof(FlowManagementService).Name);
 
-                // Prevents this host process from terminating so services keeps running. 
-                Thread.Sleep(Timeout.Infinite);
+                    // Prevents this host process from terminating so services keeps running. 
+                    Thread.Sleep(Timeout.Infinite);
+                }
+                else
+                {
+                    var webHost = WebHostBuilder.Build();
+
+                    webHost.Start();
+                    webHost.WaitForShutdown();
+                }
             }
             catch (Exception e)
             {
@@ -38,5 +51,11 @@ namespace Flow.Management
                 throw;
             }
         }
+
+        private static IWebHostBuilder WebHostBuilder
+            => new WebHostBuilder()
+                .UseKestrel()
+                .UseContentRoot(Directory.GetCurrentDirectory())
+                .UseStartup<Startup>();
     }
 }

Services/DataX.Flow/Flow.ManagementService/Properties/launchSettings.json

@@ -20,8 +20,9 @@
     "Flow.ManagementService": {
       "commandName": "Project",
       "launchBrowser": true,
-      "launchUrl": "api/values",
+      "launchUrl": "api/flow/getall",
       "environmentVariables": {
+        "AzureServicesAuthConnectionString": "RunAs=Developer; DeveloperTool=VisualStudio",
         "ASPNETCORE_ENVIRONMENT": "Development"
       },
       "applicationUrl": "http://localhost:5000"

Services/DataX.Flow/Flow.ManagementService/ServiceCollectionExtensions.cs

@@ -8,6 +8,8 @@ using System;
 using System.Collections.Generic;
 using System.Composition.Hosting;
 using System.Reflection;
+using DataX.Contract;
+using Microsoft.Extensions.Logging;
 
 namespace Flow.Management
 {
@@ -23,9 +25,9 @@ namespace Flow.Management
         /// <param name="exportTypes">Types being exported</param>
         /// <param name="instanceExports">Instances that are created outside that need to be added to the container</param>
         /// <returns></returns>
-        public static IServiceCollection AddMefExportsFromAssemblies(this IServiceCollection services, ServiceLifetime lifetime, IEnumerable<Assembly> assemblies, Type[] exportTypes, object[] instanceExports)
+        public static IServiceCollection AddMefExportsFromAssemblies(this IServiceCollection services, ServiceLifetime lifetime, IEnumerable<Assembly> assemblies, Type[] exportTypes, object[] instanceExports, ILoggerFactory loggerFactory = null, bool local = false)
         {          
-            var configuration = new ContainerConfiguration().WithAssemblies(assemblies).WithProvider(new InstanceExportDescriptorProvider(instanceExports));
+            var configuration = new ContainerConfiguration().WithAssemblies(assemblies).WithProvider(new LoggerAndInstanceExportDescriptorProvider(instanceExports, loggerFactory));
             using (var container = configuration.CreateContainer())
             {
                 foreach (var exportType in exportTypes)
@@ -37,6 +39,14 @@ namespace Flow.Management
                     }
                 }
 
+                if (local)
+                {
+                    var localTemplateInitializer = container.GetExport<DataX.Config.Local.TemplateInitializer>();
+                    var result = localTemplateInitializer?.Initialize().Result;
+
+                    Ensure.IsSuccessResult(result);
+                }
+
             }
             return services;
         }

Services/DataX.Flow/Flow.ManagementService/Startup.cs

@@ -5,8 +5,11 @@
 using DataX.Config;
 using DataX.Config.ConfigurationProviders;
 using DataX.Config.PublicService;
+using DataX.ServiceHost.AspNetCore.Authorization.Extensions;
+using DataX.ServiceHost.ServiceFabric.Extensions.Configuration;
 using DataX.Config.Storage;
 using DataX.ServiceHost.ServiceFabric;
+using DataX.ServiceHost.Settings;
 using DataX.Utilities.Blob;
 using DataX.Utilities.Telemetry;
 using Microsoft.AspNetCore.Builder;
@@ -21,14 +24,20 @@ using System.IO;
 using System.Linq;
 using System.Reflection;
 using System.Threading.Tasks;
+using DataX.Config.Local;
+using Microsoft.AspNetCore.Authentication.JwtBearer;
+using Microsoft.AspNetCore.Authorization;
+using DataX.ServiceHost.ServiceFabric.Authorization;
 
 namespace Flow.Management
 {
     public class Startup
     {
+        private const string _MetricsHttpEndpointRelativeUri = "/api/data/upload";
+
         private readonly ILoggerFactory _loggerFactory;
         public IConfiguration Configuration { get; }
-        private string _serviceKeyVaultName;
+        private readonly DataXSettings _dataXSettings = new DataXSettings();
 
         public Startup(IHostingEnvironment env, ILoggerFactory loggerFactory)
         {
@@ -36,22 +45,41 @@ namespace Flow.Management
                 .SetBasePath(env.ContentRootPath)
                 .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
                 .AddJsonFile($"appsettings.{env.EnvironmentName}.json", optional: true)
+                .AddServiceFabricSettings("Config", DataXSettingsConstants.DataX)
                 .AddEnvironmentVariables();
 
             Configuration = builder.Build();
 
+            Configuration.GetSection(DataXSettingsConstants.ServiceEnvironment).Bind(_dataXSettings);
+
             _loggerFactory = loggerFactory;
-        }        
-      
+        }
+
         // This method gets called by the runtime. Use this method to add services to the container.
         public void ConfigureServices(IServiceCollection services)
         {
             services.AddMvc().SetCompatibilityVersion(CompatibilityVersion.Version_2_1);
-            _serviceKeyVaultName = ServiceFabricUtil.GetServiceKeyVaultName().Result.ToString();
-            StartUpUtil.ConfigureServices(services);
 
-            // Configure and create a logger instance to add it to MEF container           
-            var logger = _loggerFactory.CreateLogger<RuntimeConfigGeneration>();
+            var bearerOptions = new JwtBearerOptions();
+
+            Configuration.GetSection("JwtBearerOptions").Bind(bearerOptions);
+
+            services
+                .AddSingleton(_dataXSettings)
+                .AddDataXAuthorization(DataXDefaultGatewayPolicy.ConfigurePolicy)
+                .AddAuthentication(options =>
+                {
+                    options.DefaultAuthenticateScheme = JwtBearerDefaults.AuthenticationScheme;
+                    options.DefaultChallengeScheme = JwtBearerDefaults.AuthenticationScheme;
+                    options.DefaultScheme = JwtBearerDefaults.AuthenticationScheme;
+                })
+                .AddJwtBearer(options =>
+                {
+                    options.Audience = bearerOptions.Audience;
+                    options.Authority = bearerOptions.Authority;
+                });
+
+            StartUpUtil.ConfigureServices(services, Configuration);
 
             // Initialize the settings by getting the values from settings file
             InitConfigSettings();
@@ -59,13 +87,13 @@ namespace Flow.Management
             // Export the Config dependencies
             Type[] exportTypes = new Type[] { typeof(FlowOperation), typeof(RuntimeConfigGeneration), typeof(JobOperation) };
 
-            IEnumerable<Assembly> cloudModeDependencyAssemblies = GetCloudModeDependencyAssemblies();
+            IEnumerable<Assembly> dependencyAssemblies = _dataXSettings.EnableOneBox ? OneBoxModeDependencyAssemblies : CloudModeDependencyAssemblies;
             IEnumerable<Assembly> additionalAssemblies = GetDependencyAssembliesFromStorageAsync().Result;
 
-            var allAssemblies = cloudModeDependencyAssemblies.Union(additionalAssemblies);
+            var allAssemblies = dependencyAssemblies.Union(additionalAssemblies);
 
-            services.AddMefExportsFromAssemblies(ServiceLifetime.Scoped, allAssemblies, exportTypes, new object[] { logger });            
-        }      
+            services.AddMefExportsFromAssemblies(ServiceLifetime.Scoped, allAssemblies, exportTypes, null, _loggerFactory, _dataXSettings.EnableOneBox);
+        }
 
         // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
         public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerFactory loggerFactory)
@@ -82,15 +110,16 @@ namespace Flow.Management
                 await next();
             });
 
+            app.UseAuthentication();
+
             // Configure logger that will be injected into the controller
             app.UseMvc();
         }
 
         // Get all the dependencies needed to fulfill the ConfigGen
         // requirements for cloud mode
-        private IList<Assembly> GetCloudModeDependencyAssemblies()
-        {
-            return new List<Assembly>()
+        private IList<Assembly> CloudModeDependencyAssemblies
+            => new List<Assembly>()
                 {
                     typeof(DataX.Config.ConfigGenConfiguration).Assembly,
                     typeof(DataX.Config.ConfigurationProviders.CosmosDbConfigurationProvider).Assembly,
@@ -99,40 +128,58 @@ namespace Flow.Management
                     typeof(DataX.Config.LivyClient.LivyClientFactory).Assembly,
                     typeof(DataX.Config.Input.EventHub.Processor.CreateEventHubConsumerGroup).Assembly
                 };
-        }
 
         // Get all the dependencies needed to fulfill the ConfigGen
         // requirements for oneBox mode
-        private IList<Assembly> GetOneBoxModeDependencyAssemblies()
-        {
-            throw new NotImplementedException();
-        }
+        private IList<Assembly> OneBoxModeDependencyAssemblies
+            => new List<Assembly>()
+                {
+                    typeof(DataX.Config.ConfigGenConfiguration).Assembly,
+                    typeof(DataX.Config.Local.LocalDesignTimeStorage).Assembly
+                };
 
         // Get the required settings to bootstrap the config gen
         private void InitConfigSettings()
         {
-            var cosmosDBConfigConnectionString = ServiceFabricUtil.GetServiceFabricConfigSetting("cosmosDBConfigConnectionString").Result.ToString();
-            var cosmosDBConfigDatabaseName = ServiceFabricUtil.GetServiceFabricConfigSetting("cosmosDBConfigDatabaseName").Result.ToString();
-            var cosmosDBConfigCollectionName = ServiceFabricUtil.GetServiceFabricConfigSetting("cosmosDBConfigCollectionName").Result.ToString();
+            InitialConfiguration.Set(DataX.Config.ConfigDataModel.Constants.ConfigSettingName_EnableOneBox, _dataXSettings.EnableOneBox.ToString());
+
+            if (!_dataXSettings.EnableOneBox)
+            {
+                InitialConfiguration.Set(CosmosDbConfigurationProvider.ConfigSettingName_CosmosDBConfig_ConnectionString, _dataXSettings.CosmosDBConfigConnectionString);
+                InitialConfiguration.Set(CosmosDbConfigurationProvider.ConfigSettingName_CosmosDBConfig_DatabaseName, _dataXSettings.CosmosDBConfigDatabaseName);
+                InitialConfiguration.Set(CosmosDbConfigurationProvider.ConfigSettingName_CosmosDBConfig_CollectionName, _dataXSettings.CosmosDBConfigCollectionName);
+                InitialConfiguration.Set(DataX.Config.ConfigDataModel.Constants.ConfigSettingName_ServiceKeyVaultName, _dataXSettings.ServiceKeyVaultName);
+            }
+            else
+            {
+                // Local settings
+                var metricsHttpEndpoint = _dataXSettings.MetricsHttpEndpoint.TrimEnd('/') + _MetricsHttpEndpointRelativeUri;
+                InitialConfiguration.Set(DataX.Config.ConfigDataModel.Constants.ConfigSettingName_LocalRoot, _dataXSettings.LocalRoot);
+                InitialConfiguration.Set(DataX.Config.Local.LocalSparkClient.ConfigSettingName_SparkHomeFolder, _dataXSettings.SparkHome);
+                InitialConfiguration.Set(DataX.Config.ConfigDataModel.Constants.ConfigSettingName_ClusterName, "localCluster");
+                InitialConfiguration.Set(DataX.Config.ConfigDataModel.Constants.ConfigSettingName_ServiceKeyVaultName, "local");
+                InitialConfiguration.Set(DataX.Config.ConfigDataModel.Constants.ConfigSettingName_RuntimeKeyVaultName, "local");
+                InitialConfiguration.Set(DataX.Config.ConfigDataModel.Constants.ConfigSettingName_MetricEventHubConnectionKey, "local");
+                InitialConfiguration.Set(DataX.Config.ConfigDataModel.Constants.ConfigSettingName_ConfigFolderContainerPath, "");
+                InitialConfiguration.Set(DataX.Config.ConfigDataModel.Constants.ConfigSettingName_ConfigFolderHost, new System.Uri(Environment.CurrentDirectory).AbsoluteUri);
+                InitialConfiguration.Set(DataX.Config.ConfigDataModel.Constants.ConfigSettingName_LocalMetricsHttpEndpoint, metricsHttpEndpoint);
+            }
 
-            InitialConfiguration.Set(CosmosDbConfigurationProvider.ConfigSettingName_CosmosDBConfig_ConnectionString, cosmosDBConfigConnectionString);
-            InitialConfiguration.Set(CosmosDbConfigurationProvider.ConfigSettingName_CosmosDBConfig_DatabaseName, cosmosDBConfigDatabaseName);
-            InitialConfiguration.Set(CosmosDbConfigurationProvider.ConfigSettingName_CosmosDBConfig_CollectionName, cosmosDBConfigCollectionName);
-            InitialConfiguration.Set(DataX.Config.ConfigDataModel.Constants.ConfigSettingName_ServiceKeyVaultName, _serviceKeyVaultName);
         }
 
         // Get additional assemblies from azure storage
         private async Task<IEnumerable<Assembly>> GetDependencyAssembliesFromStorageAsync()
         {
             IEnumerable<Assembly> additionalAssemblies = new List<Assembly>();
-            var mefStorageAccountName = ServiceFabricUtil.GetServiceFabricConfigSetting("MefStorageAccountName").Result.ToString();
-            var mefContainerName = ServiceFabricUtil.GetServiceFabricConfigSetting("MefContainerName").Result.ToString();
+            var mefStorageAccountName = _dataXSettings.MefStorageAccountName;
+            var mefContainerName = _dataXSettings.MefContainerName;
+
             if (string.IsNullOrEmpty(mefStorageAccountName) || string.IsNullOrEmpty(mefContainerName))
             {
                 return additionalAssemblies;
             }
-            
-            var mefBlobDirectory = ServiceFabricUtil.GetServiceFabricConfigSetting("MefBlobDirectory").Result.ToString();
+
+            var mefBlobDirectory = _dataXSettings.MefBlobDirectory;
 
             BlobStorageMSI blobStorage = new BlobStorageMSI(mefStorageAccountName);
 

Services/DataX.Flow/Flow.ManagementService/appsettings.Development.json

@@ -5,5 +5,36 @@
       "System": "Information",
       "Microsoft": "Information"
     }
+  },
+  "Kestrel": {
+    "EndPoints": {
+      "Http": {
+        "Url": "http://localhost:5000"
+      },
+      "HttpsDefaultCert": {
+        "Url": "https://localhost:5001",
+        "Protocols": "Http1AndHttp2"
+      }
+    }
+  },
+  "DataX": {
+    "ServiceEnvironment": {
+      "EnableOneBox": true,
+
+      // EnableOneBox == false settings
+      "AppInsightsIntrumentationKey": "",
+      "CosmosDBConfigCollectionName": "",
+      "CosmosDBConfigConnectionString": "",
+      "CosmosDBConfigDatabaseName": "",
+      "MefBlobDirectory": null,
+      "MefContainerName": null,
+      "MefStorageAccountName": null,
+      "ServiceKeyVaultName": "",
+
+      // EnableOneBox == true settings
+      "LocalRoot": "",
+      "MetricsHttpEndpoint": "http://localhost:2020/",
+      "SparkHome": ""
+    }
   }
 }

Services/DataX.Flow/Flow.ManagementService/appsettings.json

@@ -4,5 +4,22 @@
       "Default": "Information"
     }
   },
-  "AllowedHosts": "*"  
+  "AllowedHosts": "*",
+  "DataX": {
+    "ServiceEnvironment": {
+      "EnableOneBox": false,
+      "AppInsightsIntrumentationKey": "",
+      "CosmosDBConfigCollectionName": "",
+      "CosmosDBConfigConnectionString": "",
+      "CosmosDBConfigDatabaseName": "",
+      "MefBlobDirectory": null,
+      "MefContainerName": null,
+      "MefStorageAccountName": null,
+      "ServiceKeyVaultName": ""
+    }
+  },
+  "JwtBearerOptions": {
+    "Audience": "",
+    "Authority": null // JWT ISS (issuer)
+  }
 }

Services/DataX.Gateway/DataX.Gateway/DataX.Gateway.sfproj

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" InitialTargets=";ValidateMSBuildFiles">
-  <Import Project="..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.6\build\Microsoft.VisualStudio.Azure.Fabric.Application.props" Condition="Exists('..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.6\build\Microsoft.VisualStudio.Azure.Fabric.Application.props')" />
+  <Import Project="..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.6\build\Microsoft.VisualStudio.Azure.Fabric.Application.props" Condition="Exists('..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.6\build\Microsoft.VisualStudio.Azure.Fabric.Application.props')" />
   <PropertyGroup Label="Globals">
     <ProjectGuid>1356403c-69c6-429b-ac77-38780284a658</ProjectGuid>
     <ProjectVersion>2.1</ProjectVersion>
@@ -38,9 +38,9 @@
     <ApplicationProjectTargetsPath>$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\Service Fabric Tools\Microsoft.VisualStudio.Azure.Fabric.ApplicationProject.targets</ApplicationProjectTargetsPath>
   </PropertyGroup>
   <Import Project="$(ApplicationProjectTargetsPath)" Condition="Exists('$(ApplicationProjectTargetsPath)')" />
-  <Import Project="..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.6\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets" Condition="Exists('..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.6\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets')" />
-  <Target Name="ValidateMSBuildFiles">
-    <Error Condition="!Exists('..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.6\build\Microsoft.VisualStudio.Azure.Fabric.Application.props')" Text="Unable to find the '..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.6\build\Microsoft.VisualStudio.Azure.Fabric.Application.props' file. Please restore the 'Microsoft.VisualStudio.Azure.Fabric.MSBuild' Nuget package." />
-    <Error Condition="!Exists('..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.6\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets')" Text="Unable to find the '..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.6\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets' file. Please restore the 'Microsoft.VisualStudio.Azure.Fabric.MSBuild' Nuget package." />
+  <Import Project="..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.6\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets" Condition="Exists('..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.6\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets')" />
+  <Target Name="ValidateMSBuildFiles" BeforeTargets="PrepareForBuild">
+    <Error Condition="!Exists('..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.6\build\Microsoft.VisualStudio.Azure.Fabric.Application.props')" Text="Unable to find the '..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.6\build\Microsoft.VisualStudio.Azure.Fabric.Application.props' file. Please restore the 'Microsoft.VisualStudio.Azure.Fabric.MSBuild' Nuget package." />
+    <Error Condition="!Exists('..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.6\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets')" Text="Unable to find the '..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.6\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets' file. Please restore the 'Microsoft.VisualStudio.Azure.Fabric.MSBuild' Nuget package." />
   </Target>
 </Project>

Services/DataX.Metrics/DataX.Metrics.Ingestor/Helper/SecretStore.cs

@@ -1,7 +1,8 @@
-// *********************************************************************
+// *********************************************************************
 // Copyright (c) Microsoft Corporation.  All rights reserved.
 // Licensed under the MIT License
 // *********************************************************************
+using DataX.Contract;
 using DataX.ServiceHost.ServiceFabric;
 using DataX.Utilities.KeyVault;
 using System;
@@ -20,28 +21,30 @@ namespace DataX.Metrics.Ingestor.Helper
 
         private SecretsStore()
         {
-            _keyVaultName = (string)ServiceFabricUtil.GetServiceKeyVaultName().Result;
+            _keyVaultName = ServiceFabricUtil.GetServiceKeyVaultName().Result?.ToString();
         }
 
         public static SecretsStore Instance => _LazyInstance.Value;
 
         public async Task<string> GetMetricsEventHubListenerConnectionStringAsync()
         {
-            return await GetSecretAsync(Utility.GetConfigValue("EventhubNamespaceConnectionstring"));
+            return await GetSecretAsync(ServiceFabricUtil.GetServiceFabricConfigSetting("EventhubNamespaceConnectionstring"));
         }
 
         public async Task<string> GetMetricsStorageConnectionStringAsync()
         {
-            return await GetSecretAsync(Utility.GetConfigValue("StorageAccountConnectionstring"));
+            return await GetSecretAsync(ServiceFabricUtil.GetServiceFabricConfigSetting("StorageAccountConnectionstring"));
         }
         
         public async Task<string> GetMetricsRedisConnectionStringAsync()
         {
-            return await GetSecretAsync(Utility.GetConfigValue("RedisCacheConnectionstring"));
+            return await GetSecretAsync(ServiceFabricUtil.GetServiceFabricConfigSetting("RedisCacheConnectionstring"));
         }
 
-        private async Task<string> GetSecretAsync(string key)
+        private async Task<string> GetSecretAsync(ApiResult setting)
         {
+            string key = setting.Result?.ToString();
+
             KeyVaultManager keyManager = new KeyVaultManager();
             var secret = await keyManager.GetSecretStringAsync(_keyVaultName, key);
             return secret;

Services/DataX.Metrics/DataX.Metrics.Ingestor/Helper/Utility.cs

@@ -1,7 +1,8 @@
-// *********************************************************************
+// *********************************************************************
 // Copyright (c) Microsoft Corporation.  All rights reserved.
 // Licensed under the MIT License
 // *********************************************************************
+using System;
 using System.Fabric;
 
 namespace DataX.Metrics.Ingestor.Helper
@@ -9,6 +10,11 @@ namespace DataX.Metrics.Ingestor.Helper
     /// <summary>
     /// Utility class
     /// </summary>
+    // TODO: Remove when configuration fully consolidated
+    [Obsolete("Use another method for configuration, such as: "
+        + nameof(ServiceHost.ServiceFabric.ServiceFabricUtil) + ", "
+        + nameof(ServiceHost.Settings.DataXSettings) + ", or "
+        + nameof(Microsoft.Extensions.Configuration.IConfiguration))]
     public static class Utility
     {
         /// <summary>

Services/DataX.Metrics/DataX.Metrics.Ingestor/Ingestor.cs

@@ -134,8 +134,8 @@ namespace DataX.Metrics.Ingestor
         private async Task<EventProcessorHost> InitalizeEventProcessorHostAsync()
         {
             // start listening to the event hub 
-            var eventHubName = Utility.GetConfigValue("EventHubName");
-            var consumerGroup = Utility.GetConfigValue("ConsumerGroupName");
+            var eventHubName = ServiceFabricUtil.GetServiceFabricConfigSetting("EventHubName").Result?.ToString();
+            var consumerGroup = ServiceFabricUtil.GetServiceFabricConfigSetting("ConsumerGroupName").Result?.ToString();
 
             var eventProcessorHost = new EventProcessorHost(
                     eventHubName,

Services/DataX.Metrics/DataX.Metrics.Ingestor/Startup.cs

@@ -29,7 +29,7 @@ namespace DataX.Metrics.Ingestor
         public IConfiguration Configuration { get; }
 
         // This method gets called by the runtime. Use this method to add services to the container.
-        public static void ConfigureServices(IServiceCollection services)
+        public void ConfigureServices(IServiceCollection services)
         {
             services.Configure<CookiePolicyOptions>(options =>
             {
@@ -37,7 +37,7 @@ namespace DataX.Metrics.Ingestor
                 options.CheckConsentNeeded = context => true;
                 options.MinimumSameSitePolicy = SameSiteMode.None;
             });
-            StartUpUtil.ConfigureServices(services);            
+            StartUpUtil.ConfigureServices(services, Configuration);            
 
             services.AddMvc().SetCompatibilityVersion(CompatibilityVersion.Version_2_1);
         }

Services/DataX.Metrics/DataX.Metrics/DataX.Metrics.sfproj

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" InitialTargets=";ValidateMSBuildFiles">
-  <Import Project="..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.props" Condition="Exists('..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.props')" />
+  <Import Project="..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.props" Condition="Exists('..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.props')" />
   <PropertyGroup Label="Globals">
     <ProjectGuid>0de74d1c-16d6-42dd-beaf-ddf8d556dbfe</ProjectGuid>
     <ProjectVersion>2.4</ProjectVersion>
@@ -39,9 +39,9 @@
     <ApplicationProjectTargetsPath>$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\Service Fabric Tools\Microsoft.VisualStudio.Azure.Fabric.ApplicationProject.targets</ApplicationProjectTargetsPath>
   </PropertyGroup>
   <Import Project="$(ApplicationProjectTargetsPath)" Condition="Exists('$(ApplicationProjectTargetsPath)')" />
-  <Import Project="..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets" Condition="Exists('..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets')" />
-  <Target Name="ValidateMSBuildFiles">
-    <Error Condition="!Exists('..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.props')" Text="Unable to find the '..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.props' file. Please restore the 'Microsoft.VisualStudio.Azure.Fabric.MSBuild' Nuget package." />
-    <Error Condition="!Exists('..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets')" Text="Unable to find the '..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets' file. Please restore the 'Microsoft.VisualStudio.Azure.Fabric.MSBuild' Nuget package." />
+  <Import Project="..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets" Condition="Exists('..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets')" />
+  <Target Name="ValidateMSBuildFiles" BeforeTargets="PrepareForBuild">
+    <Error Condition="!Exists('..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.props')" Text="Unable to find the '..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.props' file. Please restore the 'Microsoft.VisualStudio.Azure.Fabric.MSBuild' Nuget package." />
+    <Error Condition="!Exists('..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets')" Text="Unable to find the '..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets' file. Please restore the 'Microsoft.VisualStudio.Azure.Fabric.MSBuild' Nuget package." />
   </Target>
 </Project>

Services/DataX.ServiceHost/DataX.ServiceHost.AspNetCore/Authorization/DataXAuthConstants.cs

@@ -0,0 +1,19 @@
+using DataX.Utilities.Web;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace DataX.ServiceHost.AspNetCore.Authorization
+{
+    /// <summary>
+    /// Constants used in DataX auth scenarios
+    /// </summary>
+    public static class DataXAuthConstants
+    {
+        public const string PolicyPrefix = "DataXAuth_";
+
+        public static string WriterPolicyName { get; } = PolicyPrefix + RolesCheck.WriterRoleName;
+
+        public static string ReaderPolicyName { get; } = PolicyPrefix + RolesCheck.ReaderRoleName;
+    }
+}

Services/DataX.ServiceHost/DataX.ServiceHost.AspNetCore/Authorization/DataXAuthorizeAttribute.cs

@@ -0,0 +1,18 @@
+using DataX.Utilities.Web;
+using Microsoft.AspNetCore.Authorization;
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace DataX.ServiceHost.AspNetCore.Authorization
+{
+    /// <inheritdoc />
+    public abstract class DataXAuthorizeAttribute : AuthorizeAttribute
+    {
+        public DataXAuthorizeAttribute()
+        {
+            Policy = DataXAuthConstants.PolicyPrefix;
+        }
+    }
+}

Services/DataX.ServiceHost/DataX.ServiceHost.AspNetCore/Authorization/DataXPolicyBuilder.cs

@@ -0,0 +1,83 @@
+namespace DataX.ServiceHost.AspNetCore.Authorization
+{
+    using DataX.ServiceHost.Authorization;
+    using DataX.ServiceHost.Settings;
+    using Microsoft.AspNetCore.Authorization;
+    using System;
+    using System.Linq;
+
+    /// <summary>
+    /// This class is meant to simplify the syntax for adding requirements for policies
+    /// </summary>
+    internal class DataXPolicyBuilder
+    {
+        private readonly AuthorizationOptions _options;
+        private readonly DataXSettings _settings;
+        private readonly Action<AuthorizationPolicyBuilder> _configurePolicy;
+
+        public DataXPolicyBuilder(
+            AuthorizationOptions options,
+            DataXSettings settings,
+            Action<AuthorizationPolicyBuilder> configurePolicy)
+        {
+            _options = options;
+            _settings = settings;
+            _configurePolicy = configurePolicy;
+        }
+
+        public DataXPolicyBuilder AddPolicy<TRequirement>(string name)
+            where TRequirement : DataXAuthRequirement, new()
+        {
+            _options.AddPolicy(name, DataXPolicy<TRequirement>);
+
+            return this;
+        }
+
+        private void DataXPolicy<TDataXRequirement>(AuthorizationPolicyBuilder policy)
+            where TDataXRequirement : DataXAuthRequirement, new()
+        {
+            AddDataXRequirements<TDataXRequirement>(policy);
+        }
+
+        /// <summary>
+        /// Adds the basic DataX auth policy to the builder
+        /// </summary>
+        private AuthorizationPolicyBuilder AddDataXRequirements<TDataXRequirement>(AuthorizationPolicyBuilder policy)
+            where TDataXRequirement : DataXAuthRequirement, new()
+        {
+            // We don't want to add the same requirement in again.
+            // If it does exist and the settings changed, then we want to make sure the new settings are used
+            RemoveDataXRequirements(policy);
+
+            var requirement = new TDataXRequirement()
+            {
+                Settings = _settings
+            };
+
+            if (!_settings.EnableOneBox)
+            {
+                policy.RequireAuthenticatedUser();
+            }
+
+            policy.AddRequirements(requirement);
+            _configurePolicy?.Invoke(policy);
+
+            return policy;
+        }
+
+        /// <summary>
+        /// Removes the DataXRequirements set in the policy builder if they exist.
+        /// </summary>
+        private static AuthorizationPolicyBuilder RemoveDataXRequirements(AuthorizationPolicyBuilder policy)
+        {
+            var requirements = policy.Requirements.Where(req => req is DataXAuthRequirement);
+
+            foreach (var req in requirements)
+            {
+                policy.Requirements.Remove(req);
+            }
+
+            return policy;
+        }
+    }
+}

Services/DataX.ServiceHost/DataX.ServiceHost.AspNetCore/Authorization/Extensions/DataXAuthorizationExtensions.cs

@@ -0,0 +1,55 @@
+using DataX.ServiceHost.Settings;
+using DataX.Utilities.Web;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.Extensions.DependencyInjection;
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Linq;
+using DataX.ServiceHost.AspNetCore.Authorization.Requirements;
+
+namespace DataX.ServiceHost.AspNetCore.Authorization.Extensions
+{
+    public static class DataXAuthorizationExtensions
+    {
+        /// <summary>
+        /// Adds DataX default authentication to the given service collection.
+        /// </summary>
+        /// <param name="services">The service collection to add to</param>
+        /// <returns>The modified service collection</returns>
+        public static IServiceCollection AddDataXAuthorization(this IServiceCollection services)
+        {
+            return services.AddDataXAuthorization(null);
+        }
+
+        /// <summary>
+        /// Adds DataX default authentication to the given service collection.
+        /// </summary>
+        /// <param name="services">The service collection to add to</param>
+        /// <param name="configurePolicy">An action to configure supplement policy configuration</param>
+        /// <returns>The modified service collection</returns>
+        public static IServiceCollection AddDataXAuthorization(this IServiceCollection services, Action<AuthorizationPolicyBuilder> configurePolicy)
+        {
+            var settings = services.BuildServiceProvider().GetService<DataXSettings>();
+
+            // EnableOneBox scenario as it requires the least configuration and we can't assume cloud connection settings
+            if (settings == null)
+            {
+                settings = new DataXSettings()
+                {
+                    EnableOneBox = true,
+                    LocalRoot = "",
+                    MetricsHttpEndpoint = "http://localhost:2020/",
+                    SparkHome = "",
+                };
+            }
+
+            return services.AddAuthorization(options =>
+            {
+                new DataXPolicyBuilder(options, settings, configurePolicy)
+                    .AddPolicy<DataXWriterRequirement>(DataXAuthConstants.WriterPolicyName)
+                    .AddPolicy<DataXReaderRequirement>(DataXAuthConstants.ReaderPolicyName);
+            });
+        }
+    }
+}

Services/DataX.ServiceHost/DataX.ServiceHost.AspNetCore/Authorization/Requirements/DataXReaderRequirement.cs

@@ -0,0 +1,21 @@
+namespace DataX.ServiceHost.AspNetCore.Authorization.Requirements
+{
+    using DataX.ServiceHost.Settings;
+    using DataX.Utilities.Web;
+    using Microsoft.AspNetCore.Authorization;
+
+    /// <inheritdoc />
+    internal class DataXReaderRequirement : DataXWriterRequirement
+    {
+        public DataXReaderRequirement() { }
+
+        public DataXReaderRequirement(DataXSettings settings)
+            : base(settings) { }
+
+        /// <inheritdoc />
+        protected override bool IsAuthorized(AuthorizationHandlerContext context, DataXSettings settings)
+        {
+            return base.IsAuthorized(context, settings) || context.User.IsInRole(RolesCheck.ReaderRoleName);
+        }
+    }
+}

Services/DataX.ServiceHost/DataX.ServiceHost.AspNetCore/Authorization/Requirements/DataXWriterRequirement.cs

@@ -0,0 +1,22 @@
+namespace DataX.ServiceHost.AspNetCore.Authorization.Requirements
+{
+    using DataX.ServiceHost.Authorization.Requirements;
+    using DataX.ServiceHost.Settings;
+    using DataX.Utilities.Web;
+    using Microsoft.AspNetCore.Authorization;
+
+    /// <inheritdoc />
+    internal class DataXWriterRequirement : DataXOneBoxRequirement
+    {
+        public DataXWriterRequirement() { }
+
+        public DataXWriterRequirement(DataXSettings settings)
+            : base(settings) { }
+
+        /// <inheritdoc />
+        protected override bool IsAuthorized(AuthorizationHandlerContext context, DataXSettings settings)
+        {
+            return base.IsAuthorized(context, settings) || context.User.IsInRole(RolesCheck.WriterRoleName);
+        }
+    }
+}

Services/DataX.ServiceHost/DataX.ServiceHost.AspNetCore/Authorization/Roles/DataXReaderAttribute.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace DataX.ServiceHost.AspNetCore.Authorization.Roles
+{
+    /// <inheritdoc />
+    public class DataXReaderAttribute : DataXAuthorizeAttribute
+    {
+        public DataXReaderAttribute()
+        {
+            Policy = DataXAuthConstants.ReaderPolicyName;
+        }
+    }
+}

Services/DataX.ServiceHost/DataX.ServiceHost.AspNetCore/Authorization/Roles/DataXWriterAttribute.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace DataX.ServiceHost.AspNetCore.Authorization.Roles
+{
+    /// <inheritdoc />
+    public class DataXWriterAttribute : DataXAuthorizeAttribute
+    {
+        public DataXWriterAttribute()
+        {
+            Policy = DataXAuthConstants.WriterPolicyName;
+        }
+    }
+}

Services/DataX.ServiceHost/DataX.ServiceHost.AspNetCore/DataX.ServiceHost.AspNetCore.csproj

@@ -18,7 +18,7 @@
   <ItemGroup>
     <ProjectReference Include="..\..\DataX.Contract\DataX.Contract.csproj" />
     <ProjectReference Include="..\..\DataX.Utilities\DataX.Utilities.Telemetry\DataX.Utilities.Telemetry.csproj" />
-    <ProjectReference Include="..\DataX.ServiceHost.ServiceFabric\DataX.ServiceHost.ServiceFabric.csproj" />
+    <ProjectReference Include="..\..\DataX.Utilities\DataX.Utilities.Web\DataX.Utilities.Web.csproj" />
     <PackageReference Include="Microsoft.CodeAnalysis.FxCopAnalyzers" Version="2.6.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>

Services/DataX.ServiceHost/DataX.ServiceHost.AspNetCore/StartUpBase.cs

@@ -2,6 +2,8 @@
 // Copyright (c) Microsoft Corporation.  All rights reserved.
 // Licensed under the MIT License
 // *********************************************************************
+using DataX.ServiceHost.ServiceFabric.Extensions.Configuration;
+using DataX.ServiceHost.Settings;
 using DataX.Utilities.Telemetry;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
@@ -20,7 +22,8 @@ namespace DataX.ServiceHost.AspNetCore
                 .SetBasePath(env.ContentRootPath)
                 .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
                 .AddJsonFile($"appsettings.{env.EnvironmentName}.json", optional: true)
-                .AddEnvironmentVariables();            
+                .AddServiceFabricSettings("Config", DataXSettingsConstants.DataX)
+                .AddEnvironmentVariables();
 
             Configuration = builder.Build();
         }
@@ -30,7 +33,7 @@ namespace DataX.ServiceHost.AspNetCore
         public void ConfigureServices(IServiceCollection services)
         {
             services.AddMvc().SetCompatibilityVersion(CompatibilityVersion.Version_2_1);
-            StartUpUtil.ConfigureServices(services);            
+            StartUpUtil.ConfigureServices(services, Configuration);            
         }
 
         // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.

Services/DataX.ServiceHost/DataX.ServiceHost.ServiceFabric/Authorization/DataXDefaultGatewayPolicy.cs

@@ -0,0 +1,38 @@
+namespace DataX.ServiceHost.ServiceFabric.Authorization
+{
+    using DataX.ServiceHost.Authorization;
+    using DataX.ServiceHost.Settings;
+    using Microsoft.AspNetCore.Authorization;
+    using Microsoft.AspNetCore.Authorization.Infrastructure;
+    using System.Linq;
+
+    /// <inheritdoc />
+    public class DataXDefaultGatewayPolicy : DataXAuthRequirement
+    {
+        private DataXDefaultGatewayPolicy() { }
+
+        /// <summary>
+        /// Configures the default, recommended policy for running in ServiceFabric behind Gateway
+        /// </summary>
+        /// <param name="builder">The AuthorizationPolicyBuilder to modify</param>
+        public static void ConfigurePolicy(AuthorizationPolicyBuilder builder)
+        {
+            // ServiceFabric setup is the only setup with the Gateway
+            if(HostUtil.InServiceFabric)
+            {
+                // Allow anonymous calls; RoleCheck will handle the checking in the controller
+                builder.Requirements = builder.Requirements.Where(r => !(r is DenyAnonymousAuthorizationRequirement || r is DataXAuthRequirement)).ToList();
+
+                // Ensure the OneBox check
+                builder.AddRequirements(new DataXDefaultGatewayPolicy());
+            }
+        }
+
+        /// <inheritdoc />
+        protected override bool IsAuthorized(AuthorizationHandlerContext context, DataXSettings settings)
+        {
+            // If OneBox, no auth. If in SF, no auth (handled in Gateway). True in both scenarios, so returning true.
+            return true;
+        }
+    }
+}

Services/DataX.ServiceHost/DataX.ServiceHost.ServiceFabric/DataX.ServiceHost.ServiceFabric.csproj

@@ -16,10 +16,12 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Microsoft.AspNetCore.Authorization" Version="2.1.0" />
     <PackageReference Include="Microsoft.CodeAnalysis.FxCopAnalyzers" Version="2.6.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
     </PackageReference>
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="2.1.0" />
     <PackageReference Include="Microsoft.ServiceFabric.Services" Version="3.3.617" />
     <PackageReference Include="Microsoft.VisualStudioEng.MicroBuild.Core" Version="0.4.1">
       <PrivateAssets>all</PrivateAssets>
@@ -29,6 +31,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\DataX.Contract\DataX.Contract.csproj" />
+    <ProjectReference Include="..\DataX.ServiceHost\DataX.ServiceHost.csproj" />
   </ItemGroup>
 
   <ItemGroup>

Services/DataX.ServiceHost/DataX.ServiceHost.ServiceFabric/Extensions/Configuration/ServiceFabricConfigurationBuilderExtensions.cs

@@ -0,0 +1,33 @@
+// *********************************************************************
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+// Licensed under the MIT License
+// *********************************************************************
+
+namespace DataX.ServiceHost.ServiceFabric.Extensions.Configuration
+{
+    using Microsoft.Extensions.Configuration;
+    using System;
+
+    public static class ServiceFabricConfigurationBuilderExtensions
+    {
+        /// <summary>
+        /// Adds the specified service fabric settings to the configuration builder. If service fabric is not detected, nothing is added.
+        /// </summary>
+        /// <param name="packageName">The configuration package object name</param>
+        /// <param name="configPrefix">An optional prefix to add to the configurations added to the builder. e.g. "MySettings:"</param>
+        public static IConfigurationBuilder AddServiceFabricSettings(this IConfigurationBuilder configurationBuilder, string packageName, string configPrefix = "")
+        {
+            if (configurationBuilder == null)
+            {
+                throw new ArgumentNullException(nameof(configurationBuilder));
+            }
+
+            if (HostUtil.InServiceFabric)
+            {
+                configurationBuilder.Add(new ServiceFabricConfigurationSource(packageName, configPrefix));
+            }
+
+            return configurationBuilder;
+        }
+    }
+}

Services/DataX.ServiceHost/DataX.ServiceHost.ServiceFabric/ServiceFabricConfigurationSource.cs

@@ -0,0 +1,45 @@
+// *********************************************************************
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+// Licensed under the MIT License
+// *********************************************************************
+
+namespace DataX.ServiceHost.ServiceFabric
+{
+    using Microsoft.Extensions.Configuration.Memory;
+    using System.Collections.Generic;
+    using System.Fabric;
+
+    /// <inheritdoc />
+    public class ServiceFabricConfigurationSource : MemoryConfigurationSource
+    {
+        private readonly string _configPrefix;
+
+        public ServiceFabricConfigurationSource(string packageName, string configPrefix)
+        {
+            _configPrefix = configPrefix.EndsWith(":") ? configPrefix : configPrefix + ":";
+
+            var package = FabricRuntime.GetActivationContext().GetConfigurationPackageObject(packageName);
+
+            this.InitialData = GetFlatConfig(package);
+        }
+
+        /// <summary>
+        /// Creates a flat configuration compatible with IConfiguration from ServiceFabric's ConfigurationPackage
+        /// </summary>
+        /// <param name="package">The ConfigurationPackage to flatten</param>
+        /// <returns>A KeyValuePair IEnumerable to be used for <see cref="MemoryConfigurationSource.InitialData"/></returns>
+        private IEnumerable<KeyValuePair<string,string>> GetFlatConfig(ConfigurationPackage package)
+        {
+            var flatConfig = new Dictionary<string, string>();
+            foreach (var section in package.Settings.Sections)
+            {
+                foreach (var parameter in section.Parameters)
+                {
+                    flatConfig[$"{_configPrefix}{section.Name}:{parameter.Name}"] = parameter.Value;
+                }
+            }
+
+            return flatConfig;
+        }
+    }
+}

Services/DataX.ServiceHost/DataX.ServiceHost/Authorization/DataXAuthRequirement.cs

@@ -0,0 +1,41 @@
+namespace DataX.ServiceHost.Authorization
+{
+    using DataX.ServiceHost.Settings;
+    using Microsoft.AspNetCore.Authorization;
+    using System.Threading.Tasks;
+
+    /// <summary>
+    /// For assertion requirements. We're extending this so that we can easily identify the DataX requirement instance
+    /// when adding in policy requirements. This lets us prevent duplication of requirements and handlers.
+    /// </summary>
+    public abstract class DataXAuthRequirement : IAuthorizationHandler, IAuthorizationRequirement
+    {
+        public DataXSettings Settings { get; set; }
+
+        public DataXAuthRequirement() { }
+
+        public DataXAuthRequirement(DataXSettings settings)
+        {
+            Settings = settings;
+        }
+
+        /// <inheritdoc />
+        public Task HandleAsync(AuthorizationHandlerContext context)
+        {
+            if(IsAuthorized(context, Settings))
+            {
+                context.Succeed(this);
+            }
+
+            return Task.CompletedTask;
+        }
+
+        /// <summary>
+        /// Given the auth context and settings, determines if a request is authorized.
+        /// </summary>
+        /// <param name="context">Provided AuthorizationHandlerContext</param>
+        /// <param name="settings">Provided DataXSettings</param>
+        /// <returns>True if determined to be authorized, else false.</returns>
+        protected abstract bool IsAuthorized(AuthorizationHandlerContext context, DataXSettings settings);
+    }
+}

Services/DataX.ServiceHost/DataX.ServiceHost/Authorization/Requirements/DataXOneBoxRequirement.cs

@@ -0,0 +1,21 @@
+using DataX.ServiceHost.Settings;
+using Microsoft.AspNetCore.Authorization;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace DataX.ServiceHost.Authorization.Requirements
+{
+    public class DataXOneBoxRequirement : DataXAuthRequirement
+    {
+        public DataXOneBoxRequirement() { }
+
+        public DataXOneBoxRequirement(DataXSettings settings)
+            : base(settings) { }
+
+        protected override bool IsAuthorized(AuthorizationHandlerContext context, DataXSettings settings)
+        {
+            return settings.EnableOneBox;
+        }
+    }
+}

Services/DataX.ServiceHost/DataX.ServiceHost/DataX.ServiceHost.csproj

@@ -1,7 +1,14 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <TargetFramework>netstandard2.0</TargetFramework>
+    <SignAssembly>true</SignAssembly>
+    <AssemblyOriginatorKeyFile>FinalPublicKey.snk</AssemblyOriginatorKeyFile>
+    <DelaySign>true</DelaySign>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="Microsoft.AspNetCore.Authorization" Version="2.1.0" />
+  </ItemGroup>
+
 </Project>

Services/DataX.ServiceHost/DataX.ServiceHost/HostUtil.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace DataX.ServiceHost
+{
+    public static class HostUtil
+    {
+        /// <summary>
+        /// Returns true if Fabric_ApplicationName environment variable is not null, otherwise false if it is null.
+        /// </summary>
+        public static bool InServiceFabric => Environment.GetEnvironmentVariable("Fabric_ApplicationName") != null;
+    }
+}

Services/DataX.ServiceHost/DataX.ServiceHost/Settings/DataXSettings.cs

@@ -0,0 +1,38 @@
+
+
+namespace DataX.ServiceHost.Settings
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Text;
+
+    /// <summary>
+    /// Used to hold basic settings for DataX.
+    /// </summary>
+    public class DataXSettings
+    {
+        // The flat settings should be better split into other settings classes
+        // For now, to keep compatibility with ServiceFabric AppManifest, this is flat
+
+        public string CosmosDBConfigConnectionString { get; set; }
+        public string CosmosDBConfigDatabaseName { get; set; }
+        public string CosmosDBConfigCollectionName { get; set; }
+
+        // DataX Settings
+        public bool EnableOneBox { get; set; }
+        public string LocalRoot { get; set; }
+        public string SparkHome { get; set; }
+        public string ClusterName { get; set; }
+        public string ServiceKeyVaultName { get; set; }
+        public string RuntimeKeyVaultName { get; set; }
+        public string MetricEventHubConnectionKey { get; set; }
+        public string ConfigFolderContainerPath { get; set; }
+        public string ConfigFolderHost { get; set; }
+        public string MetricsHttpEndpoint { get; set; }
+        public string AppInsightsIntrumentationKey { get; set; }
+
+        public string MefStorageAccountName { get; set; }
+        public string MefContainerName { get; set; }
+        public string MefBlobDirectory { get; set; }
+    }
+}

Services/DataX.ServiceHost/DataX.ServiceHost/Settings/DataXSettingsConstants.cs

@@ -0,0 +1,16 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace DataX.ServiceHost.Settings
+{
+    /// <summary>
+    /// Constants used with <see cref="DataXSettings"/>
+    /// </summary>
+    public static class DataXSettingsConstants
+    {
+        public const string DataX = nameof(DataX);
+
+        public static readonly string ServiceEnvironment = $"{DataX}:{nameof(ServiceEnvironment)}";
+    }
+}

Services/DataX.Utilities/DataX.Utilities.Telemetry/DataX.Utilities.Telemetry.csproj

@@ -31,6 +31,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\DataX.ServiceHost\DataX.ServiceHost.ServiceFabric\DataX.ServiceHost.ServiceFabric.csproj" />
+    <ProjectReference Include="..\..\DataX.ServiceHost\DataX.ServiceHost\DataX.ServiceHost.csproj" />
     <ProjectReference Include="..\DataX.Utilities.KeyVault\DataX.Utilities.KeyVault.csproj" />
   </ItemGroup>
 

Services/DataX.Utilities/DataX.Utilities.Telemetry/StartUpUtil.cs

@@ -3,6 +3,7 @@
 // Licensed under the MIT License
 // *********************************************************************
 using DataX.ServiceHost.ServiceFabric;
+using DataX.ServiceHost.Settings;
 using Microsoft.ApplicationInsights.AspNetCore.Extensions;
 using Microsoft.ApplicationInsights.Extensibility;
 using Microsoft.AspNetCore.Builder;
@@ -17,36 +18,54 @@ namespace DataX.Utilities.Telemetry
 {
     public static class StartUpUtil
     {
-        public static void ConfigureServices(IServiceCollection services)
+        public static void ConfigureServices(IServiceCollection services, IConfiguration configuration)
         {
-            services.AddSingleton<ITelemetryInitializer, OperationParentIdTelemetryInitializer>();
-            services.AddApplicationInsightsTelemetry(new ApplicationInsightsServiceOptions()
-            {
-                EnableAdaptiveSampling = false,
-                EnableDebugLogger = false,
-                InstrumentationKey = KeyVault.KeyVault.GetSecretFromKeyvault(ServiceFabricUtil.GetServiceKeyVaultName().Result.ToString(), ServiceFabricUtil.GetServiceFabricConfigSetting("AppInsightsIntrumentationKey").Result.ToString())
-            });
-            services.AddSingleton<ITelemetryInitializer, OperationParentIdTelemetryInitializer>();
-            services.AddLogging(logging =>
-            {
-                try
-                {
-                    // In order to log ILogger logs
-                    logging.AddApplicationInsights();
-                    // Optional: Apply filters to configure LogLevel Information or above is sent to
-                    // ApplicationInsights for all categories.
-                    logging.AddFilter<ApplicationInsightsLoggerProvider>("", LogLevel.Information);
+            var settings = configuration.GetValue<DataXSettings>(DataXSettingsConstants.ServiceEnvironment);
 
-                    // Additional filtering For category starting in "Microsoft",
-                    // only Warning or above will be sent to Application Insights.
-                    logging.AddFilter<ApplicationInsightsLoggerProvider>("Microsoft", LogLevel.Warning);
+            ConfigureServices(services, settings ?? new DataXSettings());
+        }
 
-                }
-                catch (Exception e)
+        public static void ConfigureServices(IServiceCollection services, DataXSettings settings)
+        {
+            services
+                .AddSingleton<ITelemetryInitializer, OperationParentIdTelemetryInitializer>()
+                .AddApplicationInsightsTelemetry(new ApplicationInsightsServiceOptions()
                 {
-                    ServiceEventSource.Current.Message($"ApplicationInsights Error: {e.Message}");
-                }
-            });
+                    EnableAdaptiveSampling = false,
+                    EnableDebugLogger = false,
+                    InstrumentationKey = GetInstrumentationKey(settings)
+                })
+                .AddSingleton<ITelemetryInitializer, OperationParentIdTelemetryInitializer>()
+                .AddLogging(logging =>
+                {
+                    try
+                    {
+                        // In order to log ILogger logs
+                        logging.AddApplicationInsights();
+                        // Optional: Apply filters to configure LogLevel Information or above is sent to
+                        // ApplicationInsights for all categories.
+                        logging.AddFilter<ApplicationInsightsLoggerProvider>("", LogLevel.Information);
+
+                        // Additional filtering For category starting in "Microsoft",
+                        // only Warning or above will be sent to Application Insights.
+                        logging.AddFilter<ApplicationInsightsLoggerProvider>("Microsoft", LogLevel.Warning);
+
+                    }
+                    catch (Exception e)
+                    {
+                        ServiceEventSource.Current.Message($"ApplicationInsights Error: {e.Message}");
+                    }
+                });
+        }
+
+        private static string GetInstrumentationKey(DataXSettings settings)
+        {
+            var secretName = settings?.AppInsightsIntrumentationKey;
+            var vaultName = settings.ServiceKeyVaultName;
+
+            return string.IsNullOrWhiteSpace(secretName) || string.IsNullOrWhiteSpace(vaultName)
+                ? Guid.Empty.ToString()
+                : KeyVault.KeyVault.GetSecretFromKeyvault(settings.ServiceKeyVaultName, settings.AppInsightsIntrumentationKey);
         }
     }
 }

Services/DataX.Utilities/DataX.Utilities.Web/DataX.Utilities.Web.csproj

@@ -26,6 +26,7 @@
   <ItemGroup>
     <ProjectReference Include="..\..\DataX.Contract\DataX.Contract.csproj" />
     <ProjectReference Include="..\..\DataX.Gateway\DataX.Gateway.Contract\DataX.Gateway.Contract.csproj" />
+    <ProjectReference Include="..\..\DataX.ServiceHost\DataX.ServiceHost\DataX.ServiceHost.csproj" />
   </ItemGroup>
 
   <ItemGroup>

Services/DataX.Utilities/DataX.Utilities.Web/RolesCheck.cs

@@ -6,6 +6,7 @@ using Microsoft.AspNetCore.Http;
 using DataX.Contract;
 using System;
 using DataX.Gateway.Contract;
+using DataX.ServiceHost;
 
 namespace DataX.Utilities.Web
 {
@@ -25,7 +26,14 @@ namespace DataX.Utilities.Web
         }
 
         public static void EnsureWriter(HttpRequest request)
-        {         
+        {
+            // Ensure* methods only work when auth is handled at the Gateway in Service Fabric setup
+            // Otherwise ASP.NET Core is used and does not require this check
+            if(!HostUtil.InServiceFabric)
+            {
+                return;
+            }
+
             var userrole = request.Headers[Constants.UserRolesHeader];
             Ensure.NotNull(userrole, "userrole");
 
@@ -46,6 +54,13 @@ namespace DataX.Utilities.Web
 
         public static void EnsureReader(HttpRequest request)
         {
+            // Ensure* methods only work when auth is handled at the Gateway in Service Fabric setup
+            // Otherwise ASP.NET Core is used and does not require this check
+            if (!HostUtil.InServiceFabric)
+            {
+                return;
+            }
+
             var userrole = request.Headers[Constants.UserRolesHeader];
             Ensure.NotNull(userrole, "userrole");
 

Services/DataX.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio 15
-VisualStudioVersion = 15.0.28307.271
+# Visual Studio Version 16
+VisualStudioVersion = 16.0.28803.202
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DataX.Contract", "DataX.Contract\DataX.Contract.csproj", "{0F0A8226-E416-4796-B6BB-817B14E25AE4}"
 EndProject
@@ -467,14 +467,6 @@ Global
 		{20E1DF46-301A-4F74-8E76-D29192E368E5}.Release|Any CPU.Build.0 = Release|Any CPU
 		{20E1DF46-301A-4F74-8E76-D29192E368E5}.Release|x64.ActiveCfg = Release|Any CPU
 		{20E1DF46-301A-4F74-8E76-D29192E368E5}.Release|x64.Build.0 = Release|Any CPU
-		{F2EAE040-4DC1-4409-8079-B7AABDCBF718}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{F2EAE040-4DC1-4409-8079-B7AABDCBF718}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{F2EAE040-4DC1-4409-8079-B7AABDCBF718}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{F2EAE040-4DC1-4409-8079-B7AABDCBF718}.Debug|x64.Build.0 = Debug|Any CPU
-		{F2EAE040-4DC1-4409-8079-B7AABDCBF718}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{F2EAE040-4DC1-4409-8079-B7AABDCBF718}.Release|Any CPU.Build.0 = Release|Any CPU
-		{F2EAE040-4DC1-4409-8079-B7AABDCBF718}.Release|x64.ActiveCfg = Release|Any CPU
-		{F2EAE040-4DC1-4409-8079-B7AABDCBF718}.Release|x64.Build.0 = Release|Any CPU
 		{C1CAC657-0322-4275-AE64-EC83FDA8C820}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{C1CAC657-0322-4275-AE64-EC83FDA8C820}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C1CAC657-0322-4275-AE64-EC83FDA8C820}.Debug|x64.ActiveCfg = Debug|Any CPU
@@ -547,7 +539,6 @@ Global
 		{702A1D3B-63AF-4519-8334-600E477D9CBC} = {6FF7998A-7952-444B-AD85-E3A9EB8683C1}
 		{13288F78-76B4-4B9F-B57B-A7415FEEF42F} = {6FF7998A-7952-444B-AD85-E3A9EB8683C1}
 		{20E1DF46-301A-4F74-8E76-D29192E368E5} = {6FF7998A-7952-444B-AD85-E3A9EB8683C1}
-		{F2EAE040-4DC1-4409-8079-B7AABDCBF718} = {6FF7998A-7952-444B-AD85-E3A9EB8683C1}
 		{C1CAC657-0322-4275-AE64-EC83FDA8C820} = {6FF7998A-7952-444B-AD85-E3A9EB8683C1}
 		{9E338FCC-793F-4048-9672-048270C856C4} = {6FF7998A-7952-444B-AD85-E3A9EB8683C1}
 		{D5D3DD2D-760A-4818-A74F-C0061719841C} = {6FF7998A-7952-444B-AD85-E3A9EB8683C1}