e2e tests, samples, docs
This commit is contained in:
Joe Raio
joescars
Родитель
e4fc978784
Коммит
64b6133361
47
README.md
47
README.md
|
|
@ -1,5 +1,50 @@
|
|||
# Azure Data Lake Store Binding for Azure Functions
|
||||
|
||||
# Contributing
|
||||
The following binding can be used on Azure Functions v1 and v2.
|
||||
|
||||
## Instructions
|
||||
|
||||
To the output binding add the following attribute
|
||||
|
||||
```c#
|
||||
[DataLakeStore(AccountFQDN = @"fqdn", ApplicationId = @"applicationid", ClientSecret = @"clientsecret", TenantID = @"tentantid")]out DataLakeStoreOutput dataLakeStoreOutput
|
||||
```
|
||||
|
||||
To use the input binding simple add 'FileName' to bring in a specific file
|
||||
|
||||
```c#
|
||||
[DataLakeStore(AccountFQDN = @"fqdn", ApplicationId = @"applicationid", ClientSecret = @"clientsecret", TenantID = @"tentantid", FileName = "/mydata/testfile.txt")]Stream myfile
|
||||
```
|
||||
|
||||
## Binding Requirements
|
||||
|
||||
1. [Azure Data Lake Store](https://azure.microsoft.com/en-us/services/data-lake-store/)
|
||||
2. Setup [Service to Service Auth](https://docs.microsoft.com/en-us/azure/data-lake-store/data-lake-store-service-to-service-authenticate-using-active-directory) using Azure AD
|
||||
3. [Azure Functions and Webjobs tools](https://marketplace.visualstudio.com/items?itemName=VisualStudioWebandAzureTools.AzureFunctionsandWebJobsTools) extension
|
||||
4. Add the application settings noted below.
|
||||
|
||||
### local.settings.json expected content
|
||||
```
|
||||
{
|
||||
"IsEncrypted": false,
|
||||
"Values": {
|
||||
"AzureWebJobsStorage": "UseDevelopmentStorage=true",
|
||||
"AzureWebJobsDashboard": "UseDevelopmentStorage=true",
|
||||
"fqdn": "<FQDN for your Azure Lake Data Store>",
|
||||
"tentantid": "<Azure Active Directory Tentant for Authentication>",
|
||||
"clientsecret": "<Azure Active Directory Client Secret>",
|
||||
"applicationid": "<Azure Active Directory Application ID>",
|
||||
"blobconn": "<Azure Blobg Storage Connection String for testing Blob Trigger>"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
This project is under the benevolent umbrella of the [.NET Foundation](http://www.dotnetfoundation.org/) and is licensed under [the MIT License](https://github.com/Azure/azure-webjobs-sdk/blob/master/LICENSE.txt)
|
||||
|
||||
|
||||
## Contributing
|
||||
|
||||
This project welcomes contributions and suggestions. Most contributions require you to agree to a
|
||||
Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebJobs.Extensions.DataLake
|
|||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DataLakeExtensionSamples", "samples\DataLakeExtensionSamples\DataLakeExtensionSamples.csproj", "{1FB453B8-5063-4347-9FB0-29B32E5F0198}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WebJobs.Extensions.DataLake.Tests", "test\WebJobs.Extensions.DataLake.Tests\WebJobs.Extensions.DataLake.Tests.csproj", "{47671FFA-CDA5-43BD-9095-1DF5FBF550A4}"
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebJobs.Extensions.DataLake.Tests", "test\WebJobs.Extensions.DataLake.Tests\WebJobs.Extensions.DataLake.Tests.csproj", "{47671FFA-CDA5-43BD-9095-1DF5FBF550A4}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
|
|
|
|||
|
|
@ -0,0 +1,32 @@
|
|||
|
||||
using System.IO;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Azure.WebJobs;
|
||||
using Microsoft.Azure.WebJobs.Extensions.Http;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.Azure.WebJobs.Host;
|
||||
using Newtonsoft.Json;
|
||||
using Microsoft.Azure.WebJobs.Extensions.DataLake;
|
||||
|
||||
namespace DataLakeExtensionSamples
|
||||
{
|
||||
public static class InputSample
|
||||
{
|
||||
[FunctionName("InputSample")]
|
||||
public static IActionResult Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)]HttpRequest req,
|
||||
[DataLakeStore(AccountFQDN = "%fqdn%", ApplicationId = "%applicationid%", ClientSecret = "%clientsecret%", TenantID = "%tentantid%", FileName = "/mydata/testfile.txt")]Stream myfile,
|
||||
TraceWriter log)
|
||||
{
|
||||
log.Info("C# HTTP trigger function processed a request.");
|
||||
|
||||
// Example assuming text file
|
||||
using (var reader = new StreamReader(myfile))
|
||||
{
|
||||
var contents = reader.ReadToEnd();
|
||||
log.Info(contents);
|
||||
return new OkObjectResult(contents);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
using System.IO;
|
||||
using Microsoft.Azure.WebJobs;
|
||||
using Microsoft.Azure.WebJobs.Extensions.DataLake;
|
||||
using Microsoft.Azure.WebJobs.Host;
|
||||
|
||||
namespace DataLakeExtensionSamples
|
||||
{
|
||||
public static class OutputFromBlob
|
||||
{
|
||||
[FunctionName("OutputFromBlob")]
|
||||
public static void Run([BlobTrigger("stuff/{name}", Connection = "blobconn")]Stream myBlob, string name,
|
||||
[DataLakeStore(AccountFQDN = "%fqdn%", ApplicationId = "%applicationid%", ClientSecret = "%clientsecret%", TenantID = "%tentantid%")]out DataLakeStoreOutput dataLakeStoreOutput,
|
||||
TraceWriter log)
|
||||
{
|
||||
log.Info($"C# Blob trigger function Processed blob\n Name:{name} \n Size: {myBlob.Length} Bytes");
|
||||
|
||||
dataLakeStoreOutput = new DataLakeStoreOutput()
|
||||
{
|
||||
FileName = "/mydata/" + name,
|
||||
FileStream = myBlob
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -7,19 +7,20 @@ using Microsoft.AspNetCore.Http;
|
|||
using Microsoft.Azure.WebJobs.Host;
|
||||
using Newtonsoft.Json;
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace DataLakeExtensionSamples
|
||||
{
|
||||
public static class OutputFromHttp
|
||||
{
|
||||
[FunctionName("OutputFromHttp")]
|
||||
public static IActionResult Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)]HttpRequest req,
|
||||
[DataLakeStore(AccountFQDN = @"fqdn", ApplicationId = @"applicationid", ClientSecret = @"clientsecret", TenantID = @"tentantid")]IAsyncCollector<DataLakeStoreOutput> items,
|
||||
public static async Task<IActionResult> RunAsync([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)]HttpRequest req,
|
||||
[DataLakeStore(AccountFQDN = "%fqdn%", ApplicationId = "%applicationid%", ClientSecret = "%clientsecret%", TenantID = "%tentantid%")]IAsyncCollector<DataLakeStoreOutput> items,
|
||||
TraceWriter log)
|
||||
{
|
||||
log.Info("C# HTTP trigger function processed a request.");
|
||||
|
||||
items.AddAsync(new DataLakeStoreOutput()
|
||||
await items.AddAsync(new DataLakeStoreOutput()
|
||||
{
|
||||
FileName = "/mydata/" + Guid.NewGuid().ToString() + ".txt",
|
||||
FileStream = req.Body
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ namespace Microsoft.Azure.WebJobs.Extensions.DataLake
|
|||
{
|
||||
/// <summary></summary>
|
||||
/// <seealso cref="Attribute" />
|
||||
[AttributeUsage(validOn: AttributeTargets.ReturnValue | AttributeTargets.Parameter)]
|
||||
//[AttributeUsage(validOn: AttributeTargets.ReturnValue | AttributeTargets.Parameter)]
|
||||
[Binding]
|
||||
public sealed class DataLakeStoreAttribute : Attribute
|
||||
{
|
||||
|
|
@ -17,7 +17,7 @@ namespace Microsoft.Azure.WebJobs.Extensions.DataLake
|
|||
/// <value>
|
||||
/// The ApplicationId also Known as ClientID setting.
|
||||
/// </value>
|
||||
[AppSetting]
|
||||
[AutoResolve]
|
||||
public string ApplicationId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -26,7 +26,7 @@ namespace Microsoft.Azure.WebJobs.Extensions.DataLake
|
|||
/// <value>
|
||||
/// The Client Secret setting.
|
||||
/// </value>
|
||||
[AppSetting]
|
||||
[AutoResolve]
|
||||
public string ClientSecret { get; set; }
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -35,7 +35,7 @@ namespace Microsoft.Azure.WebJobs.Extensions.DataLake
|
|||
/// <value>
|
||||
/// The TenantID setting.
|
||||
/// </value>
|
||||
[AppSetting]
|
||||
[AutoResolve]
|
||||
public string TenantID { get; set; }
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -44,7 +44,7 @@ namespace Microsoft.Azure.WebJobs.Extensions.DataLake
|
|||
/// <value>
|
||||
/// The DataLake full account FQDN setting.
|
||||
/// </value>
|
||||
[AppSetting]
|
||||
[AutoResolve]
|
||||
public string AccountFQDN { get; set; }
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -53,6 +53,7 @@ namespace Microsoft.Azure.WebJobs.Extensions.DataLake
|
|||
/// <value>
|
||||
/// The full path and filename for input binding.
|
||||
/// </value>
|
||||
[AutoResolve]
|
||||
public string FileName { get; set; }
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
using System.IO;
|
||||
using System;
|
||||
using System.IO;
|
||||
using Microsoft.Azure.WebJobs.Description;
|
||||
using Microsoft.Azure.WebJobs.Host.Config;
|
||||
|
||||
namespace Microsoft.Azure.WebJobs.Extensions.DataLake
|
||||
{
|
||||
public class DataLakeConfig : IExtensionConfigProvider
|
||||
{
|
||||
public class DataLakeStoreConfig : IExtensionConfigProvider
|
||||
{
|
||||
|
||||
public void Initialize(ExtensionConfigContext context)
|
||||
{
|
||||
|
||||
|
|
@ -12,12 +15,17 @@ namespace Microsoft.Azure.WebJobs.Extensions.DataLake
|
|||
|
||||
// Output binding for DataLakeStore
|
||||
rule.WhenIsNull(nameof(DataLakeStoreAttribute.FileName))
|
||||
.BindToCollector(a => new DataLakeStoreOutputAsyncCollector(a));
|
||||
.BindToCollector<DataLakeStoreOutput>(BuildCollector);
|
||||
|
||||
// Input binding for DataLakeStore
|
||||
rule.WhenIsNotNull(nameof(DataLakeStoreAttribute.FileName))
|
||||
.BindToInput<Stream>(typeof(DataLakeStoreStreamBuilder));
|
||||
|
||||
}
|
||||
|
||||
private IAsyncCollector<DataLakeStoreOutput> BuildCollector(DataLakeStoreAttribute attribute)
|
||||
{
|
||||
return new DataLakeStoreOutputAsyncCollector(attribute);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,8 +11,6 @@ namespace Microsoft.Azure.WebJobs.Extensions.DataLake
|
|||
internal class DataLakeStoreOutputAsyncCollector : IAsyncCollector<DataLakeStoreOutput>
|
||||
{
|
||||
private static AdlsClient _adlsClient;
|
||||
private readonly Collection<DataLakeStoreOutput> _items = new Collection<DataLakeStoreOutput>();
|
||||
|
||||
private DataLakeStoreAttribute _attribute;
|
||||
|
||||
public DataLakeStoreOutputAsyncCollector(DataLakeStoreAttribute attr)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFrameworks>netstandard2.0;net461</TargetFrameworks>
|
||||
<TargetFramework>netstandard2.0</TargetFramework>
|
||||
<AssemblyName>Microsoft.Azure.WebJobs.Extensions.DataLake</AssemblyName>
|
||||
<RootNamespace>Microsoft.Azure.WebJobs.Extensions.DataLake</RootNamespace>
|
||||
<Company>Microsoft Corporation</Company>
|
||||
|
|
@ -10,13 +10,6 @@
|
|||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Azure.DataLake.Store" Version="1.1.4" />
|
||||
<PackageReference Include="Microsoft.Rest.ClientRuntime.Azure.Authentication" Version="2.3.3" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup Condition="'$(TargetFramework)' == 'net461'">
|
||||
<PackageReference Include="Microsoft.Azure.WebJobs" Version="2.2.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
|
||||
<PackageReference Include="Microsoft.Azure.WebJobs" Version="3.0.0-beta5" />
|
||||
</ItemGroup>
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,264 @@
|
|||
## Ignore Visual Studio temporary files, build results, and
|
||||
## files generated by popular Visual Studio add-ons.
|
||||
|
||||
# Azure Functions localsettings file
|
||||
appsettings.json
|
||||
|
||||
# User-specific files
|
||||
*.suo
|
||||
*.user
|
||||
*.userosscache
|
||||
*.sln.docstates
|
||||
|
||||
# User-specific files (MonoDevelop/Xamarin Studio)
|
||||
*.userprefs
|
||||
|
||||
# Build results
|
||||
[Dd]ebug/
|
||||
[Dd]ebugPublic/
|
||||
[Rr]elease/
|
||||
[Rr]eleases/
|
||||
x64/
|
||||
x86/
|
||||
bld/
|
||||
[Bb]in/
|
||||
[Oo]bj/
|
||||
[Ll]og/
|
||||
|
||||
# Visual Studio 2015 cache/options directory
|
||||
.vs/
|
||||
# Uncomment if you have tasks that create the project's static files in wwwroot
|
||||
#wwwroot/
|
||||
|
||||
# MSTest test Results
|
||||
[Tt]est[Rr]esult*/
|
||||
[Bb]uild[Ll]og.*
|
||||
|
||||
# NUNIT
|
||||
*.VisualState.xml
|
||||
TestResult.xml
|
||||
|
||||
# Build Results of an ATL Project
|
||||
[Dd]ebugPS/
|
||||
[Rr]eleasePS/
|
||||
dlldata.c
|
||||
|
||||
# DNX
|
||||
project.lock.json
|
||||
project.fragment.lock.json
|
||||
artifacts/
|
||||
|
||||
*_i.c
|
||||
*_p.c
|
||||
*_i.h
|
||||
*.ilk
|
||||
*.meta
|
||||
*.obj
|
||||
*.pch
|
||||
*.pdb
|
||||
*.pgc
|
||||
*.pgd
|
||||
*.rsp
|
||||
*.sbr
|
||||
*.tlb
|
||||
*.tli
|
||||
*.tlh
|
||||
*.tmp
|
||||
*.tmp_proj
|
||||
*.log
|
||||
*.vspscc
|
||||
*.vssscc
|
||||
.builds
|
||||
*.pidb
|
||||
*.svclog
|
||||
*.scc
|
||||
|
||||
# Chutzpah Test files
|
||||
_Chutzpah*
|
||||
|
||||
# Visual C++ cache files
|
||||
ipch/
|
||||
*.aps
|
||||
*.ncb
|
||||
*.opendb
|
||||
*.opensdf
|
||||
*.sdf
|
||||
*.cachefile
|
||||
*.VC.db
|
||||
*.VC.VC.opendb
|
||||
|
||||
# Visual Studio profiler
|
||||
*.psess
|
||||
*.vsp
|
||||
*.vspx
|
||||
*.sap
|
||||
|
||||
# TFS 2012 Local Workspace
|
||||
$tf/
|
||||
|
||||
# Guidance Automation Toolkit
|
||||
*.gpState
|
||||
|
||||
# ReSharper is a .NET coding add-in
|
||||
_ReSharper*/
|
||||
*.[Rr]e[Ss]harper
|
||||
*.DotSettings.user
|
||||
|
||||
# JustCode is a .NET coding add-in
|
||||
.JustCode
|
||||
|
||||
# TeamCity is a build add-in
|
||||
_TeamCity*
|
||||
|
||||
# DotCover is a Code Coverage Tool
|
||||
*.dotCover
|
||||
|
||||
# NCrunch
|
||||
_NCrunch_*
|
||||
.*crunch*.local.xml
|
||||
nCrunchTemp_*
|
||||
|
||||
# MightyMoose
|
||||
*.mm.*
|
||||
AutoTest.Net/
|
||||
|
||||
# Web workbench (sass)
|
||||
.sass-cache/
|
||||
|
||||
# Installshield output folder
|
||||
[Ee]xpress/
|
||||
|
||||
# DocProject is a documentation generator add-in
|
||||
DocProject/buildhelp/
|
||||
DocProject/Help/*.HxT
|
||||
DocProject/Help/*.HxC
|
||||
DocProject/Help/*.hhc
|
||||
DocProject/Help/*.hhk
|
||||
DocProject/Help/*.hhp
|
||||
DocProject/Help/Html2
|
||||
DocProject/Help/html
|
||||
|
||||
# Click-Once directory
|
||||
publish/
|
||||
|
||||
# Publish Web Output
|
||||
*.[Pp]ublish.xml
|
||||
*.azurePubxml
|
||||
# TODO: Comment the next line if you want to checkin your web deploy settings
|
||||
# but database connection strings (with potential passwords) will be unencrypted
|
||||
#*.pubxml
|
||||
*.publishproj
|
||||
|
||||
# Microsoft Azure Web App publish settings. Comment the next line if you want to
|
||||
# checkin your Azure Web App publish settings, but sensitive information contained
|
||||
# in these scripts will be unencrypted
|
||||
PublishScripts/
|
||||
|
||||
# NuGet Packages
|
||||
*.nupkg
|
||||
# The packages folder can be ignored because of Package Restore
|
||||
**/packages/*
|
||||
# except build/, which is used as an MSBuild target.
|
||||
!**/packages/build/
|
||||
# Uncomment if necessary however generally it will be regenerated when needed
|
||||
#!**/packages/repositories.config
|
||||
# NuGet v3's project.json files produces more ignoreable files
|
||||
*.nuget.props
|
||||
*.nuget.targets
|
||||
|
||||
# Microsoft Azure Build Output
|
||||
csx/
|
||||
*.build.csdef
|
||||
|
||||
# Microsoft Azure Emulator
|
||||
ecf/
|
||||
rcf/
|
||||
|
||||
# Windows Store app package directories and files
|
||||
AppPackages/
|
||||
BundleArtifacts/
|
||||
Package.StoreAssociation.xml
|
||||
_pkginfo.txt
|
||||
|
||||
# Visual Studio cache files
|
||||
# files ending in .cache can be ignored
|
||||
*.[Cc]ache
|
||||
# but keep track of directories ending in .cache
|
||||
!*.[Cc]ache/
|
||||
|
||||
# Others
|
||||
ClientBin/
|
||||
~$*
|
||||
*~
|
||||
*.dbmdl
|
||||
*.dbproj.schemaview
|
||||
*.jfm
|
||||
*.pfx
|
||||
*.publishsettings
|
||||
node_modules/
|
||||
orleans.codegen.cs
|
||||
|
||||
# Since there are multiple workflows, uncomment next line to ignore bower_components
|
||||
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
|
||||
#bower_components/
|
||||
|
||||
# RIA/Silverlight projects
|
||||
Generated_Code/
|
||||
|
||||
# Backup & report files from converting an old project file
|
||||
# to a newer Visual Studio version. Backup files are not needed,
|
||||
# because we have git ;-)
|
||||
_UpgradeReport_Files/
|
||||
Backup*/
|
||||
UpgradeLog*.XML
|
||||
UpgradeLog*.htm
|
||||
|
||||
# SQL Server files
|
||||
*.mdf
|
||||
*.ldf
|
||||
|
||||
# Business Intelligence projects
|
||||
*.rdl.data
|
||||
*.bim.layout
|
||||
*.bim_*.settings
|
||||
|
||||
# Microsoft Fakes
|
||||
FakesAssemblies/
|
||||
|
||||
# GhostDoc plugin setting file
|
||||
*.GhostDoc.xml
|
||||
|
||||
# Node.js Tools for Visual Studio
|
||||
.ntvs_analysis.dat
|
||||
|
||||
# Visual Studio 6 build log
|
||||
*.plg
|
||||
|
||||
# Visual Studio 6 workspace options file
|
||||
*.opt
|
||||
|
||||
# Visual Studio LightSwitch build output
|
||||
**/*.HTMLClient/GeneratedArtifacts
|
||||
**/*.DesktopClient/GeneratedArtifacts
|
||||
**/*.DesktopClient/ModelManifest.xml
|
||||
**/*.Server/GeneratedArtifacts
|
||||
**/*.Server/ModelManifest.xml
|
||||
_Pvt_Extensions
|
||||
|
||||
# Paket dependency manager
|
||||
.paket/paket.exe
|
||||
paket-files/
|
||||
|
||||
# FAKE - F# Make
|
||||
.fake/
|
||||
|
||||
# JetBrains Rider
|
||||
.idea/
|
||||
*.sln.iml
|
||||
|
||||
# CodeRush
|
||||
.cr/
|
||||
|
||||
# Python Tools for Visual Studio (PTVS)
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
|
||||
namespace WebJobs.Extensions.DataLake.Tests
|
||||
{
|
||||
public static class FakeData
|
||||
{
|
||||
public const string SamplePayload = @"{'testkey': 'testvalue'}";
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
using Microsoft.Azure.WebJobs;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
|
||||
namespace WebJobs.Extensions.DataLake.Tests
|
||||
{ public class FakeTypeLocator<T> : ITypeLocator
|
||||
{
|
||||
public IReadOnlyList<Type> GetTypes()
|
||||
{
|
||||
return new Type[] { typeof(T) };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
using Microsoft.Extensions.Configuration;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
|
||||
namespace WebJobs.Extensions.DataLake.Tests
|
||||
{
|
||||
public class LocalRuntimeFixture : IDisposable
|
||||
{
|
||||
public IConfiguration _config;
|
||||
|
||||
public LocalRuntimeFixture()
|
||||
{
|
||||
_config = InitConfiguration();
|
||||
}
|
||||
public void Dispose()
|
||||
{
|
||||
_config = null;
|
||||
}
|
||||
|
||||
public string Resolve(string name)
|
||||
{
|
||||
return _config[name].ToString();
|
||||
}
|
||||
|
||||
public static IConfiguration InitConfiguration()
|
||||
{
|
||||
var config = new ConfigurationBuilder()
|
||||
.AddJsonFile("appsettings.test.json")
|
||||
.Build();
|
||||
return config;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
using Microsoft.Azure.WebJobs.Description;
|
||||
using Microsoft.Azure.WebJobs.Host.Config;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
|
||||
namespace WebJobs.Extensions.DataLake.Tests
|
||||
{
|
||||
[Binding]
|
||||
public class BindingDataAttribute : Attribute
|
||||
{
|
||||
public BindingDataAttribute(string toBeAutoResolve)
|
||||
{
|
||||
ToBeAutoResolve = toBeAutoResolve;
|
||||
}
|
||||
|
||||
[AutoResolve]
|
||||
public string ToBeAutoResolve { get; set; }
|
||||
}
|
||||
|
||||
public class TestExtensionConfig : IExtensionConfigProvider
|
||||
{
|
||||
public void Initialize(ExtensionConfigContext context)
|
||||
{
|
||||
context.AddBindingRule<BindingDataAttribute>().
|
||||
BindToInput<string>(attr => attr.ToBeAutoResolve);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
using Microsoft.Azure.WebJobs;
|
||||
using Microsoft.Azure.WebJobs.Extensions.DataLake;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Configuration;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
|
||||
namespace WebJobs.Extensions.DataLake.Tests
|
||||
{
|
||||
public class TestHelpers
|
||||
{
|
||||
public static JobHost NewHost<T>(DataLakeStoreConfig ext = null)
|
||||
{
|
||||
//TODO: pass configuration / attributes to DataLakeStoreConfig
|
||||
JobHostConfiguration config = new JobHostConfiguration();
|
||||
config.HostId = Guid.NewGuid().ToString("n");
|
||||
config.StorageConnectionString = "UseDevelopmentStorage=true";
|
||||
config.DashboardConnectionString = "UseDevelopmentStorage=true";
|
||||
config.TypeLocator = new FakeTypeLocator<T>();
|
||||
config.AddExtension(ext ?? new DataLakeStoreConfig());
|
||||
var host = new JobHost(config);
|
||||
return host;
|
||||
}
|
||||
|
||||
public static Stream GenerateStreamFromString(string s)
|
||||
{
|
||||
var stream = new MemoryStream();
|
||||
var writer = new StreamWriter(stream);
|
||||
writer.Write(s);
|
||||
writer.Flush();
|
||||
stream.Position = 0;
|
||||
return stream;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using Xunit;
|
||||
using Microsoft.Azure.WebJobs.Extensions.DataLake;
|
||||
|
||||
namespace WebJobs.Extensions.DataLake.Tests
|
||||
{
|
||||
public class DataLakeStoreAttributeTests
|
||||
{
|
||||
private const string _AccountFQDN = "[FQDN]";
|
||||
private const string _ApplicationId = "[APPLICATIONID]";
|
||||
private const string _ClientSecret = "[SECRET]";
|
||||
private const string _TenantID = "[TENTANT]";
|
||||
private const string _FileName = "[FILENAME]";
|
||||
|
||||
[Fact]
|
||||
public void CompleteArguments_Succeeded()
|
||||
{
|
||||
DataLakeStoreAttribute dlsa = new DataLakeStoreAttribute()
|
||||
{
|
||||
AccountFQDN = _AccountFQDN,
|
||||
ApplicationId = _ApplicationId,
|
||||
ClientSecret = _ClientSecret,
|
||||
TenantID = _TenantID,
|
||||
FileName = _FileName
|
||||
};
|
||||
|
||||
Assert.Equal(_AccountFQDN, dlsa.AccountFQDN);
|
||||
Assert.Equal(_ApplicationId, dlsa.ApplicationId);
|
||||
Assert.Equal(_ClientSecret, dlsa.ClientSecret);
|
||||
Assert.Equal(_TenantID, dlsa.TenantID);
|
||||
Assert.Equal(_FileName, dlsa.FileName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Azure.WebJobs;
|
||||
using Microsoft.Azure.WebJobs.Extensions.DataLake;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Xunit;
|
||||
|
||||
namespace WebJobs.Extensions.DataLake.Tests
|
||||
{
|
||||
public class DataLakeStoreEndToEnd
|
||||
{
|
||||
// be sure to include path to writeable folder
|
||||
private static string testFileName = $"/mydata/{Guid.NewGuid().ToString()}.txt";
|
||||
private static string functionOut = null;
|
||||
|
||||
[Fact]
|
||||
public async Task DataLakeStoreTests()
|
||||
{
|
||||
//Dummy data to use later
|
||||
JObject testData = JObject.Parse(FakeData.SamplePayload);
|
||||
var args = new Dictionary<string, object>{
|
||||
{ "fileName", testFileName }
|
||||
};
|
||||
|
||||
// host to run test function
|
||||
var host = TestHelpers.NewHost<MyProg1>();
|
||||
|
||||
// make sure we can write the file to data lake store
|
||||
await host.CallAsync("MyProg1.TestCollector", args);
|
||||
Assert.Equal("success", functionOut);
|
||||
functionOut = null;
|
||||
|
||||
// retrieve that same file and make sure contest are the same
|
||||
await host.CallAsync("MyProg1.TestInputBinding", args);
|
||||
Assert.Equal(FakeData.SamplePayload, functionOut);
|
||||
functionOut = null;
|
||||
|
||||
}
|
||||
|
||||
public class MyProg1
|
||||
{
|
||||
public async Task TestCollector(
|
||||
[DataLakeStore(
|
||||
AccountFQDN = "%fqdn%",
|
||||
ApplicationId = "%applicationid%",
|
||||
ClientSecret = "%clientsecret%",
|
||||
TenantID = "%tentantid%")]IAsyncCollector<DataLakeStoreOutput> items, string fileName)
|
||||
{
|
||||
using (var stream = TestHelpers.GenerateStreamFromString(FakeData.SamplePayload))
|
||||
{
|
||||
await items.AddAsync(new DataLakeStoreOutput()
|
||||
{
|
||||
FileName = fileName,
|
||||
FileStream = stream
|
||||
});
|
||||
}
|
||||
|
||||
functionOut = "success";
|
||||
}
|
||||
|
||||
public async Task TestInputBinding(
|
||||
[DataLakeStore(
|
||||
AccountFQDN = "%fqdn%",
|
||||
ApplicationId = "%applicationid%",
|
||||
ClientSecret = "%clientsecret%",
|
||||
TenantID = "%tentantid%",
|
||||
FileName = "{fileName}")]Stream myfile)
|
||||
{
|
||||
using (var reader = new StreamReader(myfile))
|
||||
{
|
||||
var contents = await reader.ReadToEndAsync();
|
||||
functionOut = contents;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using Xunit;
|
||||
using Microsoft.Azure.WebJobs.Extensions.DataLake;
|
||||
using Microsoft.Azure.WebJobs;
|
||||
using System.IO;
|
||||
|
||||
namespace WebJobs.Extensions.DataLake.Tests
|
||||
{
|
||||
public class DataLakeStoreTests
|
||||
{
|
||||
private const string _AccountFQDN = "[FQDN]";
|
||||
private const string _ApplicationId = "[APPLICATIONID]";
|
||||
private const string _ClientSecret = "[SECRET]";
|
||||
private const string _TenantID = "[TENTANT]";
|
||||
private const string _FileName = "[FILENAME]";
|
||||
private const string contents = "test content 1 2 3";
|
||||
[Fact]
|
||||
public void EchoTest()
|
||||
{
|
||||
var result = TestFunctions.EchoTest("John");
|
||||
Assert.Equal("Hello John!", result);
|
||||
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OutputItemTest()
|
||||
{
|
||||
// convert string to stream
|
||||
byte[] byteArray = Encoding.UTF8.GetBytes(contents);
|
||||
MemoryStream stream = new MemoryStream(byteArray);
|
||||
var result = TestFunctions.TestOutput(stream, new DataLakeStoreOutput());
|
||||
Assert.Same(stream, result.FileStream);
|
||||
}
|
||||
|
||||
public static class TestFunctions
|
||||
{
|
||||
[FunctionName("EchoTest")]
|
||||
public static string EchoTest(string name)
|
||||
{
|
||||
return $"Hello {name}!";
|
||||
}
|
||||
|
||||
[FunctionName("TestOutput")]
|
||||
public static DataLakeStoreOutput TestOutput(
|
||||
Stream myStream,
|
||||
[DataLakeStore(
|
||||
AccountFQDN = _AccountFQDN,
|
||||
ApplicationId = _ApplicationId,
|
||||
ClientSecret = _ClientSecret,
|
||||
TenantID = _TenantID)]DataLakeStoreOutput item)
|
||||
{
|
||||
var d = new DataLakeStoreOutput()
|
||||
{
|
||||
FileName = "/mydata/" + Guid.NewGuid().ToString() + ".txt",
|
||||
FileStream = myStream
|
||||
};
|
||||
return d;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -9,6 +9,7 @@
|
|||
<PackageReference Include="Microsoft.Azure.WebJobs.Logging.ApplicationInsights" Version="3.0.0-beta5" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.7.0" />
|
||||
<PackageReference Include="Moq" Version="4.7.145" />
|
||||
<PackageReference Include="System.Configuration.ConfigurationManager" Version="4.5.0" />
|
||||
<PackageReference Include="xunit" Version="2.3.1" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.3.1" />
|
||||
</ItemGroup>
|
||||
|
|
@ -17,4 +18,10 @@
|
|||
<ProjectReference Include="..\..\src\WebJobs.Extensions.DataLake\WebJobs.Extensions.DataLake.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Update="appsettings.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче