Adds backup, restore,archive, and docs
This commit is contained in:
Dave Curylo
Родитель
7f593cc5bc
Коммит
62698755af
|
|
@ -0,0 +1,23 @@
|
|||
|
||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||
#
|
||||
Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "AzureTableArchive", "AzureTableArchive\AzureTableArchive.fsproj", "{A295CD96-7607-46DA-9C43-7E81DFD0309E}"
|
||||
EndProject
|
||||
Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "AzureTableArchiveTests", "AzureTableArchiveTests\AzureTableArchiveTests.fsproj", "{F0368061-B1F5-4BA3-BDE8-D77433F23EBB}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
Release|Any CPU = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||
{A295CD96-7607-46DA-9C43-7E81DFD0309E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{A295CD96-7607-46DA-9C43-7E81DFD0309E}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{A295CD96-7607-46DA-9C43-7E81DFD0309E}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{A295CD96-7607-46DA-9C43-7E81DFD0309E}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{F0368061-B1F5-4BA3-BDE8-D77433F23EBB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{F0368061-B1F5-4BA3-BDE8-D77433F23EBB}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{F0368061-B1F5-4BA3-BDE8-D77433F23EBB}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{F0368061-B1F5-4BA3-BDE8-D77433F23EBB}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFrameworks>net5.0;netstandard2.1</TargetFrameworks>
|
||||
<RootNamespace>StorageTableBackup</RootNamespace>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Compile Include="Domain.fs" />
|
||||
<Compile Include="EntitySerialization.fs" />
|
||||
<Compile Include="Backup.fs" />
|
||||
<Compile Include="ContainerSync.fs" />
|
||||
<Compile Include="Restore.fs" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Azure.Cosmos.Table" Version="1.0.8" />
|
||||
<PackageReference Include="Microsoft.Azure.Storage.DataMovement" Version="2.0.1" />
|
||||
<PackageReference Include="System.Text.Json" Version="5.0.1" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
namespace AzureTableArchive
|
||||
|
||||
open System
|
||||
open System.IO
|
||||
open Microsoft.Azure.Cosmos.Table
|
||||
open EntitySerialization
|
||||
|
||||
module Backup =
|
||||
|
||||
/// Use a generated filepath.
|
||||
let GeneratedBackupPath = Option<FilePath>.None
|
||||
|
||||
let backupTables : BackupTables =
|
||||
fun listTables dumpTable backupPath ->
|
||||
async {
|
||||
let! tables = listTables
|
||||
let backupPath = backupPath |> Option.defaultValue (Path.Combine (Path.GetTempPath (), Guid.NewGuid().ToString()))
|
||||
do! tables
|
||||
|> Seq.map (fun table -> (table, backupPath) |> dumpTable)
|
||||
|> Async.Parallel // Download and write backups of all tables in parallel
|
||||
|> Async.Ignore
|
||||
return backupPath
|
||||
}
|
||||
|
||||
let listStorageTables (cloudTableClient:CloudTableClient) : ListTables =
|
||||
async {
|
||||
let rec getTables (continuationToken) (acc:CloudTable list) =
|
||||
async {
|
||||
let! tablesResult = cloudTableClient.ListTablesSegmentedAsync(continuationToken) |> Async.AwaitTask
|
||||
if isNull tablesResult.ContinuationToken then
|
||||
return acc @ (List.ofSeq tablesResult.Results)
|
||||
else
|
||||
return! getTables tablesResult.ContinuationToken (acc @ List.ofSeq tablesResult.Results)
|
||||
}
|
||||
let! allTables = getTables null []
|
||||
return allTables |> List.map (fun t -> t.Name)
|
||||
}
|
||||
|
||||
let dumpTableToJsonFiles (cloudTableClient:CloudTableClient) : DumpTable =
|
||||
fun (tableName, filePath) ->
|
||||
async {
|
||||
let cloudTable = cloudTableClient.GetTableReference tableName
|
||||
let rec queryAndWrite (continuationToken:TableContinuationToken) =
|
||||
async {
|
||||
let! queryResult = cloudTable.ExecuteQuerySegmentedAsync(TableQuery<DynamicTableEntity>(), continuationToken) |> Async.AwaitTask
|
||||
for entity in queryResult.Results do
|
||||
let filename = System.IO.Path.Combine(filePath, tableName, entity.PartitionKey, $"{entity.RowKey}.json")
|
||||
System.IO.Directory.CreateDirectory(System.IO.Path.Combine(filePath, tableName, entity.PartitionKey)) |> ignore
|
||||
do! System.IO.File.WriteAllTextAsync(filename, entity.ToJson()) |> Async.AwaitTask
|
||||
if not <| isNull queryResult.ContinuationToken then
|
||||
do! queryAndWrite queryResult.ContinuationToken
|
||||
}
|
||||
do! queryAndWrite null
|
||||
}
|
||||
|
||||
/// Backs tables up to a generated directory in the temporary folder.
|
||||
let BackupTables (tableClient:CloudTableClient) =
|
||||
backupTables (listStorageTables tableClient) (dumpTableToJsonFiles tableClient) GeneratedBackupPath
|
||||
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
namespace AzureTableArchive
|
||||
|
||||
open System
|
||||
open Microsoft.Azure.Storage.Blob
|
||||
open Microsoft.Azure.Storage.DataMovement
|
||||
|
||||
module ContainerSync =
|
||||
|
||||
let syncToContainer (blobClient:CloudBlobClient) (reportProgress:Progress<TransferStatus> option) : SyncBackupToStorage =
|
||||
fun backupLocation ->
|
||||
async {
|
||||
let container = blobClient.GetContainerReference backupLocation.BackupContainer
|
||||
let! _ = container.CreateIfNotExistsAsync () |> Async.AwaitTask
|
||||
let blobDirectoryName = backupLocation.ArchiveName
|
||||
let blobDirectory = container.GetDirectoryReference blobDirectoryName
|
||||
let transferCheckpoint = Unchecked.defaultof<TransferCheckpoint>
|
||||
let context = DirectoryTransferContext(transferCheckpoint)
|
||||
reportProgress |> Option.iter (fun progress -> context.ProgressHandler <- progress)
|
||||
let options = UploadDirectoryOptions(Recursive = true)
|
||||
let! transferStatus = TransferManager.UploadDirectoryAsync (backupLocation.BackupPath, blobDirectory, options, context) |> Async.AwaitTask
|
||||
if transferStatus.NumberOfFilesFailed > 0L then
|
||||
return Result.Error (FilesFailedToTransfer transferStatus.NumberOfFilesFailed)
|
||||
else
|
||||
return Ok ()
|
||||
}
|
||||
|
||||
let syncFromContainer (blobClient:CloudBlobClient) (reportProgress:Progress<TransferStatus> option) : SyncStorageToRestore =
|
||||
fun (restoreLocation:RestoreLocation) ->
|
||||
async {
|
||||
let container = blobClient.GetContainerReference restoreLocation.BackupContainer
|
||||
do! container.CreateIfNotExistsAsync () |> Async.AwaitTask |> Async.Ignore
|
||||
let blobDirectory = container.GetDirectoryReference restoreLocation.ArchiveName
|
||||
let transferCheckpoint = Unchecked.defaultof<TransferCheckpoint>
|
||||
let context = DirectoryTransferContext(transferCheckpoint)
|
||||
reportProgress |> Option.iter (fun progress -> context.ProgressHandler <- progress)
|
||||
let options = DownloadDirectoryOptions(Recursive = true)
|
||||
let! transferStatus = TransferManager.DownloadDirectoryAsync (blobDirectory, restoreLocation.RestorePath, options, context) |> Async.AwaitTask
|
||||
if transferStatus.NumberOfFilesFailed > 0L then
|
||||
return Result.Error (FilesFailedToTransfer transferStatus.NumberOfFilesFailed)
|
||||
else
|
||||
return Result.Ok ()
|
||||
}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
namespace AzureTableArchive
|
||||
|
||||
type TableName = string
|
||||
/// Lists the storage tables in a storage account.
|
||||
type ListTables = Async<TableName list>
|
||||
|
||||
type FilePath = string
|
||||
|
||||
/// Dumps a storage table to a list of files.
|
||||
type DumpTable = TableName * FilePath -> Async<unit>
|
||||
|
||||
/// Creates a backup of the tables, generating a FilePath if one is not provided.
|
||||
type BackupTables = ListTables -> DumpTable -> FilePath option -> Async<FilePath>
|
||||
|
||||
/// Restores tables from a backup.
|
||||
type RestoreTables = FilePath -> Async<unit>
|
||||
|
||||
/// Name of the backup.
|
||||
type BackupName = string
|
||||
|
||||
/// The remote and local location for the backup files.
|
||||
type BackupLocation =
|
||||
{ BackupPath : string
|
||||
BackupContainer : string
|
||||
ArchiveName : string }
|
||||
|
||||
/// Synchronization failures in synchronizing local backup to/from storage account.
|
||||
type SynchronizationFailures =
|
||||
| FilesFailedToTransfer of NumberOfFilesFailed:int64
|
||||
|
||||
/// Sync the remote and local storage the backup, returning the unique backup name.
|
||||
type SyncBackupToStorage = BackupLocation -> Async<Result<unit,SynchronizationFailures>>
|
||||
|
||||
/// The remote and local restore location.
|
||||
type RestoreLocation =
|
||||
{ RestorePath : string
|
||||
BackupContainer : string
|
||||
ArchiveName : string }
|
||||
|
||||
/// Sync the remote and local restore location for the backup name.
|
||||
type SyncStorageToRestore = RestoreLocation -> Async<Result<unit,SynchronizationFailures>>
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
namespace AzureTableArchive
|
||||
|
||||
open System
|
||||
open Microsoft.Azure.Cosmos.Table
|
||||
|
||||
module EntitySerialization =
|
||||
|
||||
/// JSON serializable view of an EntityProperty.
|
||||
type PropertyValue =
|
||||
{ EdmType : EdmType
|
||||
StringValue : string
|
||||
BinaryValue : byte array
|
||||
BooleanValue : Nullable<bool>
|
||||
DateTimeValue : Nullable<DateTime>
|
||||
DoubleValue : Nullable<Double>
|
||||
GuidValue : Nullable<Guid>
|
||||
Int32Value : Nullable<Int32>
|
||||
Int64Value : Nullable<Int64> }
|
||||
with
|
||||
static member Default =
|
||||
{
|
||||
EdmType = EdmType.String
|
||||
StringValue = null
|
||||
BinaryValue = null
|
||||
BooleanValue = Nullable ()
|
||||
DateTimeValue = Nullable ()
|
||||
DoubleValue = Nullable ()
|
||||
GuidValue = Nullable ()
|
||||
Int32Value = Nullable ()
|
||||
Int64Value = Nullable ()
|
||||
}
|
||||
static member AsEntityProperty (propertyValue:PropertyValue) : EntityProperty =
|
||||
match propertyValue.EdmType with
|
||||
| EdmType.String ->
|
||||
EntityProperty(propertyValue.StringValue)
|
||||
| EdmType.Binary ->
|
||||
EntityProperty(propertyValue.BinaryValue)
|
||||
| EdmType.Boolean ->
|
||||
EntityProperty(propertyValue.BooleanValue)
|
||||
| EdmType.DateTime ->
|
||||
EntityProperty(propertyValue.DateTimeValue)
|
||||
| EdmType.Double ->
|
||||
EntityProperty(propertyValue.DoubleValue)
|
||||
| EdmType.Guid ->
|
||||
EntityProperty(propertyValue.GuidValue)
|
||||
| EdmType.Int32 ->
|
||||
EntityProperty(propertyValue.Int32Value)
|
||||
| EdmType.Int64 ->
|
||||
EntityProperty(propertyValue.Int64Value)
|
||||
| _ -> null
|
||||
static member OfEntityProperty (entityProperty:EntityProperty) : PropertyValue =
|
||||
match entityProperty.PropertyType with
|
||||
| EdmType.String ->
|
||||
{ PropertyValue.Default with EdmType = entityProperty.PropertyType; StringValue = entityProperty.StringValue }
|
||||
| EdmType.Binary ->
|
||||
{ PropertyValue.Default with EdmType = entityProperty.PropertyType; BinaryValue = entityProperty.BinaryValue }
|
||||
| EdmType.Boolean ->
|
||||
{ PropertyValue.Default with EdmType = entityProperty.PropertyType; BooleanValue = entityProperty.BooleanValue }
|
||||
| EdmType.DateTime ->
|
||||
{ PropertyValue.Default with EdmType = entityProperty.PropertyType; DateTimeValue = entityProperty.DateTime }
|
||||
| EdmType.Double ->
|
||||
{ PropertyValue.Default with EdmType = entityProperty.PropertyType; DoubleValue = entityProperty.DoubleValue }
|
||||
| EdmType.Guid ->
|
||||
{ PropertyValue.Default with EdmType = entityProperty.PropertyType; GuidValue = entityProperty.GuidValue }
|
||||
| EdmType.Int32 ->
|
||||
{ PropertyValue.Default with EdmType = entityProperty.PropertyType; Int32Value = entityProperty.Int32Value }
|
||||
| EdmType.Int64 ->
|
||||
{ PropertyValue.Default with EdmType = entityProperty.PropertyType; Int64Value = entityProperty.Int64Value }
|
||||
| _ -> PropertyValue.Default
|
||||
|
||||
type DynamicTableEntity with
|
||||
member this.ToJson () =
|
||||
let props =
|
||||
this.Properties
|
||||
|> Seq.map (fun kvp -> kvp.Key, (PropertyValue.OfEntityProperty kvp.Value))
|
||||
|> dict
|
||||
System.Text.Json.JsonSerializer.Serialize (props, System.Text.Json.JsonSerializerOptions (IgnoreNullValues=true))
|
||||
member this.LoadJson (json:string) =
|
||||
let (dictionary:System.Collections.Generic.Dictionary<string, PropertyValue>) = System.Text.Json.JsonSerializer.Deserialize json
|
||||
for kvp in dictionary do
|
||||
this.Properties.[kvp.Key] <- kvp.Value |> PropertyValue.AsEntityProperty
|
||||
this
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
namespace AzureTableArchive
|
||||
|
||||
open Microsoft.Azure.Cosmos.Table
|
||||
open EntitySerialization
|
||||
|
||||
module Restore =
|
||||
|
||||
let entityFromJsonFile (partition:string) (rowKey:string) (filePath:string) : Async<DynamicTableEntity> =
|
||||
async {
|
||||
let! json = System.IO.File.ReadAllTextAsync filePath |> Async.AwaitTask
|
||||
return DynamicTableEntity(partition,rowKey).LoadJson json
|
||||
}
|
||||
|
||||
let restoreTables (cloudTableClient:CloudTableClient) : RestoreTables =
|
||||
fun (filePath:FilePath) ->
|
||||
async {
|
||||
let tableDirectories = filePath |> System.IO.Directory.GetDirectories
|
||||
do! tableDirectories |> Seq.map (fun tableDir ->
|
||||
async {
|
||||
let tableName = System.IO.DirectoryInfo(tableDir).Name
|
||||
let cloudTable = cloudTableClient.GetTableReference tableName
|
||||
do! cloudTable.CreateIfNotExistsAsync () |> Async.AwaitTask |> Async.Ignore
|
||||
let partitionDirectories = tableDir |> System.IO.Directory.GetDirectories
|
||||
for partitionDir in partitionDirectories do
|
||||
let partition = System.IO.DirectoryInfo(partitionDir).Name
|
||||
let rowFiles = partitionDir |> System.IO.Directory.GetFiles
|
||||
do!
|
||||
rowFiles |> Seq.map (fun rowFile ->
|
||||
async {
|
||||
let rowKey = System.IO.FileInfo(System.IO.Path.GetFileNameWithoutExtension rowFile).Name
|
||||
let! entity = entityFromJsonFile partition rowKey rowFile
|
||||
do! cloudTable.ExecuteAsync(TableOperation.InsertOrReplace entity) |> Async.AwaitTask |> Async.Ignore
|
||||
}
|
||||
) |> Async.Parallel |> Async.Ignore // rows in parallel
|
||||
}) |> Async.Parallel |> Async.Ignore // and tables in parallel
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net5.0</TargetFramework>
|
||||
<GenerateProgramFile>false</GenerateProgramFile>
|
||||
<RootNamespace>StorageTableBackupTests</RootNamespace>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Compile Include="BackupTests.fs" />
|
||||
<Compile Include="EntitySerializationTests.fs" />
|
||||
<Compile Include="Main.fs" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Expecto" Version="9.*" />
|
||||
<PackageReference Include="Moq" Version="4.16.0" />
|
||||
<PackageReference Include="YoloDev.Expecto.TestSdk" Version="0.*" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.*" />
|
||||
<PackageReference Update="FSharp.Core" Version="5.0.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\AzureTableArchive\AzureTableArchive.fsproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
|
@ -0,0 +1,161 @@
|
|||
module BackupTests
|
||||
|
||||
open System
|
||||
open System.Reflection
|
||||
open Expecto
|
||||
open Microsoft.Azure.Cosmos.Table
|
||||
open Microsoft.Azure.Storage.Blob
|
||||
open Microsoft.Azure.Storage.DataMovement
|
||||
open Moq
|
||||
open AzureTableArchive
|
||||
|
||||
// TableResultSegment is sealed with internal setters - unmockable - using reflection to make instances for mocks.
|
||||
let createTableResultSegment (cloudTables:seq<CloudTable>) =
|
||||
typeof<TableResultSegment>
|
||||
.GetConstructor(BindingFlags.NonPublic ||| BindingFlags.Instance, null, [| typeof<System.Collections.Generic.List<CloudTable>> |], null)
|
||||
.Invoke [| ResizeArray cloudTables |]
|
||||
:?> TableResultSegment
|
||||
|
||||
let setTableResultContinuation (continuationToken:TableContinuationToken) (tableResultSegment:TableResultSegment) =
|
||||
typeof<TableResultSegment>
|
||||
.GetProperty("ContinuationToken")
|
||||
.SetValue(tableResultSegment, continuationToken)
|
||||
tableResultSegment
|
||||
|
||||
// TableQuerySegment is sealed with internal setters, have to use reflection to make instances for mocks.
|
||||
let createTableQuerySegment<'TResult> (results:seq<'TResult>) =
|
||||
typeof<TableQuerySegment<'TResult>>
|
||||
.GetConstructor(BindingFlags.NonPublic ||| BindingFlags.Instance, null, [| typeof<System.Collections.Generic.List<'TResult>> |], null)
|
||||
.Invoke [| ResizeArray results |]
|
||||
:?> TableQuerySegment<'TResult>
|
||||
|
||||
let setTableQuerySegmentContinuation<'TResult> (continuationToken:TableContinuationToken) (tableQuerySegment:TableQuerySegment<'TResult>) =
|
||||
typeof<TableQuerySegment<'TResult>>
|
||||
.GetProperty("ContinuationToken")
|
||||
.SetValue(tableQuerySegment, continuationToken)
|
||||
tableQuerySegment
|
||||
|
||||
[<Tests>]
|
||||
let tests =
|
||||
testList "Backup" [
|
||||
testAsync "List tables with continuation" {
|
||||
let tableResultSegmentWithContinuation =
|
||||
[ "foo"; "bar"; "baz" ] |> List.map(fun tableName ->
|
||||
CloudTable(Uri $"https://whatever.com/{tableName}")
|
||||
)
|
||||
|> createTableResultSegment
|
||||
|> setTableResultContinuation (TableContinuationToken())
|
||||
let tableResultSegmentNoContinuation =
|
||||
[ "fizz"; "buzz" ] |> List.map(fun tableName ->
|
||||
CloudTable(Uri $"https://whatever.com/{tableName}")
|
||||
)
|
||||
|> createTableResultSegment
|
||||
let cloudTableClient = Mock<CloudTableClient>(StorageUri (Uri "https://whatever.com"), StorageCredentials())
|
||||
cloudTableClient.Setup(fun client -> client.ListTablesSegmentedAsync(It.IsNotNull<TableContinuationToken>()))
|
||||
.ReturnsAsync(tableResultSegmentNoContinuation) |> ignore
|
||||
cloudTableClient.Setup(fun client -> client.ListTablesSegmentedAsync(null))
|
||||
.ReturnsAsync(tableResultSegmentWithContinuation) |> ignore
|
||||
let! tables = Backup.listStorageTables (cloudTableClient.Object)
|
||||
Expect.sequenceEqual tables ["foo";"bar";"baz";"fizz";"buzz"] "Incorrect tables returned"
|
||||
}
|
||||
testAsync "Test all this reflection works since it's not mockable" {
|
||||
let cloudTableClient = Mock<CloudTableClient>(StorageUri (Uri "https://whatever.com"), StorageCredentials())
|
||||
cloudTableClient.Setup(fun client -> client.GetTableReference(It.IsAny()))
|
||||
.Returns<string>(fun name ->
|
||||
let cloudTable = Mock<CloudTable>(Uri $"https://wbatever.com/{name}", TableClientConfiguration())
|
||||
cloudTable.Setup(fun table -> table.ExecuteQuerySegmentedAsync(It.IsAny<TableQuery<DynamicTableEntity>>(), null))
|
||||
.ReturnsAsync(createTableQuerySegment [DynamicTableEntity("hello", "world")]) |> ignore
|
||||
cloudTable.Object) |> ignore
|
||||
let cloudTable = cloudTableClient.Object.GetTableReference "hi"
|
||||
Expect.equal cloudTable.Name "hi" "Returned wrong name for cloud table"
|
||||
let! result = cloudTable.ExecuteQuerySegmentedAsync(TableQuery<DynamicTableEntity>(), null) |> Async.AwaitTask
|
||||
Expect.isNotNull result "Result was null"
|
||||
Expect.hasLength result.Results 1 "Expected one result"
|
||||
Expect.isNotNull result.Results.[0] "First result is null"
|
||||
Expect.equal result.Results.[0].PartitionKey "hello" "Wrong entity partition key"
|
||||
Expect.equal result.Results.[0].RowKey "world" "Wrong entity row key"
|
||||
}
|
||||
testAsync "Dump table to files" {
|
||||
let mockTableResults =
|
||||
[ "table1" ] |> List.map(fun tableName ->
|
||||
CloudTable(Uri $"https://whatever.com/{tableName}")
|
||||
)
|
||||
|> createTableResultSegment
|
||||
let cloudTableClient = Mock<CloudTableClient>(StorageUri (Uri "https://whatever.com"), StorageCredentials())
|
||||
cloudTableClient.Setup(fun client -> client.GetTableReference("table1"))
|
||||
.Returns<string>(fun name ->
|
||||
let cloudTable = Mock<CloudTable>(Uri $"https://wbatever.com/{name}", TableClientConfiguration())
|
||||
cloudTable.Setup(fun table -> table.ExecuteQuerySegmentedAsync(It.IsAny<TableQuery<DynamicTableEntity>>(), null))
|
||||
.ReturnsAsync(createTableQuerySegment [
|
||||
DynamicTableEntity("one", "thing", "", ["foo", EntityProperty "bar"] |> dict)
|
||||
DynamicTableEntity("one", "more", "", ["foo", EntityProperty "baz"] |> dict)
|
||||
DynamicTableEntity("another", "item", "", ["foo", EntityProperty 13] |> dict)
|
||||
]) |> ignore
|
||||
cloudTable.Object) |> ignore
|
||||
let temporaryTestFiles = IO.Path.Combine(IO.Path.GetTempPath(), Guid.NewGuid().ToString())
|
||||
do! Backup.dumpTableToJsonFiles cloudTableClient.Object ("table1", temporaryTestFiles)
|
||||
try
|
||||
Expect.isTrue (System.IO.Directory.Exists(IO.Path.Combine(temporaryTestFiles, "table1"))) "table1 directory doesn't exist"
|
||||
Expect.isTrue (System.IO.Directory.Exists(IO.Path.Combine(temporaryTestFiles, "table1", "one"))) "table1/one directory doesn't exist"
|
||||
Expect.isTrue (System.IO.Directory.Exists(IO.Path.Combine(temporaryTestFiles, "table1", "another"))) "table1/another directory doesn't exist"
|
||||
Expect.isTrue (System.IO.File.Exists(IO.Path.Combine(temporaryTestFiles, "table1", "one", "thing.json"))) "table1/one/thing.json file doesn't exist"
|
||||
Expect.isTrue (System.IO.File.Exists(IO.Path.Combine(temporaryTestFiles, "table1", "one", "more.json"))) "table1/one/more.json file doesn't exist"
|
||||
Expect.isTrue (System.IO.File.Exists(IO.Path.Combine(temporaryTestFiles, "table1", "another", "item.json"))) "table1/another/item.json file doesn't exist"
|
||||
finally
|
||||
IO.Directory.Delete(temporaryTestFiles, true)
|
||||
}
|
||||
testAsync "Backup multiple tables to directories and files" {
|
||||
let mockTableResults =
|
||||
[ "table1"; "table2"; "table3" ] |> List.map(fun tableName ->
|
||||
CloudTable(Uri $"https://whatever.com/{tableName}")
|
||||
)
|
||||
|> createTableResultSegment
|
||||
let cloudTableClient = Mock<CloudTableClient>(StorageUri (Uri "https://whatever.com"), StorageCredentials())
|
||||
cloudTableClient.Setup(fun client -> client.ListTablesSegmentedAsync(null)) // not testing continuation here.
|
||||
.ReturnsAsync(mockTableResults) |> ignore
|
||||
cloudTableClient.Setup(fun client -> client.GetTableReference("table1"))
|
||||
.Returns<string>(fun name ->
|
||||
let cloudTable = Mock<CloudTable>(Uri $"https://wbatever.com/{name}", TableClientConfiguration())
|
||||
cloudTable.Setup(fun table -> table.ExecuteQuerySegmentedAsync(It.IsAny<TableQuery<DynamicTableEntity>>(), null))
|
||||
.ReturnsAsync(createTableQuerySegment [
|
||||
DynamicTableEntity("one", "thing", "", ["foo", EntityProperty "bar"] |> dict)
|
||||
DynamicTableEntity("one", "more", "", ["foo", EntityProperty "baz"] |> dict)
|
||||
DynamicTableEntity("another", "item", "", ["foo", EntityProperty 13] |> dict)
|
||||
]) |> ignore
|
||||
cloudTable.Object) |> ignore
|
||||
cloudTableClient.Setup(fun client -> client.GetTableReference("table2"))
|
||||
.Returns<string>(fun name ->
|
||||
let cloudTable = Mock<CloudTable>(Uri $"https://wbatever.com/{name}", TableClientConfiguration())
|
||||
cloudTable.Setup(fun table -> table.ExecuteQuerySegmentedAsync(It.IsAny<TableQuery<DynamicTableEntity>>(), null))
|
||||
.ReturnsAsync(createTableQuerySegment [
|
||||
DynamicTableEntity("two", "number7", "", ["seven", EntityProperty 7] |> dict)
|
||||
DynamicTableEntity("two", "number8", "", ["eight", EntityProperty 8.0] |> dict)
|
||||
DynamicTableEntity("another", "number", "", ["nine", EntityProperty "9"] |> dict)
|
||||
]) |> ignore
|
||||
cloudTable.Object) |> ignore
|
||||
cloudTableClient.Setup(fun client -> client.GetTableReference("table3"))
|
||||
.Returns<string>(fun name ->
|
||||
let cloudTable = Mock<CloudTable>(Uri $"https://wbatever.com/{name}", TableClientConfiguration())
|
||||
cloudTable.Setup(fun table -> table.ExecuteQuerySegmentedAsync(It.IsAny<TableQuery<DynamicTableEntity>>(), null))
|
||||
.ReturnsAsync(createTableQuerySegment [
|
||||
DynamicTableEntity("three", "things", "", ["foo", EntityProperty "bar"] |> dict)
|
||||
DynamicTableEntity("three", "items", "", ["foo", EntityProperty "baz"] |> dict)
|
||||
]) |> ignore
|
||||
cloudTable.Object) |> ignore
|
||||
let! temporaryTestFiles = Backup.backupTables (Backup.listStorageTables cloudTableClient.Object) (Backup.dumpTableToJsonFiles cloudTableClient.Object) Backup.GeneratedBackupPath
|
||||
try
|
||||
Expect.isTrue (System.IO.Directory.Exists(IO.Path.Combine(temporaryTestFiles, "table1"))) "table1 directory doesn't exist"
|
||||
Expect.isTrue (System.IO.Directory.Exists(IO.Path.Combine(temporaryTestFiles, "table1", "one"))) "table1/one directory doesn't exist"
|
||||
Expect.isTrue (System.IO.Directory.Exists(IO.Path.Combine(temporaryTestFiles, "table1", "another"))) "table1/another directory doesn't exist"
|
||||
Expect.isTrue (System.IO.File.Exists(IO.Path.Combine(temporaryTestFiles, "table1", "one", "thing.json"))) "table1/one/thing.json file doesn't exist"
|
||||
Expect.isTrue (System.IO.File.Exists(IO.Path.Combine(temporaryTestFiles, "table1", "one", "more.json"))) "table1/one/more.json file doesn't exist"
|
||||
Expect.isTrue (System.IO.File.Exists(IO.Path.Combine(temporaryTestFiles, "table1", "another", "item.json"))) "table1/another/item.json file doesn't exist"
|
||||
Expect.isTrue (System.IO.File.Exists(IO.Path.Combine(temporaryTestFiles, "table2", "two", "number7.json"))) "table2/two/number7.json file doesn't exist"
|
||||
Expect.isTrue (System.IO.File.Exists(IO.Path.Combine(temporaryTestFiles, "table2", "two", "number8.json"))) "table2/two/number8.json file doesn't exist"
|
||||
Expect.isTrue (System.IO.File.Exists(IO.Path.Combine(temporaryTestFiles, "table2", "another", "number.json"))) "table2/another/number.json file doesn't exist"
|
||||
Expect.isTrue (System.IO.File.Exists(IO.Path.Combine(temporaryTestFiles, "table3", "three", "things.json"))) "table3/three/things.json file doesn't exist"
|
||||
Expect.isTrue (System.IO.File.Exists(IO.Path.Combine(temporaryTestFiles, "table3", "three", "items.json"))) "table3/three/items.json file doesn't exist"
|
||||
finally
|
||||
IO.Directory.Delete(temporaryTestFiles, true)
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1,160 @@
|
|||
module EntitySerializationTests
|
||||
|
||||
open System
|
||||
open Expecto
|
||||
open Microsoft.Azure.Cosmos.Table
|
||||
open AzureTableArchive
|
||||
open EntitySerialization
|
||||
|
||||
[<Tests>]
|
||||
let entityPropertyTests =
|
||||
testList "Building Entity Property" [
|
||||
test "Check Defaults" {
|
||||
let defaultPropVal = PropertyValue.Default
|
||||
Expect.isNull defaultPropVal.BinaryValue "BinaryValue wasn't null"
|
||||
Expect.isFalse defaultPropVal.BooleanValue.HasValue "BooleanValue had a value"
|
||||
Expect.isFalse defaultPropVal.DoubleValue.HasValue "DoubleValue had a value"
|
||||
Expect.isFalse defaultPropVal.GuidValue.HasValue "GuidValue had a value"
|
||||
Expect.isFalse defaultPropVal.Int32Value.HasValue "Int32Value had a value"
|
||||
Expect.isFalse defaultPropVal.Int64Value.HasValue "Int64Value had a value"
|
||||
Expect.isNull defaultPropVal.StringValue "StringValue wasn't null"
|
||||
Expect.isFalse defaultPropVal.DateTimeValue.HasValue "DateTimeValue had a value"
|
||||
Expect.equal defaultPropVal.EdmType EdmType.String "Default EdmType should be a string"
|
||||
}
|
||||
test "Creates Binary EntityProperty" {
|
||||
let prop =
|
||||
{ PropertyValue.Default with BinaryValue = [| 0uy; 1uy; 2uy; 3uy |]; EdmType = EdmType.Binary }
|
||||
|> PropertyValue.AsEntityProperty
|
||||
Expect.sequenceEqual prop.BinaryValue [| 0uy; 1uy; 2uy; 3uy |] "Expecting a byte array value on the EntityProperty"
|
||||
Expect.equal prop.PropertyType EdmType.Binary "Incorrect PropertyType"
|
||||
}
|
||||
test "Creates Boolean EntityProperty" {
|
||||
let prop =
|
||||
{ PropertyValue.Default with BooleanValue = Nullable(true); EdmType = EdmType.Boolean }
|
||||
|> PropertyValue.AsEntityProperty
|
||||
Expect.isTrue prop.BooleanValue.HasValue "BooleanValue should have a value"
|
||||
Expect.isTrue prop.BooleanValue.Value "Expecting a true boolean value the EntityProperty"
|
||||
Expect.equal prop.PropertyType EdmType.Boolean "Incorrect PropertyType"
|
||||
}
|
||||
test "Creates Double EntityProperty" {
|
||||
let prop =
|
||||
{ PropertyValue.Default with DoubleValue = Nullable(0.3); EdmType = EdmType.Double }
|
||||
|> PropertyValue.AsEntityProperty
|
||||
Expect.isTrue prop.DoubleValue.HasValue "DoubleValue should have a value"
|
||||
Expect.equal prop.DoubleValue.Value 0.3 "Expecting a double value value on the EntityProperty"
|
||||
Expect.equal prop.PropertyType EdmType.Double "Incorrect PropertyType"
|
||||
}
|
||||
test "Creates Guid EntityProperty" {
|
||||
let guid = Guid("5e020cd0-cfd7-4416-b8cf-d8f57e15a9b3")
|
||||
let prop =
|
||||
{ PropertyValue.Default with GuidValue = Nullable(guid); EdmType = EdmType.Guid }
|
||||
|> PropertyValue.AsEntityProperty
|
||||
Expect.isTrue prop.GuidValue.HasValue "GuidValue should have a value"
|
||||
Expect.equal prop.GuidValue.Value guid "Expecting a guid value on the EntityProperty"
|
||||
Expect.equal prop.PropertyType EdmType.Guid "Incorrect PropertyType"
|
||||
}
|
||||
test "Creates Int32 EntityProperty" {
|
||||
let prop =
|
||||
{ PropertyValue.Default with Int32Value = Nullable(1234); EdmType = EdmType.Int32 }
|
||||
|> PropertyValue.AsEntityProperty
|
||||
Expect.isTrue prop.Int32Value.HasValue "Int32Value should have a value"
|
||||
Expect.equal prop.Int32Value.Value 1234 "Expecting an int32 value on the EntityProperty"
|
||||
Expect.equal prop.PropertyType EdmType.Int32 "Incorrect PropertyType"
|
||||
}
|
||||
test "Creates Int64 EntityProperty" {
|
||||
let prop =
|
||||
{ PropertyValue.Default with Int64Value = Nullable(1234L); EdmType = EdmType.Int64 }
|
||||
|> PropertyValue.AsEntityProperty
|
||||
Expect.isTrue prop.Int64Value.HasValue "Int64Value should have a value"
|
||||
Expect.equal prop.Int64Value.Value 1234L "Expecting an int64 value on the EntityProperty"
|
||||
Expect.equal prop.PropertyType EdmType.Int64 "Incorrect PropertyType"
|
||||
}
|
||||
test "Creates String EntityProperty" {
|
||||
let prop =
|
||||
{ PropertyValue.Default with StringValue = "abcdefg"; EdmType = EdmType.String }
|
||||
|> PropertyValue.AsEntityProperty
|
||||
Expect.equal prop.StringValue "abcdefg" "Expecting a string value on the EntityProperty"
|
||||
Expect.equal prop.PropertyType EdmType.String "Incorrect PropertyType"
|
||||
}
|
||||
test "Creates DateTime EntityProperty" {
|
||||
let now = DateTime.Now
|
||||
let prop =
|
||||
{ PropertyValue.Default with DateTimeValue = Nullable(now); EdmType = EdmType.DateTime }
|
||||
|> PropertyValue.AsEntityProperty
|
||||
Expect.isTrue prop.DateTime.HasValue "DateTime should have a value"
|
||||
Expect.equal prop.DateTime.Value now "Expecting a datetime value on the EntityProperty"
|
||||
Expect.equal prop.PropertyType EdmType.DateTime "Incorrect PropertyType"
|
||||
}
|
||||
]
|
||||
|
||||
[<Tests>]
|
||||
let propertyValueTests =
|
||||
testList "Building PropertyValue" [
|
||||
test "Creates Binary PropertyValue" {
|
||||
let prop = EntityProperty([|0uy; 1uy; 2uy; 3uy|]) |> PropertyValue.OfEntityProperty
|
||||
Expect.sequenceEqual [| 0uy; 1uy; 2uy; 3uy |] prop.BinaryValue "Expecting a byte array value on the PropertyValue"
|
||||
Expect.equal prop.EdmType EdmType.Binary "Incorrect EdmType"
|
||||
}
|
||||
test "Creates Boolean PropertyValue" {
|
||||
let prop = EntityProperty(true) |> PropertyValue.OfEntityProperty
|
||||
Expect.equal true prop.BooleanValue.Value "Expecting a boolean true on the PropertyValue"
|
||||
Expect.equal prop.EdmType EdmType.Boolean "Incorrect EdmType"
|
||||
}
|
||||
test "Creates Double PropertyValue" {
|
||||
let prop = EntityProperty(1.23) |> PropertyValue.OfEntityProperty
|
||||
Expect.equal 1.23 prop.DoubleValue.Value "Expecting a double value on the PropertyValue"
|
||||
Expect.equal prop.EdmType EdmType.Double "Incorrect EdmType"
|
||||
}
|
||||
test "Creates Guid PropertyValue" {
|
||||
let guid = Guid("93a7a7e7-9fef-4ad0-a4e4-b77889738d29")
|
||||
let prop = EntityProperty guid |> PropertyValue.OfEntityProperty
|
||||
Expect.equal guid prop.GuidValue.Value "Expecting a GUID value on the PropertyValue"
|
||||
Expect.equal prop.EdmType EdmType.Guid "Incorrect EdmType"
|
||||
}
|
||||
test "Creates Int32 PropertyValue" {
|
||||
let prop = EntityProperty(123) |> PropertyValue.OfEntityProperty
|
||||
Expect.equal 123 prop.Int32Value.Value "Expecting an int32 value on the PropertyValue"
|
||||
Expect.equal prop.EdmType EdmType.Int32 "Incorrect EdmType"
|
||||
}
|
||||
test "Creates Int64 PropertyValue" {
|
||||
let prop = EntityProperty(1234L) |> PropertyValue.OfEntityProperty
|
||||
Expect.equal 1234L prop.Int64Value.Value "Expecting an int64 value on the PropertyValue"
|
||||
Expect.equal prop.EdmType EdmType.Int64 "Incorrect EdmType"
|
||||
}
|
||||
test "Creates String PropertyValue" {
|
||||
let prop = EntityProperty("Hello world") |> PropertyValue.OfEntityProperty
|
||||
Expect.equal "Hello world" prop.StringValue "Expecting a string value on the PropertyValue"
|
||||
Expect.equal prop.EdmType EdmType.String "Incorrect EdmType"
|
||||
}
|
||||
test "Creates DateTime PropertyValue" {
|
||||
let now = DateTime.Now
|
||||
let prop = EntityProperty(now) |> PropertyValue.OfEntityProperty
|
||||
Expect.equal now prop.DateTimeValue.Value "Expecting a datetime value on the PropertyValue"
|
||||
Expect.equal prop.EdmType EdmType.DateTime "Incorrect EdmType"
|
||||
}
|
||||
]
|
||||
|
||||
[<Tests>]
|
||||
let dynamicEntityJson =
|
||||
testList "DynamicEntity to/from JSON" [
|
||||
test "DynamicEntity serializes to JSON without nulls PropertyValue fields" {
|
||||
let entity = DynamicTableEntity("somepartition","somerow")
|
||||
entity.Properties <-
|
||||
[ "Foo", EntityProperty "bar"
|
||||
"Count", EntityProperty 123 ] |> dict
|
||||
let json = entity.ToJson ()
|
||||
Expect.isFalse (json.Contains "BooleanValue") "null field from PropertyValue was included in JSON payload"
|
||||
Expect.equal
|
||||
json
|
||||
"""{"Foo":{"EdmType":0,"StringValue":"bar"},"Count":{"EdmType":6,"Int32Value":123}}"""
|
||||
"JSON serialization has incorrect structure."
|
||||
}
|
||||
test "DynamicEntity deserializes from JSON with correct EntityProperty types" {
|
||||
let entity =
|
||||
DynamicTableEntity("somepartition","somerow")
|
||||
.LoadJson """{"Foo":{"EdmType":0,"StringValue":"bar"},"Count":{"EdmType":6,"Int32Value":123}}"""
|
||||
Expect.hasLength entity.Properties 2 "Incorrect number of properties after deserializing."
|
||||
Expect.equal entity.["Foo"].StringValue "bar" "Incorrect value for 'Foo' after deserialization"
|
||||
Expect.equal entity.["Count"].Int32Value.Value 123 "Incorrect value for 'Count' after deserialization"
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
module AzureTableArchiveTests
|
||||
open Expecto
|
||||
|
||||
[<EntryPoint>]
|
||||
let main argv =
|
||||
Tests.runTestsInAssembly defaultConfig argv
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
test:
|
||||
dotnet test -v n
|
||||
check:
|
||||
dotnet test --no-build -v n
|
||||
build:
|
||||
dotnet build
|
||||
clean:
|
||||
rm -rf **/bin **/obj
|
||||
|
||||
all: build check
|
||||
|
|
@ -1,33 +1,22 @@
|
|||
# Project
|
||||
AzureTableArchive
|
||||
=================
|
||||
|
||||
> This repo has been populated by an initial template to help get you started. Please
|
||||
> make sure to update the content to build a great experience for community-building.
|
||||
The AzureTableArchive library is intended for creating archives of Azure Storage Tables from Storage Accounts or Cosmos DB and storing them in Azure Storage Blob containers. Because multiple archives may be retained, previous copies of table records are available to restore from the various points of time when archives are created.
|
||||
|
||||
As the maintainer of this project, please make a few updates:
|
||||
Archives are not pruned by this library - it's recommended to enable blob expiration to remove old archives automatically.
|
||||
|
||||
- Improving this README.MD file to provide a great experience
|
||||
- Updating SUPPORT.MD with content about this project's support experience
|
||||
- Understanding the security reporting process in SECURITY.MD
|
||||
- Remove this section from the README
|
||||
Existing records in tables are not removed prior to restore, so records that were added to the tables since the backup was taken will be left in the table.
|
||||
|
||||
## Contributing
|
||||
|
||||
This project welcomes contributions and suggestions. Most contributions require you to agree to a
|
||||
Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
|
||||
the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com.
|
||||
This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com.
|
||||
|
||||
When you submit a pull request, a CLA bot will automatically determine whether you need to provide
|
||||
a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions
|
||||
provided by the bot. You will only need to do this once across all repos using our CLA.
|
||||
When you submit a pull request, a CLA bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA.
|
||||
|
||||
This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
|
||||
For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
|
||||
contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
|
||||
This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
|
||||
|
||||
## Trademarks
|
||||
|
||||
This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft
|
||||
trademarks or logos is subject to and must follow
|
||||
[Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general).
|
||||
Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship.
|
||||
Any use of third-party trademarks or logos are subject to those third-party's policies.
|
||||
This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft trademarks or logos is subject to and must follow [Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general).
|
||||
|
||||
Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. Any use of third-party trademarks or logos are subject to those third-party's policies.
|
||||
|
|
|
|||
14
SUPPORT.md
14
SUPPORT.md
|
|
@ -1,13 +1,3 @@
|
|||
# TODO: The maintainer of this repo has not yet edited this file
|
||||
|
||||
**REPO OWNER**: Do you want Customer Service & Support (CSS) support for this product/project?
|
||||
|
||||
- **No CSS support:** Fill out this template with information about how to file issues and get help.
|
||||
- **Yes CSS support:** Fill out an intake form at [aka.ms/spot](https://aka.ms/spot). CSS will work with/help you to determine next steps. More details also available at [aka.ms/onboardsupport](https://aka.ms/onboardsupport).
|
||||
- **Not sure?** Fill out a SPOT intake as though the answer were "Yes". CSS will help you decide.
|
||||
|
||||
*Then remove this first heading from this SUPPORT.MD file before publishing your repo.*
|
||||
|
||||
# Support
|
||||
|
||||
## How to file issues and get help
|
||||
|
|
@ -16,9 +6,7 @@ This project uses GitHub Issues to track bugs and feature requests. Please searc
|
|||
issues before filing new issues to avoid duplicates. For new issues, file your bug or
|
||||
feature request as a new Issue.
|
||||
|
||||
For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE
|
||||
FOR HOW TO ENGAGE REPO OWNERS OR COMMUNITY FOR HELP. COULD BE A STACK OVERFLOW TAG OR OTHER
|
||||
CHANNEL. WHERE WILL YOU HELP PEOPLE?**.
|
||||
For help and questions about using this project, please see [usage.md](docs/usage.md) under docs. If you are contributing to this project, [development.md](docs/development.md) offers some guidance about the various modules and how to build and test locally.
|
||||
|
||||
## Microsoft Support Policy
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,55 @@
|
|||
AzureTableArchive Development Guide
|
||||
========
|
||||
|
||||
The AzureTableArchive library is broken into several modules.
|
||||
|
||||
### Domain.fs
|
||||
The Domain module contains general type definitions for both structure and behavior.
|
||||
These provide a general API surface for testing and help define the purpose of the
|
||||
implementations.
|
||||
|
||||
### EntitySerialization.fs
|
||||
THe EntitySerialization module handles the logic of processing table records generically
|
||||
as DynamicTableEntity objects. These are converted to and from JSON for persisting to
|
||||
the local file system and blob storage.
|
||||
|
||||
_Currently, etags are not stored. This may be a useful enhancement to prevent restoring
|
||||
over newer data._
|
||||
|
||||
### Backup.fs
|
||||
The Backup module queries table records, converts results to JSON, and stores them in
|
||||
the local file system.
|
||||
|
||||
### ContainerSync.fs
|
||||
The ContainerSync module synchronizes the local directory to and from blob storage. It
|
||||
relies on the Data Movement library for efficient and reliable transfer.
|
||||
|
||||
### Restore.fs
|
||||
The Restore module deserializes JSON records from the file system and uploads them to
|
||||
tables in the target storage account. It uses InsertOrReplace to create or replace
|
||||
records in the table.
|
||||
|
||||
Building
|
||||
--------
|
||||
|
||||
Prerequisites - dotnet 5.0.
|
||||
|
||||
If you have `make` installed, common build and test scenarios are covered in the file.
|
||||
|
||||
* `make build` - restores dependencies and builds
|
||||
* `make test` - restore dependencies, builds, and runs tests
|
||||
* `make check` - typically run after `make build`, this will run tests without building again
|
||||
* `make clean` - removes any existing restore or build artifacts
|
||||
* `make all` - runs `build` and then `check`
|
||||
|
||||
Alternatively, use the standard `dotnet build` and `dotnet test` commands at the
|
||||
solution level.
|
||||
|
||||
Testing
|
||||
-------
|
||||
|
||||
Testing uses `Expecto` for F# friendly assertions, parallel execution, and native
|
||||
`async` support. The Azure client library calls are mocked using `Moq` so application
|
||||
logic can be tested without requiring interaction with a storage account. This is not
|
||||
always easy, as some Azure SDK's have sealed classes with internal constructors and
|
||||
require reflection to fully mock.
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
Usage
|
||||
=========
|
||||
|
||||
Backup and restore of tables and interaction with blob containers leverages the
|
||||
`CloudTableClient` and `CloudBlobClient` instances created by the application
|
||||
hosting the library. The responsibility of authentication with the table and blob
|
||||
services is handled by the hosting application.
|
||||
|
||||
Once the application creates clients for the table and blob accounts, a backup of
|
||||
the tables can be created on the local file system where the backup to a directory
|
||||
which can then by transferred to the blob storage for archival.
|
||||
|
||||
### Backup and Archive Tables
|
||||
This snippet creates a backup of tables to a local directory and then copies that
|
||||
directory to a blob container for archival.
|
||||
|
||||
```fsharp
|
||||
let tableClient = CloudStorageAccount.Parse(sourceConnString).CreateCloudTableClient()
|
||||
let archiveBlobClient = Microsoft.Azure.Storage.CloudStorageAccount.Parse(archiveBlobsConnString).CreateCloudBlobClient()
|
||||
|
||||
// Createa a backup to a local temporary directory
|
||||
let! backupPath = Backup.BackupTables tableClient
|
||||
|
||||
// Archive the backup to a blob storage container
|
||||
|
||||
// First give it a name based on your own conventions. A timestamp is a good choice.
|
||||
let archiveName = DateTimeOffset.UtcNow.ToString "O"
|
||||
// Specify the name of the container for storing the backups, the local path to the backup,
|
||||
// and the name of the archive, which becomes the directory in the container.
|
||||
let backupLocation = {
|
||||
BackupContainer="tablebackups"
|
||||
BackupPath=backupPath
|
||||
ArchiveName=archiveName }
|
||||
|
||||
// You may also want to report progress of the transfer.
|
||||
let reportProgress = Some (Progress<TransferStatus>(fun progress -> Console.WriteLine ($"Transferred: {progress.BytesTransferred}")))
|
||||
|
||||
// Finally begin the sync and transfer the table backup to the blob for archival.
|
||||
match! ContainerSync.syncToContainer archiveBlobClient reportProgress backupLocation with
|
||||
| Error err ->
|
||||
Console.Error.WriteLine $"Some files failed to transfer {err}"
|
||||
| Ok backupName ->
|
||||
Console.WriteLine $"Syncronized {backupName} to archive {archiveName}"
|
||||
System.IO.Directory.Delete(backupPath, true)
|
||||
```
|
||||
|
||||
### Transfer Table Data to Another Account
|
||||
This snippet downloads table data from one account and uploads it to another. This can
|
||||
be used to move a set of tables between two accounts or from a Storage Account to
|
||||
CosmosDB.
|
||||
|
||||
```fsharp
|
||||
let sourceTableClient = CloudStorageAccount.Parse(sourceConnString).CreateCloudTableClient()
|
||||
let targetTableClient = CloudStorageAccount.Parse(targetConnString).CreateCloudTableClient()
|
||||
|
||||
// Backup to a local temporary directory,
|
||||
let! backupPath = Backup.BackupTables sourceTableClient
|
||||
|
||||
// Restore to the target tables account.
|
||||
do! Restore.restoreTables targetTableClient backupPath
|
||||
Console.WriteLine "Tables restored."
|
||||
```
|
||||
Загрузка…
Ссылка в новой задаче