Azure
/
azure-sdk-for-cpp
зеркало из https://github.com/Azure/azure-sdk-for-cpp.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

[Storage Blob Service] Concurrent upload for Page Blob (#486)

This commit is contained in:
JinmingHu 2020-08-20 10:54:28 +08:00 коммит произвёл GitHub
Родитель 070772e7c5
Коммит b9299478ee
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
8 изменённых файлов: 331 добавлений и 73 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

4
sdk/storage/inc/blobs/append_blob_client.hpp
Просмотреть файл

@ -115,7 +115,7 @@ namespace Azure { namespace Storage { namespace Blobs {
* @return A CreateAppendBlobResult describing the newly created append blob.
*/
Azure::Core::Response<CreateAppendBlobResult> Create(
const CreateAppendBlobOptions& options = CreateAppendBlobOptions());
const CreateAppendBlobOptions& options = CreateAppendBlobOptions()) const;
/**
* @brief Commits a new block of data, represented by the content BodyStream to the end
@ -129,7 +129,7 @@ namespace Azure { namespace Storage { namespace Blobs {
*/
Azure::Core::Response<AppendBlockResult> AppendBlock(
Azure::Core::Http::BodyStream* content,
const AppendBlockOptions& options = AppendBlockOptions());
const AppendBlockOptions& options = AppendBlockOptions()) const;
/**
* @brief Commits a new block of data, represented by the content BodyStream to the end

2
sdk/storage/inc/blobs/blob_options.hpp
Просмотреть файл

@ -920,6 +920,8 @@ namespace Azure { namespace Storage { namespace Blobs {
int Concurrency = 1;
};
using UploadPageBlobFromOptions = UploadBlockBlobFromOptions;
/**
* @brief Optional parameters for BlockBlobClient::StageBlock.
*/

7
sdk/storage/inc/blobs/blob_responses.hpp
Просмотреть файл

@ -27,6 +27,13 @@ namespace Azure { namespace Storage { namespace Blobs {
using UploadBlockBlobFromResult = UploadBlockBlobResult;
struct UploadPageBlobFromResult
{
Azure::Core::Nullable<bool> ServerEncrypted;
Azure::Core::Nullable<std::string> EncryptionKeySha256;
Azure::Core::Nullable<std::string> EncryptionScope;
};
struct PageRange
{
int64_t Offset;

42
sdk/storage/inc/blobs/page_blob_client.hpp
Просмотреть файл

@ -120,7 +120,7 @@ namespace Azure { namespace Storage { namespace Blobs {
*/
Azure::Core::Response<CreatePageBlobResult> Create(
int64_t blobContentLength,
const CreatePageBlobOptions& options = CreatePageBlobOptions());
const CreatePageBlobOptions& options = CreatePageBlobOptions()) const;
/**
* @brief Writes content to a range of pages in a page blob, starting at offset.
@ -135,7 +135,7 @@ namespace Azure { namespace Storage { namespace Blobs {
Azure::Core::Response<UploadPageBlobPagesResult> UploadPages(
Azure::Core::Http::BodyStream* content,
int64_t offset,
const UploadPageBlobPagesOptions& options = UploadPageBlobPagesOptions());
const UploadPageBlobPagesOptions& options = UploadPageBlobPagesOptions()) const;
/**
* @brief Writes a range of pages to a page blob where the contents are read from a
@ -160,7 +160,34 @@ namespace Azure { namespace Storage { namespace Blobs {
int64_t sourceOffset,
int64_t sourceLength,
int64_t destinationoffset,
const UploadPageBlobPagesFromUriOptions& options = UploadPageBlobPagesFromUriOptions());
const UploadPageBlobPagesFromUriOptions& options
= UploadPageBlobPagesFromUriOptions()) const;
/**
* @brief Creates a new page blob, or updates the content of an existing page blob. Updating
* an existing page blob overwrites any existing metadata on the blob.
*
* @param buffer A memory buffer containing the content to upload.
* @param bufferSize Size of the memory buffer.
* @param options Optional parameters to execute this function.
* @return A UploadPageBlobPagesResult describing the state of the updated page blob.
*/
Azure::Core::Response<UploadPageBlobFromResult> UploadFrom(
const uint8_t* buffer,
std::size_t bufferSize,
const UploadPageBlobFromOptions& options = UploadPageBlobFromOptions()) const;
/**
* @brief Creates a new page blob, or updates the content of an existing page blob. Updating
* an existing page blob overwrites any existing metadata on the blob.
*
* @param file A file containing the content to upload.
* @param options Optional parameters to execute this function.
* @return A UploadPageBlobFromResult describing the state of the updated page blob.
*/
Azure::Core::Response<UploadPageBlobFromResult> UploadFrom(
const std::string& file,
const UploadPageBlobFromOptions& options = UploadPageBlobFromOptions()) const;
/**
* @brief Clears one or more pages from the page blob, as specificed by offset and length.
@ -175,7 +202,7 @@ namespace Azure { namespace Storage { namespace Blobs {
Azure::Core::Response<ClearPageBlobPagesResult> ClearPages(
int64_t offset,
int64_t length,
const ClearPageBlobPagesOptions& options = ClearPageBlobPagesOptions());
const ClearPageBlobPagesOptions& options = ClearPageBlobPagesOptions()) const;
/**
* @brief Resizes the page blob to the specified size (which must be a multiple of 512). If the
@ -189,7 +216,7 @@ namespace Azure { namespace Storage { namespace Blobs {
*/
Azure::Core::Response<ResizePageBlobResult> Resize(
int64_t blobContentLength,
const ResizePageBlobOptions& options = ResizePageBlobOptions());
const ResizePageBlobOptions& options = ResizePageBlobOptions()) const;
/**
* @brief Returns the list of valid page ranges for a page blob or snapshot of a page blob.
@ -198,7 +225,7 @@ namespace Azure { namespace Storage { namespace Blobs {
* @return A GetPageBlobPageRangesResult describing the valid page ranges for this blob.
*/
Azure::Core::Response<GetPageBlobPageRangesResult> GetPageRanges(
const GetPageBlobPageRangesOptions& options = GetPageBlobPageRangesOptions());
const GetPageBlobPageRangesOptions& options = GetPageBlobPageRangesOptions()) const;
/**
* @brief Starts copying a snapshot of the sourceUri page blob to this page blob. The snapshot
@ -213,7 +240,8 @@ namespace Azure { namespace Storage { namespace Blobs {
*/
Azure::Core::Response<StartCopyPageBlobIncrementalResult> StartCopyIncremental(
const std::string& sourceUri,
const StartCopyPageBlobIncrementalOptions& options = StartCopyPageBlobIncrementalOptions());
const StartCopyPageBlobIncrementalOptions& options
= StartCopyPageBlobIncrementalOptions()) const;
private:
explicit PageBlobClient(BlobClient blobClient);

4
sdk/storage/src/blobs/append_blob_client.cpp
Просмотреть файл

@ -73,7 +73,7 @@ namespace Azure { namespace Storage { namespace Blobs {
}
Azure::Core::Response<CreateAppendBlobResult> AppendBlobClient::Create(
const CreateAppendBlobOptions& options)
const CreateAppendBlobOptions& options) const
{
BlobRestClient::AppendBlob::CreateAppendBlobOptions protocolLayerOptions;
protocolLayerOptions.HttpHeaders = options.HttpHeaders;
@ -96,7 +96,7 @@ namespace Azure { namespace Storage { namespace Blobs {
Azure::Core::Response<AppendBlockResult> AppendBlobClient::AppendBlock(
Azure::Core::Http::BodyStream* content,
const AppendBlockOptions& options)
const AppendBlockOptions& options) const
{
BlobRestClient::AppendBlob::AppendBlockOptions protocolLayerOptions;
protocolLayerOptions.ContentMd5 = options.ContentMd5;

104
sdk/storage/src/blobs/page_blob_client.cpp
Просмотреть файл

@ -3,7 +3,9 @@
#include "blobs/page_blob_client.hpp"
#include "common/concurrent_transfer.hpp"
#include "common/constants.hpp"
#include "common/file_io.hpp"
#include "common/storage_common.hpp"
namespace Azure { namespace Storage { namespace Blobs {
@ -72,7 +74,7 @@ namespace Azure { namespace Storage { namespace Blobs {
Azure::Core::Response<CreatePageBlobResult> PageBlobClient::Create(
int64_t blobContentLength,
const CreatePageBlobOptions& options)
const CreatePageBlobOptions& options) const
{
BlobRestClient::PageBlob::CreatePageBlobOptions protocolLayerOptions;
protocolLayerOptions.BlobContentLength = blobContentLength;
@ -99,7 +101,7 @@ namespace Azure { namespace Storage { namespace Blobs {
Azure::Core::Response<UploadPageBlobPagesResult> PageBlobClient::UploadPages(
Azure::Core::Http::BodyStream* content,
int64_t offset,
const UploadPageBlobPagesOptions& options)
const UploadPageBlobPagesOptions& options) const
{
BlobRestClient::PageBlob::UploadPageBlobPagesOptions protocolLayerOptions;
protocolLayerOptions.Range = std::make_pair(offset, offset + content->Length() - 1);
@ -126,7 +128,7 @@ namespace Azure { namespace Storage { namespace Blobs {
int64_t sourceOffset,
int64_t sourceLength,
int64_t destinationoffset,
const UploadPageBlobPagesFromUriOptions& options)
const UploadPageBlobPagesFromUriOptions& options) const
{
BlobRestClient::PageBlob::UploadPageBlobPagesFromUriOptions protocolLayerOptions;
protocolLayerOptions.SourceUri = sourceUri;
@ -155,7 +157,7 @@ namespace Azure { namespace Storage { namespace Blobs {
Azure::Core::Response<ClearPageBlobPagesResult> PageBlobClient::ClearPages(
int64_t offset,
int64_t length,
const ClearPageBlobPagesOptions& options)
const ClearPageBlobPagesOptions& options) const
{
BlobRestClient::PageBlob::ClearPageBlobPagesOptions protocolLayerOptions;
protocolLayerOptions.Range = std::make_pair(offset, offset + length - 1);
@ -175,9 +177,97 @@ namespace Azure { namespace Storage { namespace Blobs {
options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions);
}
Azure::Core::Response<UploadPageBlobFromResult> PageBlobClient::UploadFrom(
const uint8_t* buffer,
std::size_t bufferSize,
const UploadPageBlobFromOptions& options) const
{
BlobRestClient::PageBlob::CreatePageBlobOptions createOptions;
createOptions.BlobContentLength = bufferSize;
createOptions.HttpHeaders = options.HttpHeaders;
createOptions.Metadata = options.Metadata;
createOptions.Tier = options.Tier;
if (m_customerProvidedKey.HasValue())
{
createOptions.EncryptionKey = m_customerProvidedKey.GetValue().Key;
createOptions.EncryptionKeySha256 = m_customerProvidedKey.GetValue().KeyHash;
createOptions.EncryptionAlgorithm = m_customerProvidedKey.GetValue().Algorithm;
}
createOptions.EncryptionScope = m_encryptionScope;
auto createResult = BlobRestClient::PageBlob::Create(
options.Context, *m_pipeline, m_blobUrl.ToString(), createOptions);
constexpr int64_t c_defaultChunkSize = 8 * 1024 * 1024;
int64_t chunkSize
= options.ChunkSize.HasValue() ? options.ChunkSize.GetValue() : c_defaultChunkSize;
auto uploadPageFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) {
unused(chunkId, numChunks);
Azure::Core::Http::MemoryBodyStream contentStream(buffer + offset, length);
UploadPageBlobPagesOptions uploadPagesOptions;
uploadPagesOptions.Context = options.Context;
UploadPages(&contentStream, offset, uploadPagesOptions);
};
Details::ConcurrentTransfer(0, bufferSize, chunkSize, options.Concurrency, uploadPageFunc);
UploadPageBlobFromResult result;
result.ServerEncrypted = createResult->ServerEncrypted;
result.EncryptionKeySha256 = createResult->EncryptionKeySha256;
result.EncryptionScope = createResult->EncryptionScope;
return Azure::Core::Response<UploadPageBlobFromResult>(
std::move(result),
std::make_unique<Azure::Core::Http::RawResponse>(std::move(createResult.GetRawResponse())));
}
Azure::Core::Response<UploadPageBlobFromResult> PageBlobClient::UploadFrom(
const std::string& file,
const UploadPageBlobFromOptions& options) const
{
Details::FileReader fileReader(file);
BlobRestClient::PageBlob::CreatePageBlobOptions createOptions;
createOptions.BlobContentLength = fileReader.GetFileSize();
createOptions.HttpHeaders = options.HttpHeaders;
createOptions.Metadata = options.Metadata;
createOptions.Tier = options.Tier;
if (m_customerProvidedKey.HasValue())
{
createOptions.EncryptionKey = m_customerProvidedKey.GetValue().Key;
createOptions.EncryptionKeySha256 = m_customerProvidedKey.GetValue().KeyHash;
createOptions.EncryptionAlgorithm = m_customerProvidedKey.GetValue().Algorithm;
}
createOptions.EncryptionScope = m_encryptionScope;
auto createResult = BlobRestClient::PageBlob::Create(
options.Context, *m_pipeline, m_blobUrl.ToString(), createOptions);
constexpr int64_t c_defaultChunkSize = 8 * 1024 * 1024;
int64_t chunkSize
= options.ChunkSize.HasValue() ? options.ChunkSize.GetValue() : c_defaultChunkSize;
auto uploadPageFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) {
unused(chunkId, numChunks);
Azure::Core::Http::FileBodyStream contentStream(fileReader.GetHandle(), offset, length);
UploadPageBlobPagesOptions uploadPagesOptions;
uploadPagesOptions.Context = options.Context;
UploadPages(&contentStream, offset, uploadPagesOptions);
};
Details::ConcurrentTransfer(
0, fileReader.GetFileSize(), chunkSize, options.Concurrency, uploadPageFunc);
UploadPageBlobFromResult result;
result.ServerEncrypted = createResult->ServerEncrypted;
result.EncryptionKeySha256 = createResult->EncryptionKeySha256;
result.EncryptionScope = createResult->EncryptionScope;
return Azure::Core::Response<UploadPageBlobFromResult>(
std::move(result),
std::make_unique<Azure::Core::Http::RawResponse>(std::move(createResult.GetRawResponse())));
}
Azure::Core::Response<ResizePageBlobResult> PageBlobClient::Resize(
int64_t blobContentLength,
const ResizePageBlobOptions& options)
const ResizePageBlobOptions& options) const
{
BlobRestClient::PageBlob::ResizePageBlobOptions protocolLayerOptions;
protocolLayerOptions.BlobContentLength = blobContentLength;
@ -198,7 +288,7 @@ namespace Azure { namespace Storage { namespace Blobs {
}
Azure::Core::Response<GetPageBlobPageRangesResult> PageBlobClient::GetPageRanges(
const GetPageBlobPageRangesOptions& options)
const GetPageBlobPageRangesOptions& options) const
{
BlobRestClient::PageBlob::GetPageBlobPageRangesOptions protocolLayerOptions;
protocolLayerOptions.PreviousSnapshot = options.PreviousSnapshot;
@ -236,7 +326,7 @@ namespace Azure { namespace Storage { namespace Blobs {
Azure::Core::Response<StartCopyPageBlobIncrementalResult> PageBlobClient::StartCopyIncremental(
const std::string& sourceUri,
const StartCopyPageBlobIncrementalOptions& options)
const StartCopyPageBlobIncrementalOptions& options) const
{
BlobRestClient::PageBlob::StartCopyPageBlobIncrementalOptions protocolLayerOptions;
protocolLayerOptions.CopySource = sourceUri;

131
sdk/storage/test/blobs/block_blob_client_test.cpp
Просмотреть файл

@ -568,68 +568,89 @@ namespace Azure { namespace Storage { namespace Test {
TEST_F(BlockBlobClientTest, ConcurrentUpload)
{
std::string tempFilename = RandomString();
std::vector<uint8_t> blobContent = RandomBuffer(static_cast<std::size_t>(8_MB));
auto blockBlobClient = Azure::Storage::Blobs::BlockBlobClient::CreateFromConnectionString(
StandardStorageConnectionString(), m_containerName, RandomString());
auto testUploadFromBuffer = [&](int concurrency, int64_t blobSize) {
auto blockBlobClient = m_blobContainerClient->GetBlockBlobClient(RandomString());
Azure::Storage::Blobs::UploadBlockBlobFromOptions options;
options.ChunkSize = 1_MB;
options.Concurrency = concurrency;
options.HttpHeaders = m_blobUploadOptions.HttpHeaders;
options.HttpHeaders.ContentMd5.clear();
options.Metadata = m_blobUploadOptions.Metadata;
options.Tier = m_blobUploadOptions.Tier;
auto res = blockBlobClient.UploadFrom(
blobContent.data(), static_cast<std::size_t>(blobSize), options);
EXPECT_FALSE(res->ETag.empty());
EXPECT_FALSE(res->LastModified.empty());
auto properties = *blockBlobClient.GetProperties();
properties.HttpHeaders.ContentMd5.clear();
EXPECT_EQ(properties.ContentLength, blobSize);
EXPECT_EQ(properties.HttpHeaders, options.HttpHeaders);
EXPECT_EQ(properties.Metadata, options.Metadata);
EXPECT_EQ(properties.Tier.GetValue(), options.Tier.GetValue());
EXPECT_EQ(properties.ETag, res->ETag);
EXPECT_EQ(properties.LastModified, res->LastModified);
std::vector<uint8_t> downloadContent(static_cast<std::size_t>(blobSize), '\x00');
blockBlobClient.DownloadTo(downloadContent.data(), static_cast<std::size_t>(blobSize));
EXPECT_EQ(
downloadContent,
std::vector<uint8_t>(
blobContent.begin(), blobContent.begin() + static_cast<std::size_t>(blobSize)));
};
auto testUploadFromFile = [&](int concurrency, int64_t blobSize) {
auto blockBlobClient = m_blobContainerClient->GetBlockBlobClient(RandomString());
Azure::Storage::Blobs::UploadBlockBlobFromOptions options;
options.ChunkSize = 1_MB;
options.Concurrency = concurrency;
options.HttpHeaders = m_blobUploadOptions.HttpHeaders;
options.HttpHeaders.ContentMd5.clear();
options.Metadata = m_blobUploadOptions.Metadata;
options.Tier = m_blobUploadOptions.Tier;
std::string tempFilename = RandomString();
{
Azure::Storage::Details::FileWriter fileWriter(tempFilename);
fileWriter.Write(blobContent.data(), blobSize, 0);
}
auto res = blockBlobClient.UploadFrom(tempFilename, options);
EXPECT_FALSE(res->ETag.empty());
EXPECT_FALSE(res->LastModified.empty());
auto properties = *blockBlobClient.GetProperties();
properties.HttpHeaders.ContentMd5.clear();
EXPECT_EQ(properties.ContentLength, blobSize);
EXPECT_EQ(properties.HttpHeaders, options.HttpHeaders);
EXPECT_EQ(properties.Metadata, options.Metadata);
EXPECT_EQ(properties.Tier.GetValue(), options.Tier.GetValue());
EXPECT_EQ(properties.ETag, res->ETag);
EXPECT_EQ(properties.LastModified, res->LastModified);
std::vector<uint8_t> downloadContent(static_cast<std::size_t>(blobSize), '\x00');
blockBlobClient.DownloadTo(downloadContent.data(), static_cast<std::size_t>(blobSize));
EXPECT_EQ(
downloadContent,
std::vector<uint8_t>(
blobContent.begin(), blobContent.begin() + static_cast<std::size_t>(blobSize)));
DeleteFile(tempFilename);
};
std::vector<std::future<void>> futures;
for (int c : {1, 2, 5})
{
for (int64_t length :
for (int64_t l :
{0ULL, 1ULL, 2ULL, 2_KB, 4_KB, 999_KB, 1_MB, 2_MB - 1, 3_MB, 5_MB, 8_MB - 1234, 8_MB})
{
Azure::Storage::Blobs::UploadBlockBlobFromOptions options;
options.ChunkSize = 1_MB;
options.Concurrency = c;
options.HttpHeaders = m_blobUploadOptions.HttpHeaders;
options.HttpHeaders.ContentMd5.clear();
options.Metadata = m_blobUploadOptions.Metadata;
options.Tier = m_blobUploadOptions.Tier;
{
auto res = blockBlobClient.UploadFrom(
m_blobContent.data(), static_cast<std::size_t>(length), options);
EXPECT_FALSE(res->ETag.empty());
EXPECT_FALSE(res->LastModified.empty());
auto properties = *blockBlobClient.GetProperties();
properties.HttpHeaders.ContentMd5.clear();
EXPECT_EQ(properties.ContentLength, length);
EXPECT_EQ(properties.HttpHeaders, options.HttpHeaders);
EXPECT_EQ(properties.Metadata, options.Metadata);
EXPECT_EQ(properties.Tier.GetValue(), options.Tier.GetValue());
EXPECT_EQ(properties.ETag, res->ETag);
EXPECT_EQ(properties.LastModified, res->LastModified);
std::vector<uint8_t> downloadContent(static_cast<std::size_t>(length), '\x00');
blockBlobClient.DownloadTo(downloadContent.data(), static_cast<std::size_t>(length));
EXPECT_EQ(
downloadContent,
std::vector<uint8_t>(
m_blobContent.begin(), m_blobContent.begin() + static_cast<std::size_t>(length)));
}
{
{
Azure::Storage::Details::FileWriter fileWriter(tempFilename);
fileWriter.Write(m_blobContent.data(), length, 0);
}
auto res = blockBlobClient.UploadFrom(tempFilename, options);
EXPECT_FALSE(res->ETag.empty());
EXPECT_FALSE(res->LastModified.empty());
auto properties = *blockBlobClient.GetProperties();
properties.HttpHeaders.ContentMd5.clear();
EXPECT_EQ(properties.ContentLength, length);
EXPECT_EQ(properties.HttpHeaders, options.HttpHeaders);
EXPECT_EQ(properties.Metadata, options.Metadata);
EXPECT_EQ(properties.Tier.GetValue(), options.Tier.GetValue());
EXPECT_EQ(properties.ETag, res->ETag);
EXPECT_EQ(properties.LastModified, res->LastModified);
std::vector<uint8_t> downloadContent(static_cast<std::size_t>(length), '\x00');
blockBlobClient.DownloadTo(downloadContent.data(), static_cast<std::size_t>(length));
EXPECT_EQ(
downloadContent,
std::vector<uint8_t>(
m_blobContent.begin(), m_blobContent.begin() + static_cast<std::size_t>(length)));
}
ASSERT_GE(blobContent.size(), static_cast<std::size_t>(l));
futures.emplace_back(std::async(std::launch::async, testUploadFromBuffer, c, l));
futures.emplace_back(std::async(std::launch::async, testUploadFromFile, c, l));
}
}
DeleteFile(tempFilename);
for (auto& f : futures)
{
f.get();
}
}
TEST_F(BlockBlobClientTest, DownloadError)

110
sdk/storage/test/blobs/page_blob_client_test.cpp
Просмотреть файл

@ -2,7 +2,12 @@
// SPDX-License-Identifier: MIT
#include "page_blob_client_test.hpp"
#include <future>
#include <vector>
#include "common/crypt.hpp"
#include "common/file_io.hpp"
namespace Azure { namespace Storage { namespace Test {
@ -244,4 +249,109 @@ namespace Azure { namespace Storage { namespace Test {
EXPECT_THROW(pageBlobClient.UploadPages(&pageContent, 0, options), StorageError);
}
TEST_F(PageBlobClientTest, ConcurrentUploadFromNonExistingFile)
{
auto pageBlobClient = m_blobContainerClient->GetPageBlobClient(RandomString());
std::string emptyFilename = RandomString();
EXPECT_THROW(pageBlobClient.UploadFrom(emptyFilename), std::runtime_error);
EXPECT_THROW(pageBlobClient.Delete(), StorageError);
}
TEST_F(PageBlobClientTest, ConcurrentUploadEmptyBlob)
{
auto pageBlobClient = m_blobContainerClient->GetPageBlobClient(RandomString());
std::vector<uint8_t> emptyContent;
pageBlobClient.UploadFrom(emptyContent.data(), emptyContent.size());
EXPECT_NO_THROW(pageBlobClient.Delete());
std::string emptyFilename = RandomString();
{
Details::FileWriter writer(emptyFilename);
}
pageBlobClient.UploadFrom(emptyFilename);
EXPECT_NO_THROW(pageBlobClient.Delete());
DeleteFile(emptyFilename);
}
TEST_F(PageBlobClientTest, ConcurrentUpload)
{
std::vector<uint8_t> blobContent = RandomBuffer(static_cast<std::size_t>(8_MB));
auto testUploadFromBuffer = [&](int concurrency, int64_t blobSize) {
auto pageBlobClient = m_blobContainerClient->GetPageBlobClient(RandomString());
Azure::Storage::Blobs::UploadPageBlobFromOptions options;
options.ChunkSize = 512_KB;
options.Concurrency = concurrency;
options.HttpHeaders = m_blobUploadOptions.HttpHeaders;
options.HttpHeaders.ContentMd5.clear();
options.Metadata = m_blobUploadOptions.Metadata;
auto res = pageBlobClient.UploadFrom(
blobContent.data(), static_cast<std::size_t>(blobSize), options);
EXPECT_TRUE(res->ServerEncrypted.HasValue());
auto properties = *pageBlobClient.GetProperties();
properties.HttpHeaders.ContentMd5.clear();
EXPECT_EQ(properties.ContentLength, blobSize);
EXPECT_EQ(properties.Metadata, options.Metadata);
std::vector<uint8_t> downloadContent(static_cast<std::size_t>(blobSize), '\x00');
pageBlobClient.DownloadTo(downloadContent.data(), static_cast<std::size_t>(blobSize));
EXPECT_EQ(
downloadContent,
std::vector<uint8_t>(
blobContent.begin(), blobContent.begin() + static_cast<std::size_t>(blobSize)));
};
auto testUploadFromFile = [&](int concurrency, int64_t blobSize) {
auto pageBlobClient = m_blobContainerClient->GetPageBlobClient(RandomString());
Azure::Storage::Blobs::UploadPageBlobFromOptions options;
options.ChunkSize = 512_KB;
options.Concurrency = concurrency;
options.HttpHeaders = m_blobUploadOptions.HttpHeaders;
options.HttpHeaders.ContentMd5.clear();
options.Metadata = m_blobUploadOptions.Metadata;
std::string tempFilename = RandomString();
{
Azure::Storage::Details::FileWriter fileWriter(tempFilename);
fileWriter.Write(blobContent.data(), blobSize, 0);
}
auto res = pageBlobClient.UploadFrom(tempFilename, options);
EXPECT_TRUE(res->ServerEncrypted.HasValue());
auto properties = *pageBlobClient.GetProperties();
properties.HttpHeaders.ContentMd5.clear();
EXPECT_EQ(properties.ContentLength, blobSize);
EXPECT_EQ(properties.Metadata, options.Metadata);
std::vector<uint8_t> downloadContent(static_cast<std::size_t>(blobSize), '\x00');
pageBlobClient.DownloadTo(downloadContent.data(), static_cast<std::size_t>(blobSize));
EXPECT_EQ(
downloadContent,
std::vector<uint8_t>(
blobContent.begin(), blobContent.begin() + static_cast<std::size_t>(blobSize)));
DeleteFile(tempFilename);
};
std::vector<std::future<void>> futures;
for (int c : {1, 2, 5})
{
for (int64_t l : {0ULL, 512ULL, 1_KB, 4_KB, 1_MB, 4_MB + 512})
{
ASSERT_GE(blobContent.size(), static_cast<std::size_t>(l));
futures.emplace_back(std::async(std::launch::async, testUploadFromBuffer, c, l));
futures.emplace_back(std::async(std::launch::async, testUploadFromFile, c, l));
}
}
for (auto& f : futures)
{
f.get();
}
}
}}} // namespace Azure::Storage::Test