Azure
/
azure-cosmos-dotnet-v3
зеркало из https://github.com/Azure/azure-cosmos-dotnet-v3.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

[Client Encryption] Diagnostics: Adds basic diagnostics information regarding encryption / decryption operation (#2633) 

Modifying ResponseMessage.Diagnostics to include information related to the encryption and decryption operation from the Encryption package. This lays the foundation for adding any required diagnostics information more easily in future. Change covers all the APIs - Create, Replace, Upsert, Read, ReadMany, Patch, transactional batch and feed operations.
This commit is contained in:
anujtoshniwal 2021-09-23 19:07:57 +05:30 коммит произвёл GitHub
Родитель a154351a8a
Коммит b29a58c1ce
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
15 изменённых файлов: 737 добавлений и 637 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

4
Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableFeedResponse.cs
Просмотреть файл

@ -33,9 +33,9 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom
public override CosmosDiagnostics Diagnostics { get; }
#if SDKPROJECTREF
// TODO: https://github.com/Azure/azure-cosmos-dotnet-v3/issues/2750
public override string IndexMetrics => null;
#endif
public override IEnumerator<T> GetEnumerator()
{
return this.Resource.GetEnumerator();

79
Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs
Просмотреть файл

@ -197,31 +197,22 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom
}
EncryptionProperties encryptionProperties = encryptionPropertiesJObj.ToObject<EncryptionProperties>();
DecryptionContext decryptionContext;
switch (encryptionProperties.EncryptionAlgorithm)
DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch
{
case CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized:
decryptionContext = await EncryptionProcessor.MdeEncAlgoDecryptObjectAsync(
itemJObj,
encryptor,
encryptionProperties,
diagnosticsContext,
cancellationToken);
break;
case CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized:
decryptionContext = await EncryptionProcessor.LegacyEncAlgoDecryptContentAsync(
itemJObj,
encryptionProperties,
encryptor,
diagnosticsContext,
cancellationToken);
break;
default:
throw new NotSupportedException($"Encryption Algorithm : {encryptionProperties.EncryptionAlgorithm} is not supported.");
}
CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await EncryptionProcessor.MdeEncAlgoDecryptObjectAsync(
itemJObj,
encryptor,
encryptionProperties,
diagnosticsContext,
cancellationToken),
CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await EncryptionProcessor.LegacyEncAlgoDecryptContentAsync(
itemJObj,
encryptionProperties,
encryptor,
diagnosticsContext,
cancellationToken),
_ => throw new NotSupportedException($"Encryption Algorithm : {encryptionProperties.EncryptionAlgorithm} is not supported."),
};
input.Dispose();
return (EncryptionProcessor.BaseSerializer.ToStream(itemJObj), decryptionContext);
@ -245,32 +236,22 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom
}
EncryptionProperties encryptionProperties = encryptionPropertiesJObj.ToObject<EncryptionProperties>();
DecryptionContext decryptionContext;
switch (encryptionProperties.EncryptionAlgorithm)
DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch
{
case CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized:
decryptionContext = await EncryptionProcessor.MdeEncAlgoDecryptObjectAsync(
document,
encryptor,
encryptionProperties,
diagnosticsContext,
cancellationToken);
break;
case CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized:
decryptionContext = await EncryptionProcessor.LegacyEncAlgoDecryptContentAsync(
document,
encryptionProperties,
encryptor,
diagnosticsContext,
cancellationToken);
break;
default:
throw new NotSupportedException($"Encryption Algorithm : {encryptionProperties.EncryptionAlgorithm} is not supported.");
}
CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await EncryptionProcessor.MdeEncAlgoDecryptObjectAsync(
document,
encryptor,
encryptionProperties,
diagnosticsContext,
cancellationToken),
CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await EncryptionProcessor.LegacyEncAlgoDecryptContentAsync(
document,
encryptionProperties,
encryptor,
diagnosticsContext,
cancellationToken),
_ => throw new NotSupportedException($"Encryption Algorithm : {encryptionProperties.EncryptionAlgorithm} is not supported."),
};
return (document, decryptionContext);
}

2
Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj
Просмотреть файл

@ -25,7 +25,7 @@
</ItemGroup>
<ItemGroup Condition=" '$(SdkProjectRef)' != 'True' ">
<PackageReference Include="Microsoft.Azure.Cosmos" Version="3.20.0-preview" />
<PackageReference Include="Microsoft.Azure.Cosmos" Version="3.21.0-preview" />
</ItemGroup>
<ItemGroup Condition=" '$(SdkProjectRef)' == 'True' ">

10
Microsoft.Azure.Cosmos.Encryption/src/Constants.cs
Просмотреть файл

@ -6,9 +6,17 @@ namespace Microsoft.Azure.Cosmos.Encryption
{
internal static class Constants
{
public const string DiagnosticsCoreDiagnostics = "CoreDiagnostics";
public const string DiagnosticsDecryptOperation = "Decrypt";
public const string DiagnosticsDuration = "Duration in milliseconds";
public const string DiagnosticsEncryptionDiagnostics = "EncryptionDiagnostics";
public const string DiagnosticsEncryptOperation = "Encrypt";
public const string DiagnosticsPropertiesEncryptedCount = "Properties Encrypted Count";
public const string DiagnosticsPropertiesDecryptedCount = "Properties Decrypted Count";
public const string DiagnosticsStartTime = "Start time";
public const string DocumentsResourcePropertyName = "Documents";
public const string SubStatusHeader = "x-ms-substatus";
public const string IncorrectContainerRidSubStatus = "1024";
public const string SubStatusHeader = "x-ms-substatus";
public const int SupportedClientEncryptionPolicyFormatVersion = 1;
}
}

35
Microsoft.Azure.Cosmos.Encryption/src/CosmosDiagnosticsContext.cs
Просмотреть файл

@ -1,35 +0,0 @@
//------------------------------------------------------------
// Copyright (c) Microsoft Corporation. All rights reserved.
//------------------------------------------------------------
namespace Microsoft.Azure.Cosmos.Encryption
{
using System;
/// <summary>
/// This is an empty implementation of CosmosDiagnosticsContext which has been plumbed through the DataEncryptionKeyProvider & EncryptionContainer.
/// This may help adding diagnostics more easily in future.
/// </summary>
internal class CosmosDiagnosticsContext
{
private static readonly CosmosDiagnosticsContext UnusedSingleton = new CosmosDiagnosticsContext();
private static readonly IDisposable UnusedScopeSingleton = new Scope();
public static CosmosDiagnosticsContext Create(RequestOptions options)
{
return CosmosDiagnosticsContext.UnusedSingleton;
}
public IDisposable CreateScope(string scope)
{
return CosmosDiagnosticsContext.UnusedScopeSingleton;
}
private class Scope : IDisposable
{
public void Dispose()
{
}
}
}
}

492
Microsoft.Azure.Cosmos.Encryption/src/EncryptionContainer.cs
Просмотреть файл

@ -20,13 +20,8 @@ namespace Microsoft.Azure.Cosmos.Encryption
private readonly AsyncCache<string, EncryptionSettings> encryptionSettingsByContainerName;
public CosmosSerializer CosmosSerializer { get; }
public CosmosResponseFactory ResponseFactory { get; }
public EncryptionCosmosClient EncryptionCosmosClient { get; }
/// <summary>
/// Initializes a new instance of the <see cref="EncryptionContainer"/> class.
/// All the operations / requests for exercising client-side encryption functionality need to be made using this EncryptionContainer instance.
/// </summary>
/// <param name="container">Regular cosmos container.</param>
@ -42,6 +37,12 @@ namespace Microsoft.Azure.Cosmos.Encryption
this.encryptionSettingsByContainerName = new AsyncCache<string, EncryptionSettings>();
}
public CosmosSerializer CosmosSerializer { get; }
public CosmosResponseFactory ResponseFactory { get; }
public EncryptionCosmosClient EncryptionCosmosClient { get; }
public override string Id => this.container.Id;
public override Conflicts Conflicts => this.container.Conflicts;
@ -66,23 +67,17 @@ namespace Microsoft.Azure.Cosmos.Encryption
throw new NotSupportedException($"{nameof(partitionKey)} cannot be null for operations using {nameof(EncryptionContainer)}.");
}
CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions);
using (diagnosticsContext.CreateScope("CreateItem"))
ResponseMessage responseMessage;
using (Stream itemStream = this.CosmosSerializer.ToStream<T>(item))
{
ResponseMessage responseMessage;
using (Stream itemStream = this.CosmosSerializer.ToStream<T>(item))
{
responseMessage = await this.CreateItemHelperAsync(
itemStream,
partitionKey.Value,
requestOptions,
diagnosticsContext,
cancellationToken);
}
return this.ResponseFactory.CreateItemResponse<T>(responseMessage);
responseMessage = await this.CreateItemHelperAsync(
itemStream,
partitionKey.Value,
requestOptions,
cancellationToken);
}
return this.ResponseFactory.CreateItemResponse<T>(responseMessage);
}
public override async Task<ResponseMessage> CreateItemStreamAsync(
@ -96,16 +91,11 @@ namespace Microsoft.Azure.Cosmos.Encryption
throw new ArgumentNullException(nameof(streamPayload));
}
CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions);
using (diagnosticsContext.CreateScope("CreateItemStream"))
{
return await this.CreateItemHelperAsync(
streamPayload,
partitionKey,
requestOptions,
diagnosticsContext,
cancellationToken);
}
return await this.CreateItemHelperAsync(
streamPayload,
partitionKey,
requestOptions,
cancellationToken);
}
public override Task<ItemResponse<T>> DeleteItemAsync<T>(
@ -140,20 +130,13 @@ namespace Microsoft.Azure.Cosmos.Encryption
ItemRequestOptions requestOptions = null,
CancellationToken cancellationToken = default)
{
CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions);
using (diagnosticsContext.CreateScope("ReadItem"))
{
ResponseMessage responseMessage;
ResponseMessage responseMessage = await this.ReadItemHelperAsync(
id,
partitionKey,
requestOptions,
cancellationToken);
responseMessage = await this.ReadItemHelperAsync(
id,
partitionKey,
requestOptions,
diagnosticsContext,
cancellationToken);
return this.ResponseFactory.CreateItemResponse<T>(responseMessage);
}
return this.ResponseFactory.CreateItemResponse<T>(responseMessage);
}
public override async Task<ResponseMessage> ReadItemStreamAsync(
@ -162,16 +145,11 @@ namespace Microsoft.Azure.Cosmos.Encryption
ItemRequestOptions requestOptions = null,
CancellationToken cancellationToken = default)
{
CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions);
using (diagnosticsContext.CreateScope("ReadItemStream"))
{
return await this.ReadItemHelperAsync(
id,
partitionKey,
requestOptions,
diagnosticsContext,
cancellationToken);
}
return await this.ReadItemHelperAsync(
id,
partitionKey,
requestOptions,
cancellationToken);
}
public override async Task<ItemResponse<T>> ReplaceItemAsync<T>(
@ -196,24 +174,19 @@ namespace Microsoft.Azure.Cosmos.Encryption
throw new NotSupportedException($"{nameof(partitionKey)} cannot be null for operations using {nameof(EncryptionContainer)}.");
}
CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions);
using (diagnosticsContext.CreateScope("ReplaceItem"))
ResponseMessage responseMessage;
using (Stream itemStream = this.CosmosSerializer.ToStream<T>(item))
{
ResponseMessage responseMessage;
using (Stream itemStream = this.CosmosSerializer.ToStream<T>(item))
{
responseMessage = await this.ReplaceItemHelperAsync(
itemStream,
id,
partitionKey.Value,
requestOptions,
diagnosticsContext,
cancellationToken);
}
return this.ResponseFactory.CreateItemResponse<T>(responseMessage);
responseMessage = await this.ReplaceItemHelperAsync(
itemStream,
id,
partitionKey.Value,
requestOptions,
cancellationToken);
}
return this.ResponseFactory.CreateItemResponse<T>(responseMessage);
}
public override async Task<ResponseMessage> ReplaceItemStreamAsync(
@ -233,17 +206,12 @@ namespace Microsoft.Azure.Cosmos.Encryption
throw new ArgumentNullException(nameof(streamPayload));
}
CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions);
using (diagnosticsContext.CreateScope("ReplaceItemStream"))
{
return await this.ReplaceItemHelperAsync(
streamPayload,
id,
partitionKey,
requestOptions,
diagnosticsContext,
cancellationToken);
}
return await this.ReplaceItemHelperAsync(
streamPayload,
id,
partitionKey,
requestOptions,
cancellationToken);
}
public override async Task<ItemResponse<T>> UpsertItemAsync<T>(
@ -262,23 +230,18 @@ namespace Microsoft.Azure.Cosmos.Encryption
throw new NotSupportedException($"{nameof(partitionKey)} cannot be null for operations using {nameof(EncryptionContainer)}.");
}
CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions);
using (diagnosticsContext.CreateScope("UpsertItem"))
ResponseMessage responseMessage;
using (Stream itemStream = this.CosmosSerializer.ToStream<T>(item))
{
ResponseMessage responseMessage;
using (Stream itemStream = this.CosmosSerializer.ToStream<T>(item))
{
responseMessage = await this.UpsertItemHelperAsync(
itemStream,
partitionKey.Value,
requestOptions,
diagnosticsContext,
cancellationToken);
}
return this.ResponseFactory.CreateItemResponse<T>(responseMessage);
responseMessage = await this.UpsertItemHelperAsync(
itemStream,
partitionKey.Value,
requestOptions,
cancellationToken);
}
return this.ResponseFactory.CreateItemResponse<T>(responseMessage);
}
public override async Task<ResponseMessage> UpsertItemStreamAsync(
@ -292,16 +255,11 @@ namespace Microsoft.Azure.Cosmos.Encryption
throw new ArgumentNullException(nameof(streamPayload));
}
CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions);
using (diagnosticsContext.CreateScope("UpsertItemStream"))
{
return await this.UpsertItemHelperAsync(
streamPayload,
partitionKey,
requestOptions,
diagnosticsContext,
cancellationToken);
}
return await this.UpsertItemHelperAsync(
streamPayload,
partitionKey,
requestOptions,
cancellationToken);
}
public override TransactionalBatch CreateTransactionalBatch(
@ -452,15 +410,7 @@ namespace Microsoft.Azure.Cosmos.Encryption
string continuationToken = null,
QueryRequestOptions requestOptions = null)
{
QueryRequestOptions clonedRequestOptions;
if (requestOptions != null)
{
clonedRequestOptions = (QueryRequestOptions)requestOptions.ShallowCopy();
}
else
{
clonedRequestOptions = new QueryRequestOptions();
}
QueryRequestOptions clonedRequestOptions = requestOptions != null ? (QueryRequestOptions)requestOptions.ShallowCopy() : new QueryRequestOptions();
return new EncryptionFeedIterator(
this.container.GetItemQueryStreamIterator(
@ -476,15 +426,7 @@ namespace Microsoft.Azure.Cosmos.Encryption
string continuationToken = null,
QueryRequestOptions requestOptions = null)
{
QueryRequestOptions clonedRequestOptions;
if (requestOptions != null)
{
clonedRequestOptions = (QueryRequestOptions)requestOptions.ShallowCopy();
}
else
{
clonedRequestOptions = new QueryRequestOptions();
}
QueryRequestOptions clonedRequestOptions = requestOptions != null ? (QueryRequestOptions)requestOptions.ShallowCopy() : new QueryRequestOptions();
return new EncryptionFeedIterator(
this.container.GetItemQueryStreamIterator(
@ -525,15 +467,7 @@ namespace Microsoft.Azure.Cosmos.Encryption
string continuationToken,
QueryRequestOptions requestOptions = null)
{
QueryRequestOptions clonedRequestOptions;
if (requestOptions != null)
{
clonedRequestOptions = (QueryRequestOptions)requestOptions.ShallowCopy();
}
else
{
clonedRequestOptions = new QueryRequestOptions();
}
QueryRequestOptions clonedRequestOptions = requestOptions != null ? (QueryRequestOptions)requestOptions.ShallowCopy() : new QueryRequestOptions();
return new EncryptionFeedIterator(
this.container.GetItemQueryStreamIterator(
@ -572,15 +506,9 @@ namespace Microsoft.Azure.Cosmos.Encryption
ChangeFeedMode changeFeedMode,
ChangeFeedRequestOptions changeFeedRequestOptions = null)
{
ChangeFeedRequestOptions clonedchangeFeedRequestOptions;
if (changeFeedRequestOptions != null)
{
clonedchangeFeedRequestOptions = (ChangeFeedRequestOptions)changeFeedRequestOptions.ShallowCopy();
}
else
{
clonedchangeFeedRequestOptions = new ChangeFeedRequestOptions();
}
ChangeFeedRequestOptions clonedchangeFeedRequestOptions = changeFeedRequestOptions != null
? (ChangeFeedRequestOptions)changeFeedRequestOptions.ShallowCopy()
: new ChangeFeedRequestOptions();
return new EncryptionFeedIterator(
this.container.GetChangeFeedStreamIterator(
@ -648,96 +576,28 @@ namespace Microsoft.Azure.Cosmos.Encryption
obsoleteEncryptionSettings: null,
cancellationToken: cancellationToken);
CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions);
using (diagnosticsContext.CreateScope("PatchItem"))
{
List<PatchOperation> encryptedPatchOperations = await this.EncryptPatchOperationsAsync(
patchOperations,
encryptionSettings,
cancellationToken);
EncryptionDiagnosticsContext encryptionDiagnosticsContext = new EncryptionDiagnosticsContext();
List<PatchOperation> encryptedPatchOperations = await this.EncryptPatchOperationsAsync(
patchOperations,
encryptionSettings,
encryptionDiagnosticsContext,
cancellationToken);
ResponseMessage responseMessage = await this.container.PatchItemStreamAsync(
id,
partitionKey,
encryptedPatchOperations,
requestOptions,
cancellationToken);
ResponseMessage responseMessage = await this.container.PatchItemStreamAsync(
id,
partitionKey,
encryptedPatchOperations,
requestOptions,
cancellationToken);
responseMessage.Content = await EncryptionProcessor.DecryptAsync(
responseMessage.Content,
encryptionSettings,
diagnosticsContext,
cancellationToken);
responseMessage.Content = await EncryptionProcessor.DecryptAsync(
responseMessage.Content,
encryptionSettings,
encryptionDiagnosticsContext,
cancellationToken);
return responseMessage;
}
}
internal async Task<List<PatchOperation>> EncryptPatchOperationsAsync(
IReadOnlyList<PatchOperation> patchOperations,
EncryptionSettings encryptionSettings,
CancellationToken cancellationToken = default)
{
List<PatchOperation> encryptedPatchOperations = new List<PatchOperation>(patchOperations.Count);
foreach (PatchOperation patchOperation in patchOperations)
{
if (patchOperation.OperationType == PatchOperationType.Remove)
{
encryptedPatchOperations.Add(patchOperation);
continue;
}
if (string.IsNullOrWhiteSpace(patchOperation.Path) || patchOperation.Path[0] != '/')
{
throw new ArgumentException($"Invalid path '{patchOperation.Path}'.");
}
// get the top level path's encryption setting.
EncryptionSettingForProperty settingforProperty = encryptionSettings.GetEncryptionSettingForProperty(
patchOperation.Path.Split('/')[1]);
// non-encrypted path
if (settingforProperty == null)
{
encryptedPatchOperations.Add(patchOperation);
continue;
}
else if (patchOperation.OperationType == PatchOperationType.Increment)
{
throw new InvalidOperationException($"Increment patch operation is not allowed for encrypted path '{patchOperation.Path}'.");
}
if (!patchOperation.TrySerializeValueParameter(this.CosmosSerializer, out Stream valueParam))
{
throw new ArgumentException($"Cannot serialize value parameter for operation: {patchOperation.OperationType}, path: {patchOperation.Path}.");
}
Stream encryptedPropertyValue = await EncryptionProcessor.EncryptValueStreamAsync(
valueParam,
settingforProperty,
cancellationToken);
switch (patchOperation.OperationType)
{
case PatchOperationType.Add:
encryptedPatchOperations.Add(PatchOperation.Add(patchOperation.Path, encryptedPropertyValue));
break;
case PatchOperationType.Replace:
encryptedPatchOperations.Add(PatchOperation.Replace(patchOperation.Path, encryptedPropertyValue));
break;
case PatchOperationType.Set:
encryptedPatchOperations.Add(PatchOperation.Set(patchOperation.Path, encryptedPropertyValue));
break;
default:
throw new NotSupportedException(nameof(patchOperation.OperationType));
}
}
return encryptedPatchOperations;
encryptionDiagnosticsContext.AddEncryptionDiagnosticsToResponseMessage(responseMessage);
return responseMessage;
}
public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder<T>(
@ -815,9 +675,10 @@ namespace Microsoft.Azure.Cosmos.Encryption
obsoleteEncryptionSettings: null,
cancellationToken: cancellationToken);
Stream decryptedChanges = await this.DeserializeAndDecryptResponseAsync(
Stream decryptedChanges = await EncryptionProcessor.DeserializeAndDecryptResponseAsync(
changes,
encryptionSettings,
operationDiagnostics: null,
cancellationToken);
// Call the original passed in delegate
@ -841,9 +702,10 @@ namespace Microsoft.Azure.Cosmos.Encryption
obsoleteEncryptionSettings: null,
cancellationToken: cancellationToken);
Stream decryptedChanges = await this.DeserializeAndDecryptResponseAsync(
Stream decryptedChanges = await EncryptionProcessor.DeserializeAndDecryptResponseAsync(
changes,
encryptionSettings,
operationDiagnostics: null,
cancellationToken);
// Call the original passed in delegate
@ -886,57 +748,87 @@ namespace Microsoft.Azure.Cosmos.Encryption
cancellationToken: cancellationToken);
}
internal async Task<Stream> DeserializeAndDecryptResponseAsync(
Stream content,
EncryptionSettings encryptionSettings,
CancellationToken cancellationToken)
internal async Task<List<PatchOperation>> EncryptPatchOperationsAsync(
IReadOnlyList<PatchOperation> patchOperations,
EncryptionSettings encryptionSettings,
EncryptionDiagnosticsContext operationDiagnostics,
CancellationToken cancellationToken = default)
{
if (!encryptionSettings.PropertiesToEncrypt.Any())
{
return content;
}
List<PatchOperation> encryptedPatchOperations = new List<PatchOperation>(patchOperations.Count);
operationDiagnostics.Begin(Constants.DiagnosticsEncryptOperation);
int propertiesEncryptedCount = 0;
JObject contentJObj = EncryptionProcessor.BaseSerializer.FromStream<JObject>(content);
if (!(contentJObj.SelectToken(Constants.DocumentsResourcePropertyName) is JArray documents))
foreach (PatchOperation patchOperation in patchOperations)
{
throw new InvalidOperationException("Feed Response body contract was violated. Feed response did not have an array of Documents. ");
}
foreach (JToken value in documents)
{
if (value is not JObject document)
if (patchOperation.OperationType == PatchOperationType.Remove)
{
encryptedPatchOperations.Add(patchOperation);
continue;
}
await EncryptionProcessor.DecryptAsync(
document,
encryptionSettings,
if (string.IsNullOrWhiteSpace(patchOperation.Path) || patchOperation.Path[0] != '/')
{
throw new ArgumentException($"Invalid path '{patchOperation.Path}'.");
}
// get the top level path's encryption setting.
EncryptionSettingForProperty settingforProperty = encryptionSettings.GetEncryptionSettingForProperty(
patchOperation.Path.Split('/')[1]);
// non-encrypted path
if (settingforProperty == null)
{
encryptedPatchOperations.Add(patchOperation);
continue;
}
else if (patchOperation.OperationType == PatchOperationType.Increment)
{
throw new InvalidOperationException($"Increment patch operation is not allowed for encrypted path '{patchOperation.Path}'.");
}
if (!patchOperation.TrySerializeValueParameter(this.CosmosSerializer, out Stream valueParam))
{
throw new ArgumentException($"Cannot serialize value parameter for operation: {patchOperation.OperationType}, path: {patchOperation.Path}.");
}
Stream encryptedPropertyValue = await EncryptionProcessor.EncryptValueStreamAsync(
valueParam,
settingforProperty,
cancellationToken);
propertiesEncryptedCount++;
switch (patchOperation.OperationType)
{
case PatchOperationType.Add:
encryptedPatchOperations.Add(PatchOperation.Add(patchOperation.Path, encryptedPropertyValue));
break;
case PatchOperationType.Replace:
encryptedPatchOperations.Add(PatchOperation.Replace(patchOperation.Path, encryptedPropertyValue));
break;
case PatchOperationType.Set:
encryptedPatchOperations.Add(PatchOperation.Set(patchOperation.Path, encryptedPropertyValue));
break;
default:
throw new NotSupportedException(nameof(patchOperation.OperationType));
}
}
// the contents get decrypted in place by DecryptAsync.
return EncryptionProcessor.BaseSerializer.ToStream(contentJObj);
operationDiagnostics?.End(propertiesEncryptedCount);
return encryptedPatchOperations;
}
/// <summary>
/// Returns a cloned copy of the passed RequestOptions if passed else creates a new ItemRequestOptions.
/// </summary>
/// <param name="itemRequestOptions"> Original ItemRequestOptions</param>
/// <param name="itemRequestOptions"> Original ItemRequestOptions.</param>
/// <returns> ItemRequestOptions.</returns>
private static ItemRequestOptions GetClonedItemRequestOptions(ItemRequestOptions itemRequestOptions)
{
ItemRequestOptions clonedRequestOptions;
if (itemRequestOptions != null)
{
clonedRequestOptions = (ItemRequestOptions)itemRequestOptions.ShallowCopy();
}
else
{
clonedRequestOptions = new ItemRequestOptions();
}
ItemRequestOptions clonedRequestOptions = itemRequestOptions != null ? (ItemRequestOptions)itemRequestOptions.ShallowCopy() : new ItemRequestOptions();
return clonedRequestOptions;
}
@ -945,7 +837,6 @@ namespace Microsoft.Azure.Cosmos.Encryption
Stream streamPayload,
PartitionKey partitionKey,
ItemRequestOptions requestOptions,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken,
bool isRetry = false)
{
@ -959,11 +850,12 @@ namespace Microsoft.Azure.Cosmos.Encryption
cancellationToken);
}
EncryptionDiagnosticsContext encryptionDiagnosticsContext = new EncryptionDiagnosticsContext();
streamPayload = await EncryptionProcessor.EncryptAsync(
streamPayload,
encryptionSettings,
diagnosticsContext,
cancellationToken);
streamPayload,
encryptionSettings,
encryptionDiagnosticsContext,
cancellationToken);
ItemRequestOptions clonedRequestOptions = requestOptions;
@ -1000,7 +892,6 @@ namespace Microsoft.Azure.Cosmos.Encryption
streamPayload = await this.DecryptStreamPayloadAndUpdateEncryptionSettingsAsync(
streamPayload,
encryptionSettings,
diagnosticsContext,
cancellationToken);
// we try to recreate the item with the StreamPayload(to be encrypted) now that the encryptionSettings would have been updated with latest values if any.
@ -1008,17 +899,17 @@ namespace Microsoft.Azure.Cosmos.Encryption
streamPayload,
partitionKey,
clonedRequestOptions,
diagnosticsContext,
cancellationToken,
isRetry: true);
}
responseMessage.Content = await EncryptionProcessor.DecryptAsync(
responseMessage.Content,
encryptionSettings,
diagnosticsContext,
cancellationToken);
responseMessage.Content,
encryptionSettings,
encryptionDiagnosticsContext,
cancellationToken);
encryptionDiagnosticsContext.AddEncryptionDiagnosticsToResponseMessage(responseMessage);
return responseMessage;
}
@ -1026,7 +917,6 @@ namespace Microsoft.Azure.Cosmos.Encryption
string id,
PartitionKey partitionKey,
ItemRequestOptions requestOptions,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken,
bool isRetry = false)
{
@ -1069,17 +959,18 @@ namespace Microsoft.Azure.Cosmos.Encryption
id,
partitionKey,
clonedRequestOptions,
diagnosticsContext,
cancellationToken,
isRetry: true);
}
EncryptionDiagnosticsContext encryptionDiagnosticsContext = new EncryptionDiagnosticsContext();
responseMessage.Content = await EncryptionProcessor.DecryptAsync(
responseMessage.Content,
encryptionSettings,
diagnosticsContext,
encryptionDiagnosticsContext,
cancellationToken);
encryptionDiagnosticsContext.AddEncryptionDiagnosticsToResponseMessage(responseMessage);
return responseMessage;
}
@ -1088,7 +979,6 @@ namespace Microsoft.Azure.Cosmos.Encryption
string id,
PartitionKey partitionKey,
ItemRequestOptions requestOptions,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken,
bool isRetry = false)
{
@ -1108,10 +998,11 @@ namespace Microsoft.Azure.Cosmos.Encryption
cancellationToken);
}
EncryptionDiagnosticsContext encryptionDiagnosticsContext = new EncryptionDiagnosticsContext();
streamPayload = await EncryptionProcessor.EncryptAsync(
streamPayload,
encryptionSettings,
diagnosticsContext,
encryptionDiagnosticsContext,
cancellationToken);
ItemRequestOptions clonedRequestOptions = requestOptions;
@ -1139,7 +1030,6 @@ namespace Microsoft.Azure.Cosmos.Encryption
streamPayload = await this.DecryptStreamPayloadAndUpdateEncryptionSettingsAsync(
streamPayload,
encryptionSettings,
diagnosticsContext,
cancellationToken);
return await this.ReplaceItemHelperAsync(
@ -1147,7 +1037,6 @@ namespace Microsoft.Azure.Cosmos.Encryption
id,
partitionKey,
clonedRequestOptions,
diagnosticsContext,
cancellationToken,
isRetry: true);
}
@ -1155,9 +1044,10 @@ namespace Microsoft.Azure.Cosmos.Encryption
responseMessage.Content = await EncryptionProcessor.DecryptAsync(
responseMessage.Content,
encryptionSettings,
diagnosticsContext,
encryptionDiagnosticsContext,
cancellationToken);
encryptionDiagnosticsContext.AddEncryptionDiagnosticsToResponseMessage(responseMessage);
return responseMessage;
}
@ -1165,7 +1055,6 @@ namespace Microsoft.Azure.Cosmos.Encryption
Stream streamPayload,
PartitionKey partitionKey,
ItemRequestOptions requestOptions,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken,
bool isRetry = false)
{
@ -1184,10 +1073,11 @@ namespace Microsoft.Azure.Cosmos.Encryption
cancellationToken);
}
EncryptionDiagnosticsContext encryptionDiagnosticsContext = new EncryptionDiagnosticsContext();
streamPayload = await EncryptionProcessor.EncryptAsync(
streamPayload,
encryptionSettings,
diagnosticsContext,
encryptionDiagnosticsContext,
cancellationToken);
ItemRequestOptions clonedRequestOptions = requestOptions;
@ -1214,14 +1104,12 @@ namespace Microsoft.Azure.Cosmos.Encryption
streamPayload = await this.DecryptStreamPayloadAndUpdateEncryptionSettingsAsync(
streamPayload,
encryptionSettings,
diagnosticsContext,
cancellationToken);
return await this.UpsertItemHelperAsync(
streamPayload,
partitionKey,
clonedRequestOptions,
diagnosticsContext,
cancellationToken,
isRetry: true);
}
@ -1229,33 +1117,32 @@ namespace Microsoft.Azure.Cosmos.Encryption
responseMessage.Content = await EncryptionProcessor.DecryptAsync(
responseMessage.Content,
encryptionSettings,
diagnosticsContext,
encryptionDiagnosticsContext,
cancellationToken);
encryptionDiagnosticsContext.AddEncryptionDiagnosticsToResponseMessage(responseMessage);
return responseMessage;
}
/// <summary>
/// This method takes in an encrypted Stream payload.
/// The streamPayload is decrypted with the same policy which was used to encrypt and and then the original plain stream payload is
/// This method takes in an encrypted stream payload.
/// The streamPayload is decrypted with the same policy which was used to encrypt and then the original plain stream payload is
/// returned which can be used to re-encrypt after the latest encryption settings is retrieved.
/// The method also updates the cached Encryption Settings with the latest value if any.
/// </summary>
/// <param name="streamPayload"> Data encrypted with wrong encryption policy. </param>
/// <param name="encryptionSettings"> EncryptionSettings which was used to encrypt the payload. </param>
/// <param name="diagnosticsContext"> Diagnostics context. </param>
/// <param name="cancellationToken"> Cancellation token. </param>
/// <returns> Returns the decrypted stream payload. </returns>
/// <returns> Returns the decrypted stream payload and diagnostics content. </returns>
private async Task<Stream> DecryptStreamPayloadAndUpdateEncryptionSettingsAsync(
Stream streamPayload,
EncryptionSettings encryptionSettings,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken)
{
streamPayload = await EncryptionProcessor.DecryptAsync(
streamPayload,
encryptionSettings,
diagnosticsContext,
operationDiagnostics: null,
cancellationToken);
// get the latest encryption settings.
@ -1278,7 +1165,7 @@ namespace Microsoft.Azure.Cosmos.Encryption
foreach (JObject document in documents)
{
JObject decryptedDocument = await EncryptionProcessor.DecryptAsync(
(JObject decryptedDocument, _) = await EncryptionProcessor.DecryptAsync(
document,
encryptionSettings,
cancellationToken);
@ -1312,14 +1199,7 @@ namespace Microsoft.Azure.Cosmos.Encryption
// Clone(once) the request options since we modify it to set AddRequestHeaders to add additional headers.
if (!isRetry)
{
if (readManyRequestOptions != null)
{
clonedRequestOptions = (ReadManyRequestOptions)readManyRequestOptions.ShallowCopy();
}
else
{
clonedRequestOptions = new ReadManyRequestOptions();
}
clonedRequestOptions = readManyRequestOptions != null ? (ReadManyRequestOptions)readManyRequestOptions.ShallowCopy() : new ReadManyRequestOptions();
}
encryptionSettings.SetRequestHeaders(clonedRequestOptions);
@ -1347,11 +1227,15 @@ namespace Microsoft.Azure.Cosmos.Encryption
if (responseMessage.IsSuccessStatusCode && responseMessage.Content != null)
{
Stream decryptedContent = await this.DeserializeAndDecryptResponseAsync(
EncryptionDiagnosticsContext decryptDiagnostics = new EncryptionDiagnosticsContext();
Stream decryptedContent = await EncryptionProcessor.DeserializeAndDecryptResponseAsync(
responseMessage.Content,
encryptionSettings,
decryptDiagnostics,
cancellationToken);
decryptDiagnostics.AddEncryptionDiagnosticsToResponseMessage(responseMessage);
return new DecryptedResponseMessage(responseMessage, decryptedContent);
}

74
Microsoft.Azure.Cosmos.Encryption/src/EncryptionCosmosDiagnostics.cs Normal file
Просмотреть файл

@ -0,0 +1,74 @@
// ------------------------------------------------------------
// Copyright (c) Microsoft Corporation. All rights reserved.
// ------------------------------------------------------------
namespace Microsoft.Azure.Cosmos.Encryption
{
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
internal sealed class EncryptionCosmosDiagnostics : CosmosDiagnostics
{
private readonly CosmosDiagnostics coreDiagnostics;
private readonly JObject encryptContent;
private readonly JObject decryptContent;
public EncryptionCosmosDiagnostics(
CosmosDiagnostics coreDiagnostics,
JObject encryptContent,
JObject decryptContent)
{
this.coreDiagnostics = coreDiagnostics ?? throw new ArgumentNullException(nameof(coreDiagnostics));
if (encryptContent?.Count > 0)
{
this.encryptContent = encryptContent;
}
if (decryptContent?.Count > 0)
{
this.decryptContent = decryptContent;
}
}
public override IReadOnlyList<(string regionName, Uri uri)> GetContactedRegions()
{
return this.coreDiagnostics.GetContactedRegions();
}
public override string ToString()
{
StringBuilder stringBuilder = new StringBuilder();
StringWriter stringWriter = new StringWriter(stringBuilder);
using (JsonWriter writer = new JsonTextWriter(stringWriter))
{
writer.WriteStartObject();
writer.WritePropertyName(Constants.DiagnosticsCoreDiagnostics);
writer.WriteRawValue(this.coreDiagnostics.ToString());
writer.WritePropertyName(Constants.DiagnosticsEncryptionDiagnostics);
writer.WriteStartObject();
if (this.encryptContent != null)
{
writer.WritePropertyName(Constants.DiagnosticsEncryptOperation);
writer.WriteRawValue(this.encryptContent.ToString());
}
if (this.decryptContent != null)
{
writer.WritePropertyName(Constants.DiagnosticsDecryptOperation);
writer.WriteRawValue(this.decryptContent.ToString());
}
writer.WriteEndObject();
writer.WriteEndObject();
}
return stringWriter.ToString();
}
}
}

26
Microsoft.Azure.Cosmos.Encryption/src/EncryptionDatabaseExtensions.cs
Просмотреть файл

@ -61,16 +61,9 @@ namespace Microsoft.Azure.Cosmos.Encryption
throw new ArgumentNullException(nameof(encryptionKeyWrapMetadata));
}
EncryptionCosmosClient encryptionCosmosClient;
if (database is EncryptionDatabase encryptionDatabase)
{
encryptionCosmosClient = encryptionDatabase.EncryptionCosmosClient;
}
else
{
throw new ArgumentException("Creating a ClientEncryptionKey resource requires the use of an encryption - enabled client. Please refer to https://aka.ms/CosmosClientEncryption for more details. ");
}
EncryptionCosmosClient encryptionCosmosClient = database is EncryptionDatabase encryptionDatabase
? encryptionDatabase.EncryptionCosmosClient
: throw new ArgumentException("Creating a ClientEncryptionKey resource requires the use of an encryption - enabled client. Please refer to https://aka.ms/CosmosClientEncryption for more details. ");
EncryptionKeyStoreProvider encryptionKeyStoreProvider = encryptionCosmosClient.EncryptionKeyStoreProvider;
@ -148,16 +141,9 @@ namespace Microsoft.Azure.Cosmos.Encryption
ClientEncryptionKey clientEncryptionKey = database.GetClientEncryptionKey(clientEncryptionKeyId);
EncryptionCosmosClient encryptionCosmosClient;
if (database is EncryptionDatabase encryptionDatabase)
{
encryptionCosmosClient = encryptionDatabase.EncryptionCosmosClient;
}
else
{
throw new ArgumentException("Rewraping a ClientEncryptionKey requires the use of an encryption - enabled client. Please refer to https://aka.ms/CosmosClientEncryption for more details. ");
}
EncryptionCosmosClient encryptionCosmosClient = database is EncryptionDatabase encryptionDatabase
? encryptionDatabase.EncryptionCosmosClient
: throw new ArgumentException("Rewraping a ClientEncryptionKey requires the use of an encryption - enabled client. Please refer to https://aka.ms/CosmosClientEncryption for more details. ");
EncryptionKeyStoreProvider encryptionKeyStoreProvider = encryptionCosmosClient.EncryptionKeyStoreProvider;

85
Microsoft.Azure.Cosmos.Encryption/src/EncryptionDiagnosticsContext.cs Normal file
Просмотреть файл

@ -0,0 +1,85 @@
// ------------------------------------------------------------
// Copyright (c) Microsoft Corporation. All rights reserved.
// ------------------------------------------------------------
namespace Microsoft.Azure.Cosmos.Encryption
{
using System;
using System.Diagnostics;
using Newtonsoft.Json.Linq;
internal sealed class EncryptionDiagnosticsContext
{
private DateTime startTime;
private Stopwatch stopwatch;
private bool isDecryptionOperation;
public EncryptionDiagnosticsContext()
{
this.EncryptContent = new JObject();
this.DecryptContent = new JObject();
}
public JObject EncryptContent { get; }
public JObject DecryptContent { get; }
public void Begin(string operation)
{
this.stopwatch = Stopwatch.StartNew();
this.startTime = DateTime.UtcNow;
switch (operation)
{
case Constants.DiagnosticsEncryptOperation:
this.EncryptContent.Add(Constants.DiagnosticsStartTime, this.startTime);
this.isDecryptionOperation = false;
break;
case Constants.DiagnosticsDecryptOperation:
this.DecryptContent.Add(Constants.DiagnosticsStartTime, this.startTime);
this.isDecryptionOperation = true;
break;
default:
throw new NotSupportedException($"Operation: {operation} is not supported. " +
$"Should be either {Constants.DiagnosticsEncryptOperation} or {Constants.DiagnosticsDecryptOperation}.");
}
}
public void End(int? propertiesCount = null)
{
this.stopwatch.Stop();
if (this.isDecryptionOperation)
{
this.DecryptContent.Add(Constants.DiagnosticsDuration, this.stopwatch.ElapsedMilliseconds);
if (propertiesCount.HasValue)
{
this.DecryptContent.Add(Constants.DiagnosticsPropertiesDecryptedCount, propertiesCount);
}
}
else
{
this.EncryptContent.Add(Constants.DiagnosticsDuration, this.stopwatch.ElapsedMilliseconds);
if (propertiesCount.HasValue)
{
this.EncryptContent.Add(Constants.DiagnosticsPropertiesEncryptedCount, propertiesCount);
}
}
}
public void AddEncryptionDiagnosticsToResponseMessage(
ResponseMessage responseMessage)
{
EncryptionCosmosDiagnostics encryptionDiagnostics = new EncryptionCosmosDiagnostics(
responseMessage.Diagnostics,
this.EncryptContent,
this.DecryptContent);
responseMessage.Diagnostics = encryptionDiagnostics;
}
}
}

67
Microsoft.Azure.Cosmos.Encryption/src/EncryptionFeedIterator.cs
Просмотреть файл

@ -30,42 +30,43 @@ namespace Microsoft.Azure.Cosmos.Encryption
public override async Task<ResponseMessage> ReadNextAsync(CancellationToken cancellationToken = default)
{
CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(options: null);
using (diagnosticsContext.CreateScope("FeedIterator.ReadNext"))
EncryptionSettings encryptionSettings = await this.encryptionContainer.GetOrUpdateEncryptionSettingsFromCacheAsync(obsoleteEncryptionSettings: null, cancellationToken: cancellationToken);
encryptionSettings.SetRequestHeaders(this.requestOptions);
ResponseMessage responseMessage = await this.feedIterator.ReadNextAsync(cancellationToken);
// check for Bad Request and Wrong RID intended and update the cached RID and Client Encryption Policy.
if (responseMessage.StatusCode == HttpStatusCode.BadRequest
&& string.Equals(responseMessage.Headers.Get(Constants.SubStatusHeader), Constants.IncorrectContainerRidSubStatus))
{
EncryptionSettings encryptionSettings = await this.encryptionContainer.GetOrUpdateEncryptionSettingsFromCacheAsync(obsoleteEncryptionSettings: null, cancellationToken: cancellationToken);
encryptionSettings.SetRequestHeaders(this.requestOptions);
await this.encryptionContainer.GetOrUpdateEncryptionSettingsFromCacheAsync(
obsoleteEncryptionSettings: encryptionSettings,
cancellationToken: cancellationToken);
ResponseMessage responseMessage = await this.feedIterator.ReadNextAsync(cancellationToken);
// check for Bad Request and Wrong RID intended and update the cached RID and Client Encryption Policy.
if (responseMessage.StatusCode == HttpStatusCode.BadRequest
&& string.Equals(responseMessage.Headers.Get(Constants.SubStatusHeader), Constants.IncorrectContainerRidSubStatus))
{
await this.encryptionContainer.GetOrUpdateEncryptionSettingsFromCacheAsync(
obsoleteEncryptionSettings: encryptionSettings,
cancellationToken: cancellationToken);
throw new CosmosException(
"Operation has failed due to a possible mismatch in Client Encryption Policy configured on the container. Please refer to https://aka.ms/CosmosClientEncryption for more details. " + responseMessage.ErrorMessage,
responseMessage.StatusCode,
int.Parse(Constants.IncorrectContainerRidSubStatus),
responseMessage.Headers.ActivityId,
responseMessage.Headers.RequestCharge);
}
if (responseMessage.IsSuccessStatusCode && responseMessage.Content != null)
{
Stream decryptedContent = await this.encryptionContainer.DeserializeAndDecryptResponseAsync(
responseMessage.Content,
encryptionSettings,
cancellationToken);
return new DecryptedResponseMessage(responseMessage, decryptedContent);
}
return responseMessage;
throw new CosmosException(
"Operation has failed due to a possible mismatch in Client Encryption Policy configured on the container. Please refer to https://aka.ms/CosmosClientEncryption for more details. " + responseMessage.ErrorMessage,
responseMessage.StatusCode,
int.Parse(Constants.IncorrectContainerRidSubStatus),
responseMessage.Headers.ActivityId,
responseMessage.Headers.RequestCharge);
}
if (responseMessage.IsSuccessStatusCode && responseMessage.Content != null)
{
EncryptionDiagnosticsContext decryptDiagnostics = new EncryptionDiagnosticsContext();
Stream decryptedContent = await EncryptionProcessor.DeserializeAndDecryptResponseAsync(
responseMessage.Content,
encryptionSettings,
decryptDiagnostics,
cancellationToken);
decryptDiagnostics.AddEncryptionDiagnosticsToResponseMessage(responseMessage);
return new DecryptedResponseMessage(responseMessage, decryptedContent);
}
return responseMessage;
}
}
}

83
Microsoft.Azure.Cosmos.Encryption/src/EncryptionProcessor.cs
Просмотреть файл

@ -46,7 +46,7 @@ namespace Microsoft.Azure.Cosmos.Encryption
public static async Task<Stream> EncryptAsync(
Stream input,
EncryptionSettings encryptionSettings,
CosmosDiagnosticsContext diagnosticsContext,
EncryptionDiagnosticsContext operationDiagnostics,
CancellationToken cancellationToken)
{
if (input == null)
@ -54,7 +54,8 @@ namespace Microsoft.Azure.Cosmos.Encryption
throw new ArgumentNullException(nameof(input));
}
Debug.Assert(diagnosticsContext != null);
operationDiagnostics?.Begin(Constants.DiagnosticsEncryptOperation);
int propertiesEncryptedCount = 0;
JObject itemJObj = EncryptionProcessor.BaseSerializer.FromStream<JObject>(input);
@ -78,10 +79,15 @@ namespace Microsoft.Azure.Cosmos.Encryption
propertyToEncrypt.Value,
settingforProperty,
cancellationToken);
propertiesEncryptedCount++;
}
Stream result = EncryptionProcessor.BaseSerializer.ToStream(itemJObj);
input.Dispose();
return EncryptionProcessor.BaseSerializer.ToStream(itemJObj);
operationDiagnostics?.End(propertiesEncryptedCount);
return result;
}
/// <remarks>
@ -92,7 +98,7 @@ namespace Microsoft.Azure.Cosmos.Encryption
public static async Task<Stream> DecryptAsync(
Stream input,
EncryptionSettings encryptionSettings,
CosmosDiagnosticsContext diagnosticsContext,
EncryptionDiagnosticsContext operationDiagnostics,
CancellationToken cancellationToken)
{
if (input == null)
@ -101,34 +107,77 @@ namespace Microsoft.Azure.Cosmos.Encryption
}
Debug.Assert(input.CanSeek);
Debug.Assert(diagnosticsContext != null);
operationDiagnostics?.Begin(Constants.DiagnosticsDecryptOperation);
JObject itemJObj = RetrieveItem(input);
await DecryptObjectAsync(
int propertiesDecryptedCount = await DecryptObjectAsync(
itemJObj,
encryptionSettings,
diagnosticsContext,
cancellationToken);
Stream result = EncryptionProcessor.BaseSerializer.ToStream(itemJObj);
input.Dispose();
return EncryptionProcessor.BaseSerializer.ToStream(itemJObj);
operationDiagnostics?.End(propertiesDecryptedCount);
return result;
}
public static async Task<JObject> DecryptAsync(
public static async Task<(JObject, int)> DecryptAsync(
JObject document,
EncryptionSettings encryptionSettings,
CancellationToken cancellationToken)
{
Debug.Assert(document != null);
await DecryptObjectAsync(
int propertiesDecryptedCount = await DecryptObjectAsync(
document,
encryptionSettings,
CosmosDiagnosticsContext.Create(null),
cancellationToken);
return document;
return (document, propertiesDecryptedCount);
}
internal static async Task<Stream> DeserializeAndDecryptResponseAsync(
Stream content,
EncryptionSettings encryptionSettings,
EncryptionDiagnosticsContext operationDiagnostics,
CancellationToken cancellationToken)
{
if (!encryptionSettings.PropertiesToEncrypt.Any())
{
return content;
}
operationDiagnostics?.Begin(Constants.DiagnosticsDecryptOperation);
JObject contentJObj = EncryptionProcessor.BaseSerializer.FromStream<JObject>(content);
if (!(contentJObj.SelectToken(Constants.DocumentsResourcePropertyName) is JArray documents))
{
throw new InvalidOperationException("Feed Response body contract was violated. Feed response did not have an array of Documents. ");
}
int totalPropertiesDecryptedCount = 0;
foreach (JToken value in documents)
{
if (value is not JObject document)
{
continue;
}
(_, int propertiesDecrypted) = await EncryptionProcessor.DecryptAsync(
document,
encryptionSettings,
cancellationToken);
totalPropertiesDecryptedCount += propertiesDecrypted;
}
operationDiagnostics?.End(totalPropertiesDecryptedCount);
// the contents get decrypted in place by DecryptAsync.
return EncryptionProcessor.BaseSerializer.ToStream(contentJObj);
}
internal static async Task<Stream> EncryptValueStreamAsync(
@ -317,14 +366,12 @@ namespace Microsoft.Azure.Cosmos.Encryption
}
}
private static async Task DecryptObjectAsync(
private static async Task<int> DecryptObjectAsync(
JObject document,
EncryptionSettings encryptionSettings,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken)
{
Debug.Assert(diagnosticsContext != null);
int propertiesDecryptedCount = 0;
foreach (string propertyName in encryptionSettings.PropertiesToEncrypt)
{
JProperty propertyToDecrypt = document.Property(propertyName);
@ -341,10 +388,12 @@ namespace Microsoft.Azure.Cosmos.Encryption
propertyToDecrypt.Value,
settingsForProperty,
cancellationToken);
propertiesDecryptedCount++;
}
}
return;
return propertiesDecryptedCount;
}
private static JObject RetrieveItem(

201
Microsoft.Azure.Cosmos.Encryption/src/EncryptionTransactionalBatch.cs
Просмотреть файл

@ -44,25 +44,21 @@ namespace Microsoft.Azure.Cosmos.Encryption
Stream streamPayload,
TransactionalBatchItemRequestOptions requestOptions = null)
{
CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions);
using (diagnosticsContext.CreateScope("EncryptItemStream"))
EncryptionSettings encryptionSettings = this.encryptionContainer.GetOrUpdateEncryptionSettingsFromCacheAsync(obsoleteEncryptionSettings: null, cancellationToken: default)
.ConfigureAwait(false)
.GetAwaiter()
.GetResult();
if (encryptionSettings.PropertiesToEncrypt.Any())
{
EncryptionSettings encryptionSettings = this.encryptionContainer.GetOrUpdateEncryptionSettingsFromCacheAsync(obsoleteEncryptionSettings: null, cancellationToken: default)
streamPayload = EncryptionProcessor.EncryptAsync(
streamPayload,
encryptionSettings,
operationDiagnostics: null,
cancellationToken: default)
.ConfigureAwait(false)
.GetAwaiter()
.GetResult();
if (encryptionSettings.PropertiesToEncrypt.Any())
{
streamPayload = EncryptionProcessor.EncryptAsync(
streamPayload,
encryptionSettings,
diagnosticsContext,
cancellationToken: default)
.ConfigureAwait(false)
.GetAwaiter()
.GetResult();
}
}
this.transactionalBatch = this.transactionalBatch.CreateItemStream(
@ -111,25 +107,21 @@ namespace Microsoft.Azure.Cosmos.Encryption
Stream streamPayload,
TransactionalBatchItemRequestOptions requestOptions = null)
{
CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions);
using (diagnosticsContext.CreateScope("EncryptItemStream"))
EncryptionSettings encryptionSettings = this.encryptionContainer.GetOrUpdateEncryptionSettingsFromCacheAsync(obsoleteEncryptionSettings: null, cancellationToken: default)
.ConfigureAwait(false)
.GetAwaiter()
.GetResult();
if (encryptionSettings.PropertiesToEncrypt.Any())
{
EncryptionSettings encryptionSettings = this.encryptionContainer.GetOrUpdateEncryptionSettingsFromCacheAsync(obsoleteEncryptionSettings: null, cancellationToken: default)
streamPayload = EncryptionProcessor.EncryptAsync(
streamPayload,
encryptionSettings,
operationDiagnostics: null,
cancellationToken: default)
.ConfigureAwait(false)
.GetAwaiter()
.GetResult();
if (encryptionSettings.PropertiesToEncrypt.Any())
{
streamPayload = EncryptionProcessor.EncryptAsync(
streamPayload,
encryptionSettings,
diagnosticsContext,
default)
.ConfigureAwait(false)
.GetAwaiter()
.GetResult();
}
}
this.transactionalBatch = this.transactionalBatch.ReplaceItemStream(
@ -154,25 +146,21 @@ namespace Microsoft.Azure.Cosmos.Encryption
Stream streamPayload,
TransactionalBatchItemRequestOptions requestOptions = null)
{
CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions);
using (diagnosticsContext.CreateScope("EncryptItemStream"))
EncryptionSettings encryptionSettings = this.encryptionContainer.GetOrUpdateEncryptionSettingsFromCacheAsync(obsoleteEncryptionSettings: null, cancellationToken: default)
.ConfigureAwait(false)
.GetAwaiter()
.GetResult();
if (encryptionSettings.PropertiesToEncrypt.Any())
{
EncryptionSettings encryptionSettings = this.encryptionContainer.GetOrUpdateEncryptionSettingsFromCacheAsync(obsoleteEncryptionSettings: null, cancellationToken: default)
streamPayload = EncryptionProcessor.EncryptAsync(
streamPayload,
encryptionSettings,
operationDiagnostics: null,
cancellationToken: default)
.ConfigureAwait(false)
.GetAwaiter()
.GetResult();
if (encryptionSettings.PropertiesToEncrypt.Any())
{
streamPayload = EncryptionProcessor.EncryptAsync(
streamPayload,
encryptionSettings,
diagnosticsContext,
cancellationToken: default)
.ConfigureAwait(false)
.GetAwaiter()
.GetResult();
}
}
this.transactionalBatch = this.transactionalBatch.UpsertItemStream(
@ -192,53 +180,41 @@ namespace Microsoft.Azure.Cosmos.Encryption
TransactionalBatchRequestOptions requestOptions,
CancellationToken cancellationToken = default)
{
CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(options: null);
using (diagnosticsContext.CreateScope("TransactionalBatch.ExecuteAsync.WithRequestOptions"))
EncryptionSettings encryptionSettings = await this.encryptionContainer.GetOrUpdateEncryptionSettingsFromCacheAsync(obsoleteEncryptionSettings: null, cancellationToken: cancellationToken);
TransactionalBatchResponse response;
if (!encryptionSettings.PropertiesToEncrypt.Any())
{
TransactionalBatchResponse response = null;
EncryptionSettings encryptionSettings = await this.encryptionContainer.GetOrUpdateEncryptionSettingsFromCacheAsync(obsoleteEncryptionSettings: null, cancellationToken: cancellationToken);
if (!encryptionSettings.PropertiesToEncrypt.Any())
{
return await this.transactionalBatch.ExecuteAsync(requestOptions, cancellationToken);
}
else
{
TransactionalBatchRequestOptions clonedRequestOptions;
if (requestOptions != null)
{
clonedRequestOptions = (TransactionalBatchRequestOptions)requestOptions.ShallowCopy();
}
else
{
clonedRequestOptions = new TransactionalBatchRequestOptions();
}
encryptionSettings.SetRequestHeaders(clonedRequestOptions);
response = await this.transactionalBatch.ExecuteAsync(clonedRequestOptions, cancellationToken);
}
// FIXME this should check for BadRequest StatusCode too, requires a service fix to return 400 instead of -1 which is currently returned.
if (string.Equals(response.Headers.Get(Constants.SubStatusHeader), Constants.IncorrectContainerRidSubStatus))
{
await this.encryptionContainer.GetOrUpdateEncryptionSettingsFromCacheAsync(
obsoleteEncryptionSettings: encryptionSettings,
cancellationToken: cancellationToken);
throw new CosmosException(
"Operation has failed due to a possible mismatch in Client Encryption Policy configured on the container. Please refer to https://aka.ms/CosmosClientEncryption for more details. " + response.ErrorMessage,
HttpStatusCode.BadRequest,
int.Parse(Constants.IncorrectContainerRidSubStatus),
response.Headers.ActivityId,
response.Headers.RequestCharge);
}
return await this.DecryptTransactionalBatchResponseAsync(
response,
encryptionSettings,
diagnosticsContext,
cancellationToken);
return await this.transactionalBatch.ExecuteAsync(requestOptions, cancellationToken);
}
else
{
TransactionalBatchRequestOptions clonedRequestOptions = requestOptions != null
? (TransactionalBatchRequestOptions)requestOptions.ShallowCopy()
: new TransactionalBatchRequestOptions();
encryptionSettings.SetRequestHeaders(clonedRequestOptions);
response = await this.transactionalBatch.ExecuteAsync(clonedRequestOptions, cancellationToken);
}
// FIXME this should check for BadRequest StatusCode too, requires a service fix to return 400 instead of -1 which is currently returned.
if (string.Equals(response.Headers.Get(Constants.SubStatusHeader), Constants.IncorrectContainerRidSubStatus))
{
await this.encryptionContainer.GetOrUpdateEncryptionSettingsFromCacheAsync(
obsoleteEncryptionSettings: encryptionSettings,
cancellationToken: cancellationToken);
throw new CosmosException(
"Operation has failed due to a possible mismatch in Client Encryption Policy configured on the container. Please refer to https://aka.ms/CosmosClientEncryption for more details. " + response.ErrorMessage,
HttpStatusCode.BadRequest,
int.Parse(Constants.IncorrectContainerRidSubStatus),
response.Headers.ActivityId,
response.Headers.RequestCharge);
}
return await this.DecryptTransactionalBatchResponseAsync(
response,
encryptionSettings,
cancellationToken);
}
public override TransactionalBatch PatchItem(
@ -264,30 +240,27 @@ namespace Microsoft.Azure.Cosmos.Encryption
.GetAwaiter()
.GetResult();
CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions);
using (diagnosticsContext.CreateScope("PatchItem"))
{
List<PatchOperation> encryptedPatchOperations = this.encryptionContainer.EncryptPatchOperationsAsync(
patchOperations,
encryptionSettings,
cancellationToken: default)
.ConfigureAwait(false)
.GetAwaiter()
.GetResult();
EncryptionDiagnosticsContext encryptionDiagnosticsContext = new EncryptionDiagnosticsContext();
List<PatchOperation> encryptedPatchOperations = this.encryptionContainer.EncryptPatchOperationsAsync(
patchOperations,
encryptionSettings,
encryptionDiagnosticsContext,
cancellationToken: default)
.ConfigureAwait(false)
.GetAwaiter()
.GetResult();
this.transactionalBatch = this.transactionalBatch.PatchItem(
id,
encryptedPatchOperations,
requestOptions);
this.transactionalBatch = this.transactionalBatch.PatchItem(
id,
encryptedPatchOperations,
requestOptions);
return this;
}
return this;
}
private async Task<TransactionalBatchResponse> DecryptTransactionalBatchResponseAsync(
TransactionalBatchResponse response,
EncryptionSettings encryptionSettings,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken)
{
if (!encryptionSettings.PropertiesToEncrypt.Any())
@ -297,6 +270,9 @@ namespace Microsoft.Azure.Cosmos.Encryption
List<TransactionalBatchOperationResult> decryptedTransactionalBatchOperationResults = new List<TransactionalBatchOperationResult>();
EncryptionDiagnosticsContext encryptionDiagnosticsContext = new EncryptionDiagnosticsContext();
encryptionDiagnosticsContext.Begin(Constants.DiagnosticsDecryptOperation);
foreach (TransactionalBatchOperationResult result in response)
{
if (response.IsSuccessStatusCode && result.ResourceStream != null)
@ -304,7 +280,7 @@ namespace Microsoft.Azure.Cosmos.Encryption
Stream decryptedStream = await EncryptionProcessor.DecryptAsync(
result.ResourceStream,
encryptionSettings,
diagnosticsContext,
operationDiagnostics: null,
cancellationToken);
decryptedTransactionalBatchOperationResults.Add(new EncryptionTransactionalBatchOperationResult(result, decryptedStream));
@ -315,10 +291,17 @@ namespace Microsoft.Azure.Cosmos.Encryption
}
}
encryptionDiagnosticsContext.End();
EncryptionCosmosDiagnostics encryptionDiagnostics = new EncryptionCosmosDiagnostics(
response.Diagnostics,
encryptContent: null,
decryptContent: encryptionDiagnosticsContext.DecryptContent);
return new EncryptionTransactionalBatchResponse(
decryptedTransactionalBatchOperationResults,
response,
this.cosmosSerializer);
this.cosmosSerializer,
encryptionDiagnostics);
}
}
}

41
Microsoft.Azure.Cosmos.Encryption/src/EncryptionTransactionalBatchResponse.cs
Просмотреть файл

@ -13,18 +13,39 @@ namespace Microsoft.Azure.Cosmos.Encryption
private readonly IReadOnlyList<TransactionalBatchOperationResult> results;
private readonly TransactionalBatchResponse response;
private readonly CosmosSerializer cosmosSerializer;
private readonly CosmosDiagnostics diagnostics;
private bool isDisposed = false;
public EncryptionTransactionalBatchResponse(
IReadOnlyList<TransactionalBatchOperationResult> results,
TransactionalBatchResponse response,
CosmosSerializer cosmosSerializer)
CosmosSerializer cosmosSerializer,
CosmosDiagnostics diagnostics)
{
this.results = results;
this.response = response;
this.cosmosSerializer = cosmosSerializer;
this.diagnostics = diagnostics;
}
public override Headers Headers => this.response.Headers;
public override string ActivityId => this.response.ActivityId;
public override double RequestCharge => this.response.RequestCharge;
public override TimeSpan? RetryAfter => this.response.RetryAfter;
public override HttpStatusCode StatusCode => this.response.StatusCode;
public override string ErrorMessage => this.response.ErrorMessage;
public override bool IsSuccessStatusCode => this.response.IsSuccessStatusCode;
public override int Count => this.results?.Count ?? 0;
public override CosmosDiagnostics Diagnostics => this.diagnostics;
public override TransactionalBatchOperationResult this[int index] => this.results[index];
public override TransactionalBatchOperationResult<T> GetOperationResultAtIndex<T>(int index)
@ -45,24 +66,6 @@ namespace Microsoft.Azure.Cosmos.Encryption
return this.results.GetEnumerator();
}
public override Headers Headers => this.response.Headers;
public override string ActivityId => this.response.ActivityId;
public override double RequestCharge => this.response.RequestCharge;
public override TimeSpan? RetryAfter => this.response.RetryAfter;
public override HttpStatusCode StatusCode => this.response.StatusCode;
public override string ErrorMessage => this.response.ErrorMessage;
public override bool IsSuccessStatusCode => this.response.IsSuccessStatusCode;
public override int Count => this.results?.Count ?? 0;
public override CosmosDiagnostics Diagnostics => this.response.Diagnostics;
protected override void Dispose(bool disposing)
{
if (disposing && !this.isDisposed)

2
Microsoft.Azure.Cosmos.Encryption/src/Microsoft.Azure.Cosmos.Encryption.csproj
Просмотреть файл

@ -25,7 +25,7 @@
</ItemGroup>
<ItemGroup Condition=" '$(SdkProjectRef)' != 'True' ">
<PackageReference Include="Microsoft.Azure.Cosmos" Version="3.20.0-preview" />
<PackageReference Include="Microsoft.Azure.Cosmos" Version="3.21.0-preview" />
</ItemGroup>
<ItemGroup Condition=" '$(SdkProjectRef)' == 'True' ">

173
Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs
Просмотреть файл

@ -240,7 +240,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.EmulatorTests
Container encryptionContainerWithBulk = databaseWithBulk.GetContainer(MdeEncryptionTests.encryptionContainer.Id);
List<Task> tasks = new List<Task>()
{
MdeEncryptionTests.MdeCreateItemAsync(encryptionContainerWithBulk),
@ -551,7 +550,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.EmulatorTests
await MdeEncryptionTests.ValidateQueryResultsAsync(
MdeEncryptionTests.encryptionContainer,
"SELECT c.id, c.PK, c.NonSensitive, c.NonSensitiveInt FROM c",
expectedDoc);
expectedDoc,
expectedPropertiesDecryptedCount: 0);
}
[TestMethod]
@ -640,6 +640,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.EmulatorTests
FeedResponse<TestDoc> readDocs = await queryResponseIterator.ReadNextAsync();
Assert.AreNotEqual(0, readDocs.Count);
VerifyDiagnostics(readDocs.Diagnostics, encryptOperation: false, expectedPropertiesDecryptedCount: 2);
}
[TestMethod]
@ -695,6 +696,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.EmulatorTests
.ExecuteAsync();
Assert.AreEqual(HttpStatusCode.OK, batchResponse.StatusCode);
VerifyDiagnostics(batchResponse.Diagnostics, encryptOperation: false, expectedPropertiesDecryptedCount: 0);
TransactionalBatchOperationResult<TestDoc> doc1 = batchResponse.GetOperationResultAtIndex<TestDoc>(0);
VerifyExpectedDocResponse(doc1ToCreate, doc1.Resource);
@ -759,7 +761,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.EmulatorTests
public async Task EncryptionReadManyItemAsync()
{
TestDoc testDoc = await MdeEncryptionTests.MdeCreateItemAsync(MdeEncryptionTests.encryptionContainer);
TestDoc testDoc2 = await MdeEncryptionTests.MdeCreateItemAsync(MdeEncryptionTests.encryptionContainer);
List<(string, PartitionKey)> itemList = new List<(string, PartitionKey)>
@ -769,6 +770,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.EmulatorTests
};
FeedResponse<TestDoc> response = await encryptionContainer.ReadManyItemsAsync<TestDoc>(itemList);
VerifyDiagnostics(response.Diagnostics, encryptOperation: false, expectedPropertiesDecryptedCount: 24);
Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
Assert.AreEqual(2, response.Count);
@ -776,7 +778,11 @@ namespace Microsoft.Azure.Cosmos.Encryption.EmulatorTests
VerifyExpectedDocResponse(testDoc2, response.Resource.ElementAt(1));
// stream test.
TestDoc testDoc3 = await MdeEncryptionTests.MdeCreateItemAsync(MdeEncryptionTests.encryptionContainer);
itemList.Add((testDoc3.Id, new PartitionKey(testDoc3.PK)));
ResponseMessage responseStream = await encryptionContainer.ReadManyItemsStreamAsync(itemList);
VerifyDiagnostics(responseStream.Diagnostics, encryptOperation: false, expectedPropertiesDecryptedCount: 36);
Assert.IsTrue(responseStream.IsSuccessStatusCode);
Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
@ -786,6 +792,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.EmulatorTests
{
VerifyExpectedDocResponse(testDoc, documents.ElementAt(0).ToObject<TestDoc>());
VerifyExpectedDocResponse(testDoc2, documents.ElementAt(1).ToObject<TestDoc>());
VerifyExpectedDocResponse(testDoc3, documents.ElementAt(2).ToObject<TestDoc>());
}
else
{
@ -825,14 +832,14 @@ namespace Microsoft.Azure.Cosmos.Encryption.EmulatorTests
TestDoc testDoc2 = await MdeEncryptionTests.MdeCreateItemAsync(MdeEncryptionTests.encryptionContainer);
// test GetItemLinqQueryable
await MdeEncryptionTests.ValidateQueryResultsMultipleDocumentsAsync(MdeEncryptionTests.encryptionContainer, testDoc1, testDoc2, null);
await MdeEncryptionTests.ValidateQueryResultsMultipleDocumentsAsync(MdeEncryptionTests.encryptionContainer, testDoc1, testDoc2, null, expectedPropertiesDecryptedCount: 24);
string query = $"SELECT * FROM c WHERE c.PK in ('{testDoc1.PK}', '{testDoc2.PK}')";
await MdeEncryptionTests.ValidateQueryResultsMultipleDocumentsAsync(MdeEncryptionTests.encryptionContainer, testDoc1, testDoc2, query);
await MdeEncryptionTests.ValidateQueryResultsMultipleDocumentsAsync(MdeEncryptionTests.encryptionContainer, testDoc1, testDoc2, query, expectedPropertiesDecryptedCount: 24);
// ORDER BY query
query += " ORDER BY c._ts";
await MdeEncryptionTests.ValidateQueryResultsMultipleDocumentsAsync(MdeEncryptionTests.encryptionContainer, testDoc1, testDoc2, query);
await MdeEncryptionTests.ValidateQueryResultsMultipleDocumentsAsync(MdeEncryptionTests.encryptionContainer, testDoc1, testDoc2, query, expectedPropertiesDecryptedCount: 24);
}
[TestMethod]
@ -875,19 +882,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.EmulatorTests
Assert.IsTrue(response.IsSuccessStatusCode);
Assert.IsNull(response.ErrorMessage);
}
}
[TestMethod]
public async Task EncryptionHandleDecryptionFailure()
{
TestDoc testDoc1 = await MdeEncryptionTests.MdeCreateItemAsync(MdeEncryptionTests.encryptionContainer);
TestDoc testDoc2 = await MdeEncryptionTests.MdeCreateItemAsync(MdeEncryptionTests.encryptionContainer);
string query = $"SELECT * FROM c WHERE c.PK in ('{testDoc1.PK}', '{testDoc2.PK}')";
// success
await MdeEncryptionTests.ValidateQueryResultsMultipleDocumentsAsync(MdeEncryptionTests.encryptionContainer, testDoc1, testDoc2, query);
}
[TestMethod]
@ -1289,9 +1283,10 @@ namespace Microsoft.Azure.Cosmos.Encryption.EmulatorTests
// previous failure would have updated the policy in the cache.
await MdeEncryptionTests.ValidateQueryResultsAsync(
otherEncryptionContainer,
"SELECT * FROM c",
testDoc);
otherEncryptionContainer,
"SELECT * FROM c",
testDoc,
expectedPropertiesDecryptedCount: 2);
}
[TestMethod]
@ -1570,42 +1565,43 @@ namespace Microsoft.Azure.Cosmos.Encryption.EmulatorTests
[TestMethod]
public async Task EncryptionCreateItemWithNoClientEncryptionPolicy()
{
// a database can have both Containers with Policies Configured and with no Encryption Policy
await MdeEncryptionTests.MdeCreateItemAsync(MdeEncryptionTests.encryptionContainer);
// a database can have both types of Containers - with and without ClientEncryptionPolicy configured
ContainerProperties containerProperties = new ContainerProperties(Guid.NewGuid().ToString(), "/PK");
Container encryptionContainer = await database.CreateContainerAsync(containerProperties, 400);
await encryptionContainer.InitializeEncryptionAsync();
Container encryptionContainerWithNoPolicy = await database.CreateContainerAsync(containerProperties, 400);
await encryptionContainerWithNoPolicy.InitializeEncryptionAsync();
TestDoc testDoc = TestDoc.Create();
ItemResponse<TestDoc> createResponse = await encryptionContainer.CreateItemAsync(
ItemResponse<TestDoc> createResponse = await encryptionContainerWithNoPolicy.CreateItemAsync(
testDoc,
new PartitionKey(testDoc.PK));
Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode);
VerifyExpectedDocResponse(testDoc, createResponse.Resource);
QueryDefinition withEncryptedParameter = encryptionContainer.CreateQueryDefinition(
"SELECT * FROM c where c.Sensitive_StringFormat = @Sensitive_StringFormat AND c.Sensitive_IntFormat = @Sensitive_IntFormat");
QueryDefinition withEncryptedParameter = encryptionContainerWithNoPolicy.CreateQueryDefinition(
"SELECT * FROM c where c.Sensitive_StringFormat = @Sensitive_StringFormat AND c.Sensitive_IntFormat = @Sensitive_IntFormat");
await withEncryptedParameter.AddParameterAsync(
"@Sensitive_StringFormat",
testDoc.Sensitive_StringFormat,
"/Sensitive_StringFormat");
"@Sensitive_StringFormat",
testDoc.Sensitive_StringFormat,
"/Sensitive_StringFormat");
await withEncryptedParameter.AddParameterAsync(
"@Sensitive_IntFormat",
testDoc.Sensitive_IntFormat,
"/Sensitive_IntFormat");
"@Sensitive_IntFormat",
testDoc.Sensitive_IntFormat,
"/Sensitive_IntFormat");
TestDoc expectedDoc = new TestDoc(testDoc);
await MdeEncryptionTests.ValidateQueryResultsAsync(
encryptionContainer,
encryptionContainerWithNoPolicy,
queryDefinition: withEncryptedParameter,
expectedDoc: expectedDoc);
expectedDoc: expectedDoc,
decryptOperation: false);
await encryptionContainer.DeleteContainerAsync();
await encryptionContainerWithNoPolicy.DeleteContainerAsync();
}
[TestMethod]
@ -1639,7 +1635,38 @@ namespace Microsoft.Azure.Cosmos.Encryption.EmulatorTests
await MdeEncryptionTests.MdeCreateItemAsync(MdeEncryptionTests.encryptionContainer);
// test GetItemLinqQueryable with ToEncryptionStreamIterator extension
await MdeEncryptionTests.ValidateQueryResponseAsync(MdeEncryptionTests.encryptionContainer);
await MdeEncryptionTests.ValidateQueryResponseAsync(MdeEncryptionTests.encryptionContainer, expectedPropertiesDecryptedCount: 24);
}
[TestMethod]
public async Task EncryptionDiagnosticsTest()
{
ItemResponse<TestDoc> createResponse = await MdeEncryptionTests.MdeCreateItemAsync(MdeEncryptionTests.encryptionContainer);
VerifyDiagnostics(createResponse.Diagnostics);
TestDoc testDoc = createResponse.Resource;
ResponseMessage readResponse = await MdeEncryptionTests.encryptionContainer.ReadItemStreamAsync(testDoc.Id, new PartitionKey(testDoc.PK));
VerifyDiagnostics(readResponse.Diagnostics, encryptOperation: false, decryptOperation: true);
TestDoc testDoc1 = TestDoc.Create();
testDoc1.NonSensitive = Guid.NewGuid().ToString();
testDoc1.Sensitive_StringFormat = Guid.NewGuid().ToString();
ItemResponse<TestDoc> upsertResponse = await MdeEncryptionTests.MdeUpsertItemAsync(
MdeEncryptionTests.encryptionContainer,
testDoc1,
HttpStatusCode.Created);
TestDoc upsertedDoc = upsertResponse.Resource;
VerifyDiagnostics(upsertResponse.Diagnostics);
upsertedDoc.NonSensitive = Guid.NewGuid().ToString();
upsertedDoc.Sensitive_StringFormat = Guid.NewGuid().ToString();
ItemResponse<TestDoc> replaceResponse = await MdeEncryptionTests.MdeReplaceItemAsync(
MdeEncryptionTests.encryptionContainer,
upsertedDoc,
upsertResponse.ETag);
VerifyDiagnostics(replaceResponse.Diagnostics);
}
[TestMethod]
@ -1706,12 +1733,14 @@ namespace Microsoft.Azure.Cosmos.Encryption.EmulatorTests
PatchOperation.Replace("/Sensitive_FloatFormat", docPostPatching.Sensitive_FloatFormat),
};
await MdeEncryptionTests.MdePatchItemAsync(
ItemResponse<TestDoc> patchResponse = await MdeEncryptionTests.MdePatchItemAsync(
MdeEncryptionTests.encryptionContainer,
patchOperations,
docPostPatching,
HttpStatusCode.OK);
VerifyDiagnostics(patchResponse.Diagnostics, expectedPropertiesEncryptedCount: 8);
docPostPatching.Sensitive_ArrayFormat = new TestDoc.Sensitive_ArrayData[]
{
new TestDoc.Sensitive_ArrayData
@ -1772,12 +1801,14 @@ namespace Microsoft.Azure.Cosmos.Encryption.EmulatorTests
patchOperations.Add(PatchOperation.Remove("/Sensitive_NestedObjectFormatL1/Sensitive_NestedObjectFormatL2"));
patchOperations.Add(PatchOperation.Set("/Sensitive_NestedObjectFormatL1/Sensitive_ArrayFormatL1/0", docPostPatching.Sensitive_NestedObjectFormatL1.Sensitive_ArrayFormatL1[0]));
await MdeEncryptionTests.MdePatchItemAsync(
patchResponse = await MdeEncryptionTests.MdePatchItemAsync(
MdeEncryptionTests.encryptionContainer,
patchOperations,
docPostPatching,
HttpStatusCode.OK);
VerifyDiagnostics(patchResponse.Diagnostics, expectedPropertiesEncryptedCount: 3);
patchOperations.Add(PatchOperation.Increment("/Sensitive_IntFormat", 1));
try
{
@ -1898,7 +1929,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.EmulatorTests
TestDoc testDoc1,
TestDoc testDoc2,
string query,
bool compareEncryptedProperty = true)
bool compareEncryptedProperty = true,
int expectedPropertiesDecryptedCount = 0)
{
FeedIterator<TestDoc> queryResponseIterator;
@ -1915,7 +1947,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.EmulatorTests
FeedResponse<TestDoc> readDocs = await queryResponseIterator.ReadNextAsync();
Assert.AreEqual(null, readDocs.ContinuationToken);
if (query == null)
{
Assert.IsTrue(readDocs.Count >= 2);
@ -1950,10 +1981,14 @@ namespace Microsoft.Azure.Cosmos.Encryption.EmulatorTests
}
}
}
VerifyDiagnostics(readDocs.Diagnostics, encryptOperation: false, expectedPropertiesDecryptedCount: expectedPropertiesDecryptedCount);
}
private static async Task ValidateQueryResponseAsync(Container container,
string query = null)
private static async Task ValidateQueryResponseAsync(
Container container,
string query = null,
int expectedPropertiesDecryptedCount = 0)
{
FeedIterator feedIterator;
if (query == null)
@ -1971,6 +2006,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.EmulatorTests
ResponseMessage response = await feedIterator.ReadNextAsync();
Assert.IsTrue(response.IsSuccessStatusCode);
Assert.IsNull(response.ErrorMessage);
VerifyDiagnostics(response.Diagnostics, encryptOperation: false, expectedPropertiesDecryptedCount: expectedPropertiesDecryptedCount);
}
}
@ -1995,6 +2031,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.EmulatorTests
VerifyExpectedDocResponse(testDoc1, testDocs.Resource.ElementAt(0));
VerifyExpectedDocResponse(testDoc2, testDocs.Resource.ElementAt(1));
VerifyDiagnostics(testDocs.Diagnostics, encryptOperation: false, expectedPropertiesDecryptedCount: 24);
}
}
@ -2258,7 +2296,9 @@ namespace Microsoft.Azure.Cosmos.Encryption.EmulatorTests
Container container,
string query = null,
TestDoc expectedDoc = null,
QueryDefinition queryDefinition = null)
QueryDefinition queryDefinition = null,
bool decryptOperation = true,
int expectedPropertiesDecryptedCount = 12)
{
QueryRequestOptions requestOptions = expectedDoc != null
? new QueryRequestOptions()
@ -2277,7 +2317,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.EmulatorTests
{
Assert.AreEqual(1, readDocs.Count);
TestDoc readDoc = readDocs.Single();
VerifyExpectedDocResponse(expectedDoc, readDoc);
VerifyExpectedDocResponse(expectedDoc, readDoc);
VerifyDiagnostics(readDocs.Diagnostics, encryptOperation: false, decryptOperation: decryptOperation, expectedPropertiesDecryptedCount: expectedPropertiesDecryptedCount);
}
else
{
@ -2524,6 +2565,46 @@ namespace Microsoft.Azure.Cosmos.Encryption.EmulatorTests
Assert.AreEqual(expectedDoc.NonSensitiveInt, verifyDoc.NonSensitiveInt);
}
private static void VerifyDiagnostics(
CosmosDiagnostics diagnostics,
bool encryptOperation = true,
bool decryptOperation = true,
int expectedPropertiesEncryptedCount = 12,
int expectedPropertiesDecryptedCount = 12)
{
Assert.IsNotNull(diagnostics);
JObject diagnosticsObject = JObject.Parse(diagnostics.ToString());
JObject coreDiagnostics = diagnosticsObject.Value<JObject>(Constants.DiagnosticsCoreDiagnostics);
Assert.IsNotNull(coreDiagnostics);
JObject encryptionDiagnostics = diagnosticsObject.Value<JObject>(Constants.DiagnosticsEncryptionDiagnostics);
Assert.IsNotNull(encryptionDiagnostics);
if (encryptOperation)
{
JObject encryptOperationDiagnostics = encryptionDiagnostics.Value<JObject>(Constants.DiagnosticsEncryptOperation);
Assert.IsNotNull(encryptOperationDiagnostics);
Assert.IsNotNull(encryptOperationDiagnostics.GetValue(Constants.DiagnosticsStartTime));
Assert.IsNotNull(encryptOperationDiagnostics.GetValue(Constants.DiagnosticsDuration));
int propertiesEncrypted = encryptOperationDiagnostics.Value<int>(Constants.DiagnosticsPropertiesEncryptedCount);
Assert.AreEqual(expectedPropertiesEncryptedCount, propertiesEncrypted);
}
if (decryptOperation)
{
JObject decryptOperationDiagnostics = encryptionDiagnostics.Value<JObject>(Constants.DiagnosticsDecryptOperation);
Assert.IsNotNull(decryptOperationDiagnostics);
Assert.IsNotNull(decryptOperationDiagnostics.GetValue(Constants.DiagnosticsStartTime));
Assert.IsNotNull(decryptOperationDiagnostics.GetValue(Constants.DiagnosticsDuration));
if (expectedPropertiesDecryptedCount > 0)
{
int propertiesDecrypted = decryptOperationDiagnostics.Value<int>(Constants.DiagnosticsPropertiesDecryptedCount);
Assert.IsTrue(propertiesDecrypted >= expectedPropertiesDecryptedCount);
}
}
}
public class TestDoc
{