microsoft
/
CDM
зеркало из https://github.com/microsoft/CDM.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Microsoft CDM Release Version Update:

This commit is contained in:
cdm-publisher 2024-02-23 20:00:32 +00:00
Родитель a62fae5061
Коммит bb7d676c1b
782 изменённых файлов: 418514 добавлений и 4912 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

4
objectModel/CSharp/Directory.Build.props
Просмотреть файл

@ -1,9 +1,9 @@
<Project>
<PropertyGroup>
<CdmObjectModelVersion>1.7.5</CdmObjectModelVersion>
<CdmObjectModelVersion>1.7.6</CdmObjectModelVersion>
<CdmStandardsVersion>2.8.0</CdmStandardsVersion>
<CdmPQModelsVersion>2.8.0</CdmPQModelsVersion>
<CdmTelemetryVersion>1.7.5-preview1</CdmTelemetryVersion>
<CdmTelemetryVersion>1.7.6-preview1</CdmTelemetryVersion>
<RunSettingsFilePath>$(MSBuildThisFileDirectory)\CodeCoverage.runsettings</RunSettingsFilePath>
</PropertyGroup>
</Project>

70
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Adapter.Adls/Storage/ADLSAdapter.cs
Просмотреть файл

@ -166,7 +166,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Storage
/// <summary>
/// A cache for storing last modified times of file paths.
/// </summary>
private Dictionary<string, DateTimeOffset> fileModifiedTimeCache = new Dictionary<string, DateTimeOffset>();
private Dictionary<string, CdmFileMetadata> fileMetadataCache = new Dictionary<string, CdmFileMetadata>();
/// <summary>
/// The Scopes.
@ -388,37 +388,51 @@ namespace Microsoft.CommonDataModel.ObjectModel.Storage
public override void ClearCache()
{
this.fileModifiedTimeCache.Clear();
this.fileMetadataCache.Clear();
}
/// <inheritdoc />
public override async Task<DateTimeOffset?> ComputeLastModifiedTimeAsync(string corpusPath)
{
if (this.IsCacheEnabled && fileModifiedTimeCache.TryGetValue(corpusPath, out DateTimeOffset time))
var fileMetadata = await this.FetchFileMetadataAsync(corpusPath);
if (fileMetadata == null)
{
return time;
}
else
{
var url = this.CreateFormattedAdapterPath(corpusPath);
var httpRequest = await this.BuildRequest(url, HttpMethod.Head);
using (var cdmResponse = await base.ExecuteRequest(httpRequest))
{
if (cdmResponse.StatusCode.Equals(HttpStatusCode.OK))
{
var lastTime = cdmResponse.Content.Headers.LastModified;
if (this.IsCacheEnabled && lastTime.HasValue)
{
this.fileModifiedTimeCache[corpusPath] = lastTime.Value;
}
return lastTime;
}
}
return null;
}
return fileMetadata.LastModifiedTime;
}
/// <inheritdoc />
public override async Task<CdmFileMetadata> FetchFileMetadataAsync(string corpusPath)
{
if (this.IsCacheEnabled && fileMetadataCache.TryGetValue(corpusPath, out CdmFileMetadata metadata))
{
return metadata;
}
var url = this.CreateFormattedAdapterPath(corpusPath);
var httpRequest = await this.BuildRequest(url, HttpMethod.Head);
using (var cdmResponse = await base.ExecuteRequest(httpRequest))
{
if (cdmResponse.StatusCode.Equals(HttpStatusCode.OK))
{
var lastTime = cdmResponse.Content.Headers.LastModified;
var fileSize = cdmResponse.Content.Headers.ContentLength;
var fileMetadata = new CdmFileMetadata { FileSizeBytes = fileSize.Value, LastModifiedTime = lastTime.Value };
if (this.IsCacheEnabled && lastTime.HasValue)
{
this.fileMetadataCache[corpusPath] = fileMetadata;
}
return fileMetadata;
}
}
return null;
}
/// <inheritdoc />
@ -541,11 +555,13 @@ namespace Microsoft.CommonDataModel.ObjectModel.Storage
if (!isDirectory)
{
result.Add(path, new CdmFileMetadata { FileSizeBytes = contentLength });
DateTimeOffset.TryParse(lastModified, out DateTimeOffset offset);
var fileMetadata = new CdmFileMetadata { FileSizeBytes = contentLength, LastModifiedTime = offset };
result.Add(path, fileMetadata);
if (this.IsCacheEnabled && DateTimeOffset.TryParse(lastModified, out DateTimeOffset offset))
if (this.IsCacheEnabled)
{
fileModifiedTimeCache[path] = offset;
fileMetadataCache[path] = fileMetadata;
}
}
}

30
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Tests/Cdm/CorpusTests.cs
Просмотреть файл

@ -1,4 +1,4 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.
namespace Microsoft.CommonDataModel.ObjectModel.Tests.Cdm
@ -235,5 +235,33 @@ namespace Microsoft.CommonDataModel.ObjectModel.Tests.Cdm
await corpus.CalculateEntityGraphAsync(manifest);
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrInvalidCast, true);
}
/// <summary>
/// Test warning correctly logged when max depth is exceeded for Resolution Guidance
/// </summary>
[TestMethod]
public async Task TestMaxDepthExceededResolutionGuidance()
{
var expectedLogCodes = new HashSet<CdmLogCode> { CdmLogCode.WarnMaxDepthExceeded };
var corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestMaxDepthExceededResolutionGuidance", expectedCodes: expectedLogCodes);
var entity = await corpus.FetchObjectAsync<CdmEntityDefinition>("local:/firstEntity.cdm.json/firstEntity");
await entity.CreateResolvedEntityAsync("resFirstEntity");
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.WarnMaxDepthExceeded, true);
}
/// <summary>
/// Test warning correctly logged when max depth is exceeded for Projections
/// </summary>
[TestMethod]
public async Task TestMaxDepthExceededProjections()
{
var expectedLogCodes = new HashSet<CdmLogCode> { CdmLogCode.WarnMaxDepthExceeded };
var corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestMaxDepthExceededProjections", expectedCodes: expectedLogCodes);
var entity = await corpus.FetchObjectAsync<CdmEntityDefinition>("local:/A.cdm.json/A");
await entity.CreateResolvedEntityAsync("resA");
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.WarnMaxDepthExceeded, true);
}
}
}

43
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Tests/Cdm/DataPartition/DataPartitionTest.cs Normal file
Просмотреть файл

@ -0,0 +1,43 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.
namespace Microsoft.CommonDataModel.ObjectModel.Tests.Cdm.DataPartition
{
using Microsoft.CommonDataModel.ObjectModel.Cdm;
using Microsoft.CommonDataModel.ObjectModel.Utilities;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System.IO;
using System.Threading.Tasks;
[TestClass]
public class DataPartitionTest
{
/// <summary>
/// The path between TestDataPath and TestName.
/// </summary>
private string testsSubpath = Path.Combine("Cdm", "DataPartition");
/// <summary>
/// Tests refreshing data partition gets file size
/// </summary>
[TestMethod]
public async Task TestRefreshesDataPartition()
{
var cdmCorpus = TestHelper.GetLocalCorpus(testsSubpath, "TestRefreshesDataPartition");
var cdmManifest = await cdmCorpus.FetchObjectAsync<CdmManifestDefinition>("local:/partitions.manifest.cdm.json");
var fileStatusCheckOptions = new FileStatusCheckOptions { IncludeDataPartitionSize = true };
var partitionEntity = cdmManifest.Entities[0];
Assert.AreEqual(partitionEntity.DataPartitions.Count, 1);
var partition = partitionEntity.DataPartitions[0];
await cdmManifest.FileStatusCheckAsync(fileStatusCheckOptions: fileStatusCheckOptions);
var localTraitIndex = partition.ExhibitsTraits.IndexOf("is.partition.size");
Assert.AreNotEqual(localTraitIndex, -1);
var localTrait = partition.ExhibitsTraits[localTraitIndex] as CdmTraitReference;
Assert.AreEqual(localTrait.NamedReference, "is.partition.size");
Assert.AreEqual(localTrait.Arguments[0].Value, 2);
}
}
}

64
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Tests/Cdm/DataPartitionPattern/DataPartitionPatternTest.cs
Просмотреть файл

@ -10,6 +10,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Tests.Cdm.DataPartitionPattern
using Microsoft.CommonDataModel.ObjectModel.Storage;
using Microsoft.CommonDataModel.ObjectModel.Tests.Storage.TestAdapters;
using Microsoft.CommonDataModel.ObjectModel.Utilities;
using Microsoft.CommonDataModel.ObjectModel.Utilities.Exceptions;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Newtonsoft.Json;
using System;
@ -830,7 +831,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Tests.Cdm.DataPartitionPattern
Assert.AreEqual(statusLevel, CdmStatusLevel.Error, "Error level message should have been reported");
Assert.IsTrue(
message == "StorageManager | The object path cannot be null or empty. | CreateAbsoluteCorpusPath" ||
message == "CdmCorpusDefinition | The object path cannot be null or empty. | GetLastModifiedTimeFromPartitionPathAsync",
message == "CdmCorpusDefinition | The object path cannot be null or empty. | GetFileMetadataFromPartitionPathAsync",
"Unexpected error message received");
}
}, CdmStatusLevel.Warning);
@ -911,7 +912,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Tests.Cdm.DataPartitionPattern
var overrideFetchDataPartitionList = overrideFetchAllManifest.Entities[0].DataPartitions;
Assert.AreEqual(overrideFetchDataPartitionList.Count, 1);
var overrideFetchTraitIndex = overrideFetchDataPartitionList[0].ExhibitsTraits.IndexOf("is.partition.size");
Assert.AreEqual(overrideFetchTraitIndex , -1);
Assert.AreEqual(overrideFetchTraitIndex, -1);
Assert.AreEqual(overrideFetchDataPartitionList[0].ExhibitsTraits.Count, 1);
// check that error is correctly logged when FetchAllFilesMetadata is misconfigured and returns null
@ -933,5 +934,64 @@ namespace Microsoft.CommonDataModel.ObjectModel.Tests.Cdm.DataPartitionPattern
var outOfRangeManifest = await outOfRangeCorpus.FetchObjectAsync<CdmManifestDefinition>("manifest.manifest.cdm.json");
await outOfRangeManifest.FileStatusCheckAsync(fileStatusCheckOptions: fileStatusCheckOptions);
}
/// <summary>
/// Test that error is thrown when FileStatusCheckOption is set
/// </summary>
[TestMethod]
public async Task TestThrowOnPartitionError()
{
var expectedLogCodes = new HashSet<CdmLogCode> { CdmLogCode.WarnPartitionFileFetchFailed };
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, nameof(TestFetchAllFilesMetadata), expectedCodes: expectedLogCodes);
var testLocalAdapter = (LocalAdapter)corpus.Storage.NamespaceAdapters[corpus.Storage.DefaultNamespace];
corpus.Storage.Mount("error", new FetchAllMetadataThrowErrorAdapter(testLocalAdapter));
var fileStatusCheckOptions = new FileStatusCheckOptions() { ThrowOnPartitionError = true };
var manifestThrowsError = false;
var entityDecThrowsError = false;
var partitionPatternThrowsError = false;
var manifest = await corpus.FetchObjectAsync<CdmManifestDefinition>("error:/manifest.manifest.cdm.json");
try
{
await manifest.FileStatusCheckAsync(fileStatusCheckOptions: fileStatusCheckOptions);
}
catch (CdmReadPartitionFromPatternException e)
{
Assert.IsNotNull(e.InnerException);
Assert.AreEqual("Exception of type 'System.Exception' was thrown.", e.InnerException.Message);
manifestThrowsError = true;
}
var entityDec = (CdmLocalEntityDeclarationDefinition)manifest.Entities[0];
try
{
await entityDec.FileStatusCheckAsync(fileStatusCheckOptions: fileStatusCheckOptions);
}
catch (CdmReadPartitionFromPatternException e)
{
Assert.IsNotNull(e.InnerException);
Assert.AreEqual("Exception of type 'System.Exception' was thrown.", e.InnerException.Message);
entityDecThrowsError = true;
}
var partitionPattern = ((CdmLocalEntityDeclarationDefinition)manifest.Entities[0]).DataPartitionPatterns[0];
try
{
await partitionPattern.FileStatusCheckAsync(fileStatusCheckOptions: fileStatusCheckOptions);
}
catch (CdmReadPartitionFromPatternException e)
{
Assert.IsNotNull(e.InnerException);
Assert.AreEqual("Exception of type 'System.Exception' was thrown.", e.InnerException.Message);
partitionPatternThrowsError = true;
}
Assert.IsTrue(manifestThrowsError && entityDecThrowsError && partitionPatternThrowsError);
}
}
}

5
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Tests/Cdm/Projection/ProjectionMiscellaneousTest.cs
Просмотреть файл

@ -200,7 +200,8 @@ namespace Microsoft.CommonDataModel.ObjectModel.Tests.Cdm.Projection
string testName = "TestMaxDepthOnPolymorphicEntity";
string entityName = "A";
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, testName);
var expectedLogCodes = new HashSet<CdmLogCode> { CdmLogCode.WarnMaxDepthExceeded };
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, testName, expectedCodes: expectedLogCodes);
CdmEntityDefinition entity = await corpus.FetchObjectAsync<CdmEntityDefinition>($"{entityName}.cdm.json/{entityName}");
@ -211,7 +212,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Tests.Cdm.Projection
CdmEntityDefinition resEntity = await entity.CreateResolvedEntityAsync($"resolved-{entityName}", resOpt);
Assert.IsNotNull(resEntity);
Assert.AreEqual(4, resEntity.Attributes.Count);
Assert.AreEqual(2, resEntity.Attributes.Count);
}
/// <summary>

7
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Tests/Cdm/Resolution/CacheTest.cs
Просмотреть файл

@ -2,6 +2,7 @@
// Licensed under the MIT License. See License.txt in the project root for license information.
using Microsoft.CommonDataModel.ObjectModel.Cdm;
using Microsoft.CommonDataModel.ObjectModel.Enums;
using Microsoft.CommonDataModel.ObjectModel.Utilities;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System.Collections.Generic;
@ -30,7 +31,8 @@ namespace Microsoft.CommonDataModel.ObjectModel.Tests.Cdm.Resolution
[TestMethod]
public async Task TestMaxDepthCached()
{
CdmCorpusDefinition cdmCorpus = TestHelper.GetLocalCorpus(testsSubpath, testPath);
var expectedLogCodes = new HashSet<CdmLogCode> { CdmLogCode.WarnMaxDepthExceeded };
CdmCorpusDefinition cdmCorpus = TestHelper.GetLocalCorpus(testsSubpath, testPath, expectedCodes: expectedLogCodes);
CdmEntityDefinition aEnt = await cdmCorpus.FetchObjectAsync<CdmEntityDefinition>("A.cdm.json/A");
CdmEntityDefinition bEnt = await cdmCorpus.FetchObjectAsync<CdmEntityDefinition>("B.cdm.json/B");
CdmEntityDefinition cEnt = await cdmCorpus.FetchObjectAsync<CdmEntityDefinition>("C.cdm.json/C");
@ -75,7 +77,8 @@ namespace Microsoft.CommonDataModel.ObjectModel.Tests.Cdm.Resolution
[TestMethod]
public async Task TestNonMaxDepthCached()
{
CdmCorpusDefinition cdmCorpus = TestHelper.GetLocalCorpus(testsSubpath, "TestMaxDepth");
var expectedLogCodes = new HashSet<CdmLogCode> { CdmLogCode.WarnMaxDepthExceeded };
CdmCorpusDefinition cdmCorpus = TestHelper.GetLocalCorpus(testsSubpath, "TestMaxDepth", expectedCodes: expectedLogCodes);
CdmEntityDefinition aEnt = await cdmCorpus.FetchObjectAsync<CdmEntityDefinition>("A.cdm.json/A");
CdmEntityDefinition bEnt = await cdmCorpus.FetchObjectAsync<CdmEntityDefinition>("B.cdm.json/B");
CdmEntityDefinition cEnt = await cdmCorpus.FetchObjectAsync<CdmEntityDefinition>("C.cdm.json/C");

6
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Tests/Cdm/Resolution/ManifestResolutionTest.cs
Просмотреть файл

@ -121,7 +121,11 @@ namespace Microsoft.CommonDataModel.ObjectModel.Tests.Cdm.Resolution
var cdmCorpus = TestHelper.GetLocalCorpus(testsSubpath, nameof(TestResolveManifestWithInterdependentPolymorphicSource));
var manifest = await cdmCorpus.FetchObjectAsync<CdmManifestDefinition>("local:/Input.manifest.cdm.json");
var resolvedManifest = await manifest.CreateResolvedManifestAsync("resolved", null);
var resOpt = new ResolveOptions
{
MaxDepth = 3
};
var resolvedManifest = await manifest.CreateResolvedManifestAsync("resolved", null, resOpt: resOpt);
Assert.AreEqual(2, resolvedManifest.Entities.Count);
Assert.AreEqual("resolved/group.cdm.json/group", resolvedManifest.Entities[0].EntityPath.ToLowerInvariant());

28
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Tests/Storage/AdlsAdapterTests.cs
Просмотреть файл

@ -707,5 +707,33 @@ namespace Microsoft.CommonDataModel.ObjectModel.Tests.Storage
Assert.IsTrue(notFlushedErrorHit);
}
/// <summary>
/// Tests refreshing data partition gets file size in ADLS
/// </summary>
[TestMethod]
public async Task TestADLSRefreshesDataPartition()
{
AdlsTestHelper.CheckADLSEnvironment();
var adlsAdapter = AdlsTestHelper.CreateAdapterWithSharedKey();
var corpus = new CdmCorpusDefinition();
corpus.Storage.Mount("adls", adlsAdapter);
var cdmManifest = await corpus.FetchObjectAsync<CdmManifestDefinition>("adls:/TestPartitionMetadata/partitions.manifest.cdm.json");
var fileStatusCheckOptions = new FileStatusCheckOptions { IncludeDataPartitionSize = true };
var partitionEntity = cdmManifest.Entities.AllItems[0];
Assert.AreEqual(partitionEntity.DataPartitions.Count, 1);
var partition = partitionEntity.DataPartitions[0];
await cdmManifest.FileStatusCheckAsync(fileStatusCheckOptions: fileStatusCheckOptions);
var localTraitIndex = partition.ExhibitsTraits.IndexOf("is.partition.size");
Assert.AreNotEqual(localTraitIndex, -1);
var localTrait = partition.ExhibitsTraits[localTraitIndex] as CdmTraitReference;
Assert.AreEqual(localTrait.NamedReference, "is.partition.size");
Assert.AreEqual(localTrait.Arguments[0].Value, 2);
}
}
}

28
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Tests/Storage/CdmCustomPackageAdapterTests.cs
Просмотреть файл

@ -1,7 +1,9 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.
using Microsoft.CommonDataModel.ObjectModel.Cdm;
using Microsoft.CommonDataModel.ObjectModel.Storage;
using Microsoft.CommonDataModel.ObjectModel.Utilities;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System;
using System.Reflection;
@ -42,7 +44,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Tests.Storage
Assert.IsTrue(errorCalled);
}
/// <summary>
/// Tests if the calls to CreateCorpusPath return the expected corpus path.
/// </summary>
@ -102,7 +104,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Tests.Storage
Assert.IsTrue(errorWasThrown);
}
/// <summary>
/// Tests if the CdmCustomPackageAdapter works when assembly is passed in the constructor.
/// </summary>
@ -114,5 +116,27 @@ namespace Microsoft.CommonDataModel.ObjectModel.Tests.Storage
Assert.IsNotNull(await adapter.ReadAsync(FoundationsFile));
}
/// <summary>
/// Test mounting CdmStandards adapter from config does not cause an error
/// </summary>
[TestMethod]
public void TestCdmStandardsMountFromConfig()
{
CdmCorpusDefinition corpus = new CdmCorpusDefinition();
corpus.SetEventCallback(new EventCallback
{
Invoke = (CdmStatusLevel statusLevel, string message) =>
{
Assert.Fail(message);
}
}, CdmStatusLevel.Warning);
corpus.Storage.MountFromConfig("{\"adapters\": [{\"config\": {\"locationHint\": \"\", \"maximumTimeout\": 20000, \"numberOfRetries\": 2, \"root\": \"/logical\", \"timeout\": 5000}, \"namespace\": \"cdm\", \"type\": \"cdm-standards\"}], \"defaultNamespace\": \"local\"}");
corpus.Storage.Mount("cdm", new CdmStandardsAdapter());
String config = corpus.Storage.FetchConfig();
corpus.Storage.MountFromConfig(config);
}
}
}

20
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Tests/Storage/TestAdapters/FetchAllMetadataThrowErrorAdapter.cs Normal file
Просмотреть файл

@ -0,0 +1,20 @@
using Microsoft.CommonDataModel.ObjectModel.Storage;
using Microsoft.CommonDataModel.ObjectModel.Utilities;
using System.Collections.Generic;
using System.Threading.Tasks;
namespace Microsoft.CommonDataModel.ObjectModel.Tests.Storage.TestAdapters
{
class FetchAllMetadataThrowErrorAdapter : NoOverrideAdapter
{
public FetchAllMetadataThrowErrorAdapter(LocalAdapter baseAdapter)
: base(baseAdapter)
{
}
public override async Task<IDictionary<string, CdmFileMetadata>> FetchAllFilesMetadataAsync(string folderCorpusPath)
{
throw new System.Exception();
}
}
}

65
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmCorpusDefinition.cs
Просмотреть файл

@ -1363,7 +1363,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
/// </summary>
/// <param name="currManifest">The manifest (and any sub-manifests it contains) that we want to calculate relationships for.</param>
/// <returns>A <see cref="Task"/> for the completion of entity graph calculation.</returns>
public async Task CalculateEntityGraphAsync(CdmManifestDefinition currManifest)
public async Task CalculateEntityGraphAsync(CdmManifestDefinition currManifest, ResolveOptions resOpt = null)
{
using (Logger.EnterScope(nameof(CdmCorpusDefinition), Ctx, nameof(CalculateEntityGraphAsync)))
{
@ -1385,7 +1385,9 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
CdmEntityDefinition resEntity;
// make options wrt this entity document and "relational" always
var resOpt = new ResolveOptions(entity.InDocument, new AttributeResolutionDirectiveSet(new HashSet<string>() { "normalized", "referenceOnly" }));
var resOptCopy = resOpt != null ? resOpt.Copy() : new ResolveOptions();
resOptCopy.WrtDoc = entity.InDocument;
resOptCopy.Directives = new AttributeResolutionDirectiveSet(new HashSet<string>() { "normalized", "referenceOnly" });
bool isResolvedEntity = entity.IsResolved;
@ -1393,7 +1395,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
if (!isResolvedEntity)
{
// first get the resolved entity so that all of the references are present
resEntity = await entity.CreateResolvedEntityAsync($"wrtSelf_{entity.EntityName}", resOpt);
resEntity = await entity.CreateResolvedEntityAsync($"wrtSelf_{entity.EntityName}", resOptCopy);
}
else
{
@ -1402,7 +1404,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
// find outgoing entity relationships using attribute context
List<CdmE2ERelationship> newOutgoingRelationships = this.FindOutgoingRelationships(
resOpt,
resOptCopy,
resEntity,
resEntity.AttributeContext,
isResolvedEntity);
@ -1472,7 +1474,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
var subManifest = await this.FetchObjectAsync<CdmManifestDefinition>(corpusPath);
if (subManifest != null)
{
await this.CalculateEntityGraphAsync(subManifest);
await this.CalculateEntityGraphAsync(subManifest, resOpt);
}
}
}
@ -1937,34 +1939,49 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
/// </summary>
internal async Task<DateTimeOffset?> GetLastModifiedTimeFromPartitionPathAsync(string corpusPath)
{
// we do not want to load partitions from file, just check the modified times
var fileMetadata = await this.GetFileMetadataFromPartitionPathAsync(corpusPath);
if (fileMetadata == null)
{
return null;
}
return fileMetadata.LastModifiedTime;
}
/// <summary>
/// Gets the file metadata of the partition without trying to read the file itself.
/// </summary>
internal async Task<CdmFileMetadata> GetFileMetadataFromPartitionPathAsync(string corpusPath)
{
Tuple<string, string> pathTuple = StorageUtils.SplitNamespacePath(corpusPath);
if (pathTuple == null)
{
Logger.Error(this.Ctx, Tag, nameof(GetLastModifiedTimeFromPartitionPathAsync), corpusPath, CdmLogCode.ErrPathNullObjectPath);
Logger.Error(this.Ctx, Tag, nameof(GetFileMetadataFromPartitionPathAsync), corpusPath, CdmLogCode.ErrPathNullObjectPath);
return null;
}
string nameSpace = pathTuple.Item1;
StorageAdapterBase adapter = null;
if (!string.IsNullOrWhiteSpace(nameSpace))
{
StorageAdapterBase adapter = this.Storage.FetchAdapter(nameSpace);
if (adapter == null)
{
Logger.Error(this.Ctx, Tag, nameof(GetLastModifiedTimeFromPartitionPathAsync), corpusPath, CdmLogCode.ErrStorageAdapterNotFound, corpusPath);
return null;
}
try
{
return await adapter.ComputeLastModifiedTimeAsync(pathTuple.Item2);
}
catch (Exception e)
{
Logger.Error(this.Ctx, Tag, nameof(GetLastModifiedTimeFromPartitionPathAsync), corpusPath, CdmLogCode.ErrPartitionFileModTimeFailure, pathTuple.Item2, e.Message);
}
adapter = this.Storage.FetchAdapter(nameSpace);
}
if (adapter == null)
{
Logger.Error(this.Ctx, Tag, nameof(GetFileMetadataFromPartitionPathAsync), corpusPath, CdmLogCode.ErrStorageAdapterNotFound, corpusPath);
return null;
}
try
{
return await adapter.FetchFileMetadataAsync(pathTuple.Item2);
}
catch (Exception e)
{
Logger.Error(this.Ctx, Tag, nameof(GetFileMetadataFromPartitionPathAsync), corpusPath, CdmLogCode.ErrPartitionFileMetadataFailure, pathTuple.Item2, e.Message);
return null;
}
return null;
}
/// <summary>

20
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmDataPartitionDefinition.cs
Просмотреть файл

@ -5,6 +5,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
{
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.CommonDataModel.ObjectModel.Enums;
using Microsoft.CommonDataModel.ObjectModel.Utilities;
@ -100,7 +101,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
return true;
}
/// <inheritdoc />
public override CdmObject Copy(ResolveOptions resOpt = null, CdmObject host = null)
{
@ -184,14 +185,27 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
/// <inheritdoc />
public async Task FileStatusCheckAsync()
{
await this.FileStatusCheckAsync(null);
}
/// <inheritdoc />
public async Task FileStatusCheckAsync(FileStatusCheckOptions fileStatusCheckOptions, CancellationToken ct = default)
{
ct.ThrowIfCancellationRequested();
using (Logger.EnterScope(nameof(CdmDataPartitionDefinition), Ctx, nameof(FileStatusCheckAsync)))
{
string fullPath = this.Ctx.Corpus.Storage.CreateAbsoluteCorpusPath(this.Location, this.InDocument);
DateTimeOffset? modifiedTime = await this.Ctx.Corpus.GetLastModifiedTimeFromPartitionPathAsync(fullPath);
CdmFileMetadata partitionMetadata = await this.Ctx.Corpus.GetFileMetadataFromPartitionPathAsync(fullPath);
// update modified times
this.LastFileStatusCheckTime = DateTimeOffset.UtcNow;
this.LastFileModifiedTime = TimeUtils.MaxTime(modifiedTime, this.LastFileModifiedTime);
this.LastFileModifiedTime = TimeUtils.MaxTime(partitionMetadata?.LastModifiedTime, this.LastFileModifiedTime);
if (fileStatusCheckOptions?.IncludeDataPartitionSize == true && partitionMetadata?.FileSizeBytes != null)
{
this.ExhibitsTraits.Add("is.partition.size", new List<Tuple<string, dynamic>> { new Tuple<string, dynamic>("value", partitionMetadata.FileSizeBytes) });
}
await this.ReportMostRecentTimeAsync(this.LastFileModifiedTime);
}

19
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmDataPartitionPatternDefinition.cs
Просмотреть файл

@ -7,10 +7,12 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.CommonDataModel.ObjectModel.Enums;
using Microsoft.CommonDataModel.ObjectModel.Storage;
using Microsoft.CommonDataModel.ObjectModel.Utilities;
using Microsoft.CommonDataModel.ObjectModel.Utilities.Exceptions;
using Microsoft.CommonDataModel.ObjectModel.Utilities.Logging;
/// <summary>
@ -186,12 +188,12 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
}
/// <inheritdoc />
public async Task FileStatusCheckAsync(FileStatusCheckOptions fileStatusCheckOptions)
public async Task FileStatusCheckAsync(FileStatusCheckOptions fileStatusCheckOptions, CancellationToken ct = default)
{
await this.FileStatusCheckAsyncInternal(fileStatusCheckOptions);
await this.FileStatusCheckAsyncInternal(fileStatusCheckOptions, ct);
}
internal async Task<bool> FileStatusCheckAsyncInternal(FileStatusCheckOptions fileStatusCheckOptions)
internal async Task<bool> FileStatusCheckAsyncInternal(FileStatusCheckOptions fileStatusCheckOptions, CancellationToken ct)
{
using (Logger.EnterScope(nameof(CdmDataPartitionPatternDefinition), Ctx, nameof(FileStatusCheckAsync)))
{
@ -226,12 +228,19 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
return true;
}
ct.ThrowIfCancellationRequested();
// get a list of all corpusPaths under the root
fileInfoList = await adapter.FetchAllFilesMetadataAsync(pathTuple.Item2);
}
catch (Exception e)
{
Logger.Warning(this.Ctx, Tag, nameof(FileStatusCheckAsync), this.AtCorpusPath, CdmLogCode.WarnPartitionFileFetchFailed, rootCorpus, e.Message);
if (fileStatusCheckOptions?.ThrowOnPartitionError == true)
{
throw new CdmReadPartitionFromPatternException($"There was an error fetching partitions from '{rootCorpus}', see the inner exception.", e);
}
}
// update modified times
@ -258,7 +267,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
if (this.Owner is CdmLocalEntityDeclarationDefinition localEntDecDefOwner)
{
// if both are present log warning and use glob pattern, otherwise use regularExpression
if (!String.IsNullOrWhiteSpace(this.GlobPattern) && !String.IsNullOrWhiteSpace(this.RegularExpression))
if (!string.IsNullOrWhiteSpace(this.GlobPattern) && !string.IsNullOrWhiteSpace(this.RegularExpression))
{
Logger.Warning(this.Ctx,
Tag,
@ -320,6 +329,8 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
foreach (var fi in cleanedFileList)
{
ct.ThrowIfCancellationRequested();
string fileName = fi.Key;
CdmFileMetadata partitionMetadata = fi.Value;

10
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmEntityAttributeDefinition.cs
Просмотреть файл

@ -203,7 +203,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
// this context object holds all of the info about what needs to happen to resolve these attributes.
// make a copy and add defaults if missing
CdmAttributeResolutionGuidance resGuideWithDefault = this.ResolutionGuidance == null
CdmAttributeResolutionGuidance resGuideWithDefault = this.ResolutionGuidance == null
? new CdmAttributeResolutionGuidance(this.Ctx) : this.ResolutionGuidance?.Copy(resOpt) as CdmAttributeResolutionGuidance;
resGuideWithDefault.UpdateAttributeDefaults(this.Name, this);
@ -259,7 +259,11 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
// A Projection
// if the max depth is exceeded it should not try to execute the projection
if (!resOpt.DepthInfo.MaxDepthExceeded)
if (resOpt.DepthInfo.MaxDepthExceeded)
{
Logger.Warning(this.Ctx, Tag, nameof(ConstructResolvedAttributes), this.AtCorpusPath, CdmLogCode.WarnMaxDepthExceeded, resOpt.DepthInfo.MaxDepth?.ToString(), this.Entity.FetchObjectDefinitionName());
}
else
{
CdmProjection projDef = this.Entity.FetchObjectDefinition<CdmProjection>(resOpt);
ProjectionDirective projDirective = new ProjectionDirective(resOpt, this, ownerRef: this.Entity);
@ -346,6 +350,8 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
// if we got here because of the max depth, need to impose the directives to make the trait work as expected
if (resOpt.DepthInfo.MaxDepthExceeded)
{
Logger.Warning(this.Ctx, Tag, nameof(ConstructResolvedAttributes), this.AtCorpusPath, CdmLogCode.WarnMaxDepthExceeded, resOpt.DepthInfo.MaxDepth?.ToString(), this.Entity.FetchObjectDefinitionName());
if (arc.ResOpt.Directives == null)
arc.ResOpt.Directives = new AttributeResolutionDirectiveSet();
arc.ResOpt.Directives.Add("referenceOnly");

1
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmFileStatus.cs
Просмотреть файл

@ -4,6 +4,7 @@
namespace Microsoft.CommonDataModel.ObjectModel.Cdm
{
using System;
using System.Threading;
using System.Threading.Tasks;
public interface CdmFileStatus : CdmObject

25
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmLocalEntityDeclarationDefinition.cs
Просмотреть файл

@ -6,6 +6,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.CommonDataModel.ObjectModel.Enums;
using Microsoft.CommonDataModel.ObjectModel.Storage;
@ -214,12 +215,12 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
await this.FileStatusCheckAsync(PartitionFileStatusCheckType.Full);
}
public async Task FileStatusCheckAsync(PartitionFileStatusCheckType partitionFileStatusCheckType = PartitionFileStatusCheckType.Full, CdmIncrementalPartitionType incrementalType = CdmIncrementalPartitionType.None, FileStatusCheckOptions fileStatusCheckOptions = null)
public async Task FileStatusCheckAsync(PartitionFileStatusCheckType partitionFileStatusCheckType = PartitionFileStatusCheckType.Full, CdmIncrementalPartitionType incrementalType = CdmIncrementalPartitionType.None, FileStatusCheckOptions fileStatusCheckOptions = null, CancellationToken ct = default)
{
await FileStatusCheckAsyncInternal(partitionFileStatusCheckType, incrementalType, fileStatusCheckOptions);
await FileStatusCheckAsyncInternal(partitionFileStatusCheckType, incrementalType, fileStatusCheckOptions, ct);
}
internal async Task<bool> FileStatusCheckAsyncInternal(PartitionFileStatusCheckType partitionFileStatusCheckType = PartitionFileStatusCheckType.Full, CdmIncrementalPartitionType incrementalType = CdmIncrementalPartitionType.None, FileStatusCheckOptions fileStatusCheckOptions = null)
internal async Task<bool> FileStatusCheckAsyncInternal(PartitionFileStatusCheckType partitionFileStatusCheckType, CdmIncrementalPartitionType incrementalType, FileStatusCheckOptions fileStatusCheckOptions, CancellationToken ct)
{
using ((this.Ctx.Corpus.Storage.FetchAdapter(this.InDocument.Namespace) as StorageAdapterBase)?.CreateFileQueryCacheContext())
{
@ -234,6 +235,8 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
{
foreach (var pattern in this.DataPartitionPatterns)
{
ct.ThrowIfCancellationRequested();
if (pattern.IsIncremental)
{
Logger.Error(pattern.Ctx, Tag, nameof(FileStatusCheckAsync), pattern.AtCorpusPath, CdmLogCode.ErrUnexpectedIncrementalPartitionTrait,
@ -241,7 +244,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
}
else
{
bool shouldContinue = await pattern.FileStatusCheckAsyncInternal(fileStatusCheckOptions);
bool shouldContinue = await pattern.FileStatusCheckAsyncInternal(fileStatusCheckOptions, ct);
if (!shouldContinue)
{
@ -252,6 +255,8 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
foreach (var partition in this.DataPartitions)
{
ct.ThrowIfCancellationRequested();
if (partition.IsIncremental)
{
Logger.Error(partition.Ctx, Tag, nameof(FileStatusCheckAsync), partition.AtCorpusPath, CdmLogCode.ErrUnexpectedIncrementalPartitionTrait,
@ -259,7 +264,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
}
else
{
await partition.FileStatusCheckAsync();
await partition.FileStatusCheckAsync(fileStatusCheckOptions, ct);
}
}
}
@ -268,17 +273,23 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
{
foreach (var pattern in this.IncrementalPartitionPatterns)
{
ct.ThrowIfCancellationRequested();
if (this.ShouldCallFileStatusCheck(incrementalType, true, pattern))
{
await pattern.FileStatusCheckAsync();
await pattern.FileStatusCheckAsync(null, ct);
}
}
foreach (var partition in this.IncrementalPartitions)
{
ct.ThrowIfCancellationRequested();
if (this.ShouldCallFileStatusCheck(incrementalType, false, partition))
{
await partition.FileStatusCheckAsync();
await partition.FileStatusCheckAsync(null, ct);
}
}
}
// update modified times

33
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmManifestDefinition.cs
Просмотреть файл

@ -5,11 +5,12 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
{
using Microsoft.CommonDataModel.ObjectModel.Enums;
using Microsoft.CommonDataModel.ObjectModel.Utilities;
using System.Collections.Generic;
using System;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.CommonDataModel.ObjectModel.Utilities.Logging;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
public class CdmManifestDefinition : CdmDocumentDefinition, CdmObjectDefinition, CdmFileStatus
{
@ -175,7 +176,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
/// <param name="newEntityDocumentNameFormat"></param>
/// <param name="Directives"></param>
/// <returns></returns>
public async Task<CdmManifestDefinition> CreateResolvedManifestAsync(string newManifestName, string newEntityDocumentNameFormat, AttributeResolutionDirectiveSet Directives = null)
public async Task<CdmManifestDefinition> CreateResolvedManifestAsync(string newManifestName, string newEntityDocumentNameFormat, AttributeResolutionDirectiveSet Directives = null, ResolveOptions resOpt = null)
{
using (Logger.EnterScope(nameof(CdmManifestDefinition), Ctx, nameof(CreateResolvedManifestAsync)))
{
@ -294,22 +295,20 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
// Next create the resolved entity
AttributeResolutionDirectiveSet withDirectives = Directives != null ? Directives : this.Ctx.Corpus.DefaultResolutionDirectives;
var resOpt = new ResolveOptions
{
WrtDoc = entDef.InDocument,
Directives = withDirectives?.Copy()
};
var entityResOpt = resOpt != null ? resOpt.Copy() : new ResolveOptions();
entityResOpt.WrtDoc = entDef.InDocument;
entityResOpt.Directives = withDirectives?.Copy();
Logger.Debug(this.Ctx as ResolveContext, Tag, nameof(CreateResolvedManifestAsync), this.AtCorpusPath, $"resolving entity {sourceEntityFullPath} to document {newDocumentFullPath}");
var resolvedEntity = await entDef.CreateResolvedEntityAsync(entDef.EntityName, resOpt, folder, newDocumentName);
var resolvedEntity = await entDef.CreateResolvedEntityAsync(entDef.EntityName, entityResOpt, folder, newDocumentName);
if (resolvedEntity == null)
{
// Fail all resolution, if any one entity resolution fails
return null;
}
var result = entity.Copy(resOpt) as CdmEntityDeclarationDefinition;
var result = entity.Copy(entityResOpt) as CdmEntityDeclarationDefinition;
if (result.ObjectType == CdmObjectType.LocalEntityDeclarationDef)
{
result.EntityPath = this.Ctx.Corpus.Storage.CreateRelativeCorpusPath(resolvedEntity.AtCorpusPath, resolvedManifest) ?? result.AtCorpusPath;
@ -321,7 +320,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
Logger.Debug(this.Ctx as ResolveContext, Tag, nameof(CreateResolvedManifestAsync), this.AtCorpusPath, "calculating relationships");
// calculate the entity graph for just this manifest and any submanifests
await this.Ctx.Corpus.CalculateEntityGraphAsync(resolvedManifest);
await this.Ctx.Corpus.CalculateEntityGraphAsync(resolvedManifest, resOpt);
// stick results into the relationships list for the manifest
// only put in relationships that are between the entities that are used in the manifest
await resolvedManifest.PopulateManifestRelationshipsAsync(CdmRelationshipDiscoveryStyle.Exclusive);
@ -535,7 +534,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
await this.FileStatusCheckAsync(PartitionFileStatusCheckType.Full);
}
public async Task FileStatusCheckAsync(PartitionFileStatusCheckType partitionFileStatusCheckType = PartitionFileStatusCheckType.Full, CdmIncrementalPartitionType incrementalType = CdmIncrementalPartitionType.None, FileStatusCheckOptions fileStatusCheckOptions = null)
public async Task FileStatusCheckAsync(PartitionFileStatusCheckType partitionFileStatusCheckType = PartitionFileStatusCheckType.Full, CdmIncrementalPartitionType incrementalType = CdmIncrementalPartitionType.None, FileStatusCheckOptions fileStatusCheckOptions = null, CancellationToken ct = default)
{
using (Logger.EnterScope(nameof(CdmManifestDefinition), Ctx, nameof(FileStatusCheckAsync)))
{
@ -557,13 +556,15 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
foreach (var entity in this.Entities)
{
ct.ThrowIfCancellationRequested();
if (entity is CdmReferencedEntityDeclarationDefinition)
{
await entity.FileStatusCheckAsync();
}
else if (entity is CdmLocalEntityDeclarationDefinition localEntity)
{
bool shouldContinue = await localEntity.FileStatusCheckAsyncInternal(partitionFileStatusCheckType, incrementalType, fileStatusCheckOptions);
bool shouldContinue = await localEntity.FileStatusCheckAsyncInternal(partitionFileStatusCheckType, incrementalType, fileStatusCheckOptions, ct);
if (!shouldContinue)
{
@ -574,6 +575,8 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
foreach (var subManifest in this.SubManifests)
{
ct.ThrowIfCancellationRequested();
await subManifest.FileStatusCheckAsync();
}
}

3
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmObjectBase.cs
Просмотреть файл

@ -7,11 +7,13 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
using Microsoft.CommonDataModel.ObjectModel.Persistence;
using Microsoft.CommonDataModel.ObjectModel.ResolvedModel;
using Microsoft.CommonDataModel.ObjectModel.Utilities;
using Microsoft.CommonDataModel.ObjectModel.Utilities.Logging;
using System;
using System.Collections.Generic;
public abstract class CdmObjectBase : CdmObject
{
private static readonly string Tag = nameof(CdmObjectBase);
/// <summary>
/// The minimum json semantic versions that can be loaded by this ObjectModel version.
/// </summary>
@ -353,6 +355,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
// if using the cache passes the maxDepth, we cannot use it
if (rasbCache != null && resOpt.DepthInfo.CurrentDepth + rasbCache.ResolvedAttributeSet.DepthTraveled > resOpt.DepthInfo.MaxDepth)
{
Logger.Warning(ctx, Tag, nameof(FetchResolvedAttributes), this.AtCorpusPath, CdmLogCode.WarnMaxDepthExceeded, resOpt.DepthInfo.MaxDepth?.ToString(), this.FetchObjectDefinitionName());
rasbCache = null;
}

2
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Enums/CdmLogCode.cs
Просмотреть файл

@ -27,6 +27,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Enums
ErrManifestFileModTimeFailure,
ErrMissingIncrementalPartitionTrait,
ErrObjectWithoutOwnerFound,
ErrPartitionFileMetadataFailure,
ErrPartitionFileModTimeFailure,
ErrPathIsDuplicate,
ErrPathNullObjectPath,
@ -156,6 +157,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Enums
WarnDocImportNotLoaded,
WarnPartitionFileFetchFailed,
WarnLinkEntIdentArgsNotSupported,
WarnMaxDepthExceeded,
WarnPartitionGlobAndRegexPresent,
WarnPartitionInvalidArguments,
WarnPersistCustomExtNotSupported,

20
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Resx/LogMessages.Designer.cs сгенерированный
Просмотреть файл

@ -187,7 +187,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Resx {
}
/// <summary>
/// Looks up a localized string similar to Could not fetch file metadata. The FetchAllFilesMetadata method of the &apos;{0}&apos; adapter may be misconfigured..
/// Looks up a localized string similar to Could not fetch file metadata from root location, the &apos;{0}&apos; adapter may be misconfigured..
/// </summary>
internal static string ErrFetchingFileMetadataNull {
get {
@ -249,6 +249,15 @@ namespace Microsoft.CommonDataModel.ObjectModel.Resx {
}
}
/// <summary>
/// Looks up a localized string similar to Failed to get file metadata for partition file {0}. Exception: {1}.
/// </summary>
internal static string ErrPartitionFileMetadataFailure {
get {
return ResourceManager.GetString("ErrPartitionFileMetadataFailure", resourceCulture);
}
}
/// <summary>
/// Looks up a localized string similar to Failed to compute last modified time for partition file {0}. Exception: {1}.
/// </summary>
@ -1455,6 +1464,15 @@ namespace Microsoft.CommonDataModel.ObjectModel.Resx {
}
}
/// <summary>
/// Looks up a localized string similar to The entity max depth of {0} has been exceeded. Entity {1} will not be resolved any further..
/// </summary>
internal static string WarnMaxDepthExceeded {
get {
return ResourceManager.GetString("WarnMaxDepthExceeded", resourceCulture);
}
}
/// <summary>
/// Looks up a localized string similar to Failed to fetch all files in the folder location &apos;{0}&apos; described by a partition pattern. Exception: {1}.
/// </summary>

6
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Resx/LogMessages.resx
Просмотреть файл

@ -693,4 +693,10 @@
<data name="ErrRegexTimeoutOutOfRange" xml:space="preserve">
<value>Regex timeout '{0}' is not valid. Reason '{1}'</value>
</data>
<data name="ErrPartitionFileMetadataFailure" xml:space="preserve">
<value>Failed to get file metadata for partition file {0}. Exception: {1}</value>
</data>
<data name="WarnMaxDepthExceeded" xml:space="preserve">
<value>The entity max depth of {0} has been exceeded. Entity {1} will not be resolved any further.</value>
</data>
</root>

5
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Storage/CdmStandardsAdapter.cs
Просмотреть файл

@ -11,5 +11,10 @@ namespace Microsoft.CommonDataModel.ObjectModel.Storage
: base(cdmStandardsPackageName)
{
}
public override string FetchConfig()
{
return "{\"config\":{},\"type\":\"cdm-standards\"}";
}
}
}

19
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Storage/LocalAdapter.cs
Просмотреть файл

@ -140,14 +140,27 @@ namespace Microsoft.CommonDataModel.ObjectModel.Storage
}
/// <inheritdoc />
public override Task<DateTimeOffset?> ComputeLastModifiedTimeAsync(string corpusPath)
public override async Task<DateTimeOffset?> ComputeLastModifiedTimeAsync(string corpusPath)
{
var fileMetadata = await this.FetchFileMetadataAsync(corpusPath);
if (fileMetadata == null)
{
return null;
}
return fileMetadata.LastModifiedTime;
}
/// <inheritdoc />
public override Task<CdmFileMetadata> FetchFileMetadataAsync(string corpusPath)
{
var adapterPath = this.CreateAdapterPath(corpusPath);
FileInfo fileInfo = new FileInfo(adapterPath);
if (fileInfo.Exists)
return Task.FromResult((DateTimeOffset?)fileInfo.LastWriteTimeUtc);
return Task.FromResult(new CdmFileMetadata { FileSizeBytes = fileInfo.Length, LastModifiedTime = (DateTimeOffset?)fileInfo.LastWriteTimeUtc});
else
return Task.FromResult<DateTimeOffset?>(null);
return Task.FromResult<CdmFileMetadata>(null);
}
/// <inheritdoc />

8
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Storage/StorageAdapterBase.cs
Просмотреть файл

@ -83,6 +83,14 @@ namespace Microsoft.CommonDataModel.ObjectModel.Storage
return Task.FromResult<DateTimeOffset?>(DateTimeOffset.UtcNow);
}
/// <summary>
/// Returns the file metadata info about the specified document.
/// </summary>
public virtual Task<CdmFileMetadata> FetchFileMetadataAsync(string corpusPath)
{
return Task.FromResult<CdmFileMetadata>(null);
}
/// <summary>
/// Returns a list of corpus paths to all files and folders at or under the provided corpus path to a folder.
/// </summary>

4
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Utilities/CdmFileMetadata.cs
Просмотреть файл

@ -3,8 +3,12 @@
namespace Microsoft.CommonDataModel.ObjectModel.Utilities
{
using System;
public class CdmFileMetadata
{
public long? FileSizeBytes { get; set; }
public DateTimeOffset? LastModifiedTime { get; set; }
}
}

18
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Utilities/Exceptions/CdmReadPartitionFromPatternException.cs Normal file
Просмотреть файл

@ -0,0 +1,18 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.
namespace Microsoft.CommonDataModel.ObjectModel.Utilities.Exceptions
{
using System;
public class CdmReadPartitionFromPatternException : Exception
{
/// <summary>
/// Initializes a new instance of the <see cref="CdmReadPartitionFromPatternException"/> class.
/// </summary>
/// <param name="message">The message.</param>
public CdmReadPartitionFromPatternException(string message, Exception innerException) : base(message, innerException)
{
}
}
}

2
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Utilities/FileStatusCheckOptions.cs
Просмотреть файл

@ -8,5 +8,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Utilities
public bool IncludeDataPartitionSize { get; set; }
public double? RegexTimeoutSeconds { get; set; }
public bool ThrowOnPartitionError { get; set; }
}
}

1
objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Utilities/ResolveOptions.cs
Просмотреть файл

@ -113,6 +113,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Utilities
{
WrtDoc = this.WrtDoc,
DepthInfo = this.DepthInfo.Copy(),
MaxDepth = this.MaxDepth,
LocalizeReferencesFor = this.LocalizeReferencesFor,
IndexingDoc = this.IndexingDoc,
ShallowValidation = this.ShallowValidation,

4
objectModel/Java/objectmodel/resx/LogMessages.properties
Просмотреть файл

@ -20,6 +20,7 @@ ErrInvalidPath=Invalid path '{0}'
ErrManifestFileModTimeFailure=Failed to compute last modified time for manifest file {0}. Exception: {1}
ErrMissingIncrementalPartitionTrait=This '{0}' object '{1}' does not contain the trait '{2}', so it should not be in the collection '{3}'.
ErrObjectWithoutOwnerFound=Found object without owner when calculating relationships.
ErrPartitionFileMetadataFailure=Failed to get file metadata for partition file {0}. Exception: {1}
ErrPartitionFileModTimeFailure=Failed to compute last modified time for partition file {0}. Exception: {1}
ErrPathIsDuplicate=Duplicate declaration for item '{0}'
ErrPathNullObjectPath=The object path cannot be null or empty.
@ -99,7 +100,7 @@ ErrStorageNamespaceNotRegistered=The namespace {0} has not been registered
ErrStorageNullAdapter=The adapter cannot be null.
ErrStorageNullAdapterConfig=Adapter config cannot be null or empty.
ErrStorageNullCorpusPath=The corpus path is null or empty
ErrStorageNullNamespace=The namespace cannot be null or empty
ErrStorageNullNamespace=The namespace '{0}' cannot be null or empty
ErrStorageObjectNodeCastFailed=Config cannot be cast to objectNode. Config: {0}, Error: {1}
ErrSymbolNotFound=Cannot find symbol '{0}'
ErrTraitArgumentMissing=No argument supplied for required parameter '{0}' of trait '{1}' on '{2}'
@ -124,6 +125,7 @@ WarnPartitionGlobAndRegexPresent=The Data Partition Pattern contains both a glob
WarnDeprecatedResolutionGuidance=Resolution guidance is being deprecated in favor of Projections. https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#projection-overview.
WarnDocChangesDiscarded=discarding changes in document: {0}
WarnDocImportNotLoaded=Import document {0} not loaded. This may cause unexpected behavior of the model.
WarnMaxDepthExceeded=The entity max depth of {0} has been exceeded. Entity {1} will not be resolved any further.
WarnPartitionFileFetchFailed=Failed to fetch all files in the folder location '{0}' described by a partition pattern. Exception: {1}
WarnIdentifierArgumentsNotSupported=Trait 'is.linkedEntity.identifier' is not accepting arguments, skipping relationship recording for attribute '{0}' in entity '{1}'.
WarnPartitionInvalidArguments=Invalid set of arguments provided for data partition corresponding to location: {0}

74
objectModel/Java/objectmodel/src/main/java/com/microsoft/commondatamodel/objectmodel/cdm/CdmCorpusDefinition.java
Просмотреть файл

@ -29,16 +29,7 @@ import com.microsoft.commondatamodel.objectmodel.resolvedmodel.ResolvedTrait;
import com.microsoft.commondatamodel.objectmodel.resolvedmodel.ResolvedTraitSet;
import com.microsoft.commondatamodel.objectmodel.storage.StorageAdapterBase;
import com.microsoft.commondatamodel.objectmodel.storage.StorageManager;
import com.microsoft.commondatamodel.objectmodel.utilities.AttributeResolutionDirectiveSet;
import com.microsoft.commondatamodel.objectmodel.utilities.DepthInfo;
import com.microsoft.commondatamodel.objectmodel.utilities.DocsResult;
import com.microsoft.commondatamodel.objectmodel.utilities.EventCallback;
import com.microsoft.commondatamodel.objectmodel.utilities.ImportInfo;
import com.microsoft.commondatamodel.objectmodel.utilities.ResolveOptions;
import com.microsoft.commondatamodel.objectmodel.utilities.StorageUtils;
import com.microsoft.commondatamodel.objectmodel.utilities.StringUtils;
import com.microsoft.commondatamodel.objectmodel.utilities.SymbolSet;
import com.microsoft.commondatamodel.objectmodel.utilities.VisitCallback;
import com.microsoft.commondatamodel.objectmodel.utilities.*;
import com.microsoft.commondatamodel.objectmodel.utilities.logger.TelemetryClient;
import java.time.OffsetDateTime;
@ -1325,6 +1316,19 @@ public class CdmCorpusDefinition {
* @return A link of CompletableFuture for the completion of entity graph calculation.
*/
public CompletableFuture<Void> calculateEntityGraphAsync(final CdmManifestDefinition currManifest) {
return this.calculateEntityGraphAsync(currManifest, null);
}
/**
* Calculates the entity to entity relationships for all the entities present in the manifest and
* its sub-manifests.
*
* @param currManifest The manifest (and any sub-manifests it contains) that we want to calculate
* relationships for.
* @param resOpt The resolve options.
* @return A link of CompletableFuture for the completion of entity graph calculation.
*/
public CompletableFuture<Void> calculateEntityGraphAsync(final CdmManifestDefinition currManifest, final ResolveOptions resOpt) {
return CompletableFuture.runAsync(() -> {
try (Logger.LoggerScope logScope = Logger.enterScope(CdmCorpusDefinition.class.getSimpleName(), ctx, "calculateEntityGraphAsync")) {
if (currManifest.getEntities() != null) {
@ -1349,20 +1353,22 @@ public class CdmCorpusDefinition {
Set<String> directives = new LinkedHashSet<>();
directives.add("normalized");
directives.add("referenceOnly");
final ResolveOptions resOpt = new ResolveOptions(entity.getInDocument(), new AttributeResolutionDirectiveSet(directives));
final ResolveOptions resOptCopy = resOpt != null ? resOpt.copy() : new ResolveOptions();
resOptCopy.setWrtDoc(entity.getInDocument());
resOptCopy.setDirectives(new AttributeResolutionDirectiveSet(directives));
final boolean isResolvedEntity = entity.getIsResolved();
// only create a resolved entity if the entity passed in was not a resolved entity
if (!isResolvedEntity) {
// first get the resolved entity so that all of the references are present
resEntity = entity.createResolvedEntityAsync("wrtSelf_" + entity.getEntityName(), resOpt).join();
resEntity = entity.createResolvedEntityAsync("wrtSelf_" + entity.getEntityName(), resOptCopy).join();
} else {
resEntity = entity;
}
// find outgoing entity relationships using attribute context
final ArrayList<CdmE2ERelationship> newOutgoingRelationships =
this.findOutgoingRelationships(resOpt, resEntity, resEntity.getAttributeContext(), isResolvedEntity);
this.findOutgoingRelationships(resOptCopy, resEntity, resEntity.getAttributeContext(), isResolvedEntity);
final ArrayList<CdmE2ERelationship> oldOutgoingRelationships = this.outgoingRelationships.get(entity.getAtCorpusPath());
@ -1443,7 +1449,7 @@ public class CdmCorpusDefinition {
Logger.error(this.getCtx(), TAG, "calculateEntityGraphAsync", currManifest.getAtCorpusPath(), CdmLogCode.ErrInvalidCast, subManifestDef.getDefinition(), "CdmManifestDefinition");
}
if (subManifest != null) {
this.calculateEntityGraphAsync(subManifest).join();
this.calculateEntityGraphAsync(subManifest, resOpt).join();
}
}
}
@ -2351,7 +2357,7 @@ public class CdmCorpusDefinition {
return adapter.computeLastModifiedTimeAsync(path);
} catch (Exception e) {
Logger.error(this.ctx, TAG, "getLastModifiedTimeAsyncFromObjectAsync", currObject.getAtCorpusPath(), CdmLogCode.ErrManifestFileModTimeFailure, path, e.getMessage());
return null;
return CompletableFuture.completedFuture(null);
}
} else {
return getLastModifiedTimeFromObjectAsync(currObject.getInDocument());
@ -2373,26 +2379,42 @@ public class CdmCorpusDefinition {
* @return The last modified time
*/
CompletableFuture<OffsetDateTime> getLastModifiedTimeFromPartitionPathAsync(final String corpusPath) {
CdmFileMetadata fileMetadata = this.getFileMetadataFromPartitionPathAsync(corpusPath).join();
if (fileMetadata == null) {
return null;
}
return CompletableFuture.completedFuture(fileMetadata.getLastModifiedTime());
}
/**
*
* @param corpusPath The corpus path
* @return The Cdm File Metadata object
*/
CompletableFuture<CdmFileMetadata> getFileMetadataFromPartitionPathAsync(final String corpusPath) {
// we do not want to load partitions from file, just check the modified times
final Pair<String, String> pathTuple = StorageUtils.splitNamespacePath(corpusPath);
if (pathTuple == null) {
Logger.error(this.ctx, TAG, "getLastModifiedTimeFromPartitionPathAsync", corpusPath, CdmLogCode.ErrPathNullObjectPath);
Logger.error(this.ctx, TAG, "getFileMetadataFromPartitionPathAsync", corpusPath, CdmLogCode.ErrPathNullObjectPath);
return CompletableFuture.completedFuture(null);
}
final String nameSpace = pathTuple.getLeft();
StorageAdapterBase adapter = null;
if (!StringUtils.isNullOrTrimEmpty(nameSpace)) {
final StorageAdapterBase adapter = this.storage.fetchAdapter(nameSpace);
adapter = this.storage.fetchAdapter(nameSpace);
}
if (adapter == null) {
Logger.error(this.ctx, TAG, "getLastModifiedTimeFromPartitionPathAsync", corpusPath, CdmLogCode.ErrStorageAdapterNotFound, nameSpace);
return CompletableFuture.completedFuture(null);
}
try {
return adapter.computeLastModifiedTimeAsync(pathTuple.getRight());
} catch (Exception e) {
Logger.error(this.ctx, TAG, "getLastModifiedTimeFromPartitionPathAsync", corpusPath, CdmLogCode.ErrPartitionFileModTimeFailure, pathTuple.getRight(), e.getMessage());
}
if (adapter == null) {
Logger.error(this.ctx, TAG, "getFileMetadataFromPartitionPathAsync", corpusPath, CdmLogCode.ErrStorageAdapterNotFound, nameSpace);
return CompletableFuture.completedFuture(null);
}
try {
return adapter.fetchFileMetadataAsync(pathTuple.getRight());
} catch (Exception e) {
Logger.error(this.ctx, TAG, "getFileMetadataFromPartitionPathAsync", corpusPath, CdmLogCode.ErrPartitionFileModTimeFailure, pathTuple.getRight(), e.getMessage());
}
return CompletableFuture.completedFuture(null);

29
objectModel/Java/objectmodel/src/main/java/com/microsoft/commondatamodel/objectmodel/cdm/CdmDataPartitionDefinition.java
Просмотреть файл

@ -5,20 +5,13 @@ package com.microsoft.commondatamodel.objectmodel.cdm;
import com.microsoft.commondatamodel.objectmodel.enums.CdmObjectType;
import com.microsoft.commondatamodel.objectmodel.enums.CdmPropertyName;
import com.microsoft.commondatamodel.objectmodel.utilities.CopyOptions;
import com.microsoft.commondatamodel.objectmodel.utilities.ResolveOptions;
import com.microsoft.commondatamodel.objectmodel.utilities.TimeUtils;
import com.microsoft.commondatamodel.objectmodel.utilities.TraitToPropertyMap;
import com.microsoft.commondatamodel.objectmodel.utilities.VisitCallback;
import com.microsoft.commondatamodel.objectmodel.utilities.*;
import com.microsoft.commondatamodel.objectmodel.utilities.logger.Logger;
import org.apache.commons.lang3.tuple.ImmutablePair;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.concurrent.CompletableFuture;
public class CdmDataPartitionDefinition extends CdmObjectDefinitionBase implements CdmFileStatus {
@ -279,6 +272,12 @@ public class CdmDataPartitionDefinition extends CdmObjectDefinitionBase implemen
*/
@Override
public CompletableFuture<Void> fileStatusCheckAsync() {
this.fileStatusCheckAsync(null);
return CompletableFuture.completedFuture(null);
}
public CompletableFuture<Void> fileStatusCheckAsync(FileStatusCheckOptions fileStatusCheckOptions) {
try (Logger.LoggerScope logScope = Logger.enterScope(CdmDataPartitionDefinition.class.getSimpleName(), getCtx(), "fileStatusCheckAsync")) {
final String fullPath =
this.getCtx()
@ -286,12 +285,16 @@ public class CdmDataPartitionDefinition extends CdmObjectDefinitionBase implemen
.getStorage()
.createAbsoluteCorpusPath(this.getLocation(), this.getInDocument());
final OffsetDateTime modifiedTime =
this.getCtx().getCorpus().getLastModifiedTimeFromPartitionPathAsync(fullPath).join();
final CdmFileMetadata fileMetadata =
this.getCtx().getCorpus().getFileMetadataFromPartitionPathAsync(fullPath).join();
// update modified times
setLastFileStatusCheckTime(OffsetDateTime.now(ZoneOffset.UTC));
setLastFileModifiedTime(TimeUtils.maxTime(modifiedTime, getLastFileModifiedTime()));
setLastFileModifiedTime(TimeUtils.maxTime(fileMetadata != null ? fileMetadata.getLastModifiedTime() : null, getLastFileModifiedTime()));
if (fileStatusCheckOptions != null && fileStatusCheckOptions.getIncludeDataPartitionSize() == true && fileMetadata != null) {
this.getExhibitsTraits().add("is.partition.size", new ArrayList<>(Collections.singletonList(new ImmutablePair<>("value", fileMetadata.getSize()))));
}
return reportMostRecentTimeAsync(getLastFileModifiedTime());
}

276
objectModel/Java/objectmodel/src/main/java/com/microsoft/commondatamodel/objectmodel/cdm/CdmDataPartitionPatternDefinition.java
Просмотреть файл

@ -16,6 +16,7 @@ import com.microsoft.commondatamodel.objectmodel.utilities.StorageUtils;
import com.microsoft.commondatamodel.objectmodel.utilities.StringUtils;
import com.microsoft.commondatamodel.objectmodel.utilities.TraitToPropertyMap;
import com.microsoft.commondatamodel.objectmodel.utilities.VisitCallback;
import com.microsoft.commondatamodel.objectmodel.utilities.exceptions.CdmReadPartitionFromPatternException;
import com.microsoft.commondatamodel.objectmodel.utilities.logger.Logger;
import org.apache.commons.lang3.tuple.ImmutablePair;
@ -32,6 +33,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
@ -287,162 +289,166 @@ public class CdmDataPartitionPatternDefinition extends CdmObjectDefinitionBase i
* @param fileStatusCheckOptions The set of options used to determine what information should be set in found partitions
* @return CompletableFuture
*/
public CompletableFuture<Void> fileStatusCheckAsync(FileStatusCheckOptions fileStatusCheckOptions) {
return CompletableFuture.runAsync(() -> {
try (Logger.LoggerScope logScope = Logger.enterScope(CdmDataPartitionPatternDefinition.class.getSimpleName(), getCtx(), "fileStatusCheckAsync")) {
String nameSpace = null;
StorageAdapterBase adapter = null;
public CompletableFuture<Void> fileStatusCheckAsync(FileStatusCheckOptions fileStatusCheckOptions) throws CdmReadPartitionFromPatternException {
return CompletableFuture.runAsync(() -> {
try (Logger.LoggerScope logScope = Logger.enterScope(CdmDataPartitionPatternDefinition.class.getSimpleName(), getCtx(), "fileStatusCheckAsync")) {
String nameSpace = null;
StorageAdapterBase adapter = null;
// make sure the root is a good full corpus path
String rootCleaned = getRootLocation() != null && getRootLocation().endsWith("/") ? getRootLocation().substring(0, getRootLocation().length() - 1) : getRootLocation();
// make sure the root is a good full corpus path
String rootCleaned = getRootLocation() != null && getRootLocation().endsWith("/") ? getRootLocation().substring(0, getRootLocation().length() - 1) : getRootLocation();
if (rootCleaned == null) {
rootCleaned = "";
}
if (rootCleaned == null) {
rootCleaned = "";
}
final String rootCorpus = getCtx().getCorpus().getStorage().createAbsoluteCorpusPath(rootCleaned, getInDocument());
final String rootCorpus = getCtx().getCorpus().getStorage().createAbsoluteCorpusPath(rootCleaned, getInDocument());
HashMap<String, CdmFileMetadata> fileInfoList = null;
try {
// Remove namespace from path
final Pair<String, String> pathTuple = StorageUtils.splitNamespacePath(rootCorpus);
if (pathTuple == null) {
Logger.error(this.getCtx(), TAG, "fileStatusCheckAsync", this.getAtCorpusPath(), CdmLogCode.ErrStorageNullCorpusPath);
HashMap<String, CdmFileMetadata> fileInfoList = null;
try {
// Remove namespace from path
final Pair<String, String> pathTuple = StorageUtils.splitNamespacePath(rootCorpus);
if (pathTuple == null) {
Logger.error(this.getCtx(), TAG, "fileStatusCheckAsync", this.getAtCorpusPath(), CdmLogCode.ErrStorageNullCorpusPath);
return;
}
nameSpace = pathTuple.getLeft();
adapter = this.getCtx().getCorpus().getStorage().fetchAdapter(nameSpace);
if (adapter == null) {
Logger.error(this.getCtx(), TAG, "fileStatusCheckAsync", this.getAtCorpusPath(), CdmLogCode.ErrDocAdapterNotFound, this.getInDocument().getName());
return;
}
// get a list of all corpusPaths under the root
fileInfoList = adapter.fetchAllFilesMetadataAsync(pathTuple.getRight()).join();
} catch (Exception e) {
Logger.warning(this.getCtx(), TAG, "fileStatusCheckAsync", rootCorpus, CdmLogCode.WarnPartitionFileFetchFailed, rootCorpus, e.getMessage());
if (fileStatusCheckOptions != null && fileStatusCheckOptions.getThrowOnPartitionError()) {
throw new CdmReadPartitionFromPatternException(String.format("There was an error fetching partitions from '%s', see the inner exception.", rootCorpus), e);
}
}
// update modified times
setLastFileStatusCheckTime(OffsetDateTime.now(ZoneOffset.UTC));
if (fileInfoList == null) {
Logger.error(this.getCtx(), TAG, "fileStatusCheckAsync", rootCorpus, CdmLogCode.ErrFetchingFileMetadataNull, nameSpace);
return;
}
nameSpace = pathTuple.getLeft();
adapter = this.getCtx().getCorpus().getStorage().fetchAdapter(nameSpace);
if (adapter == null) {
Logger.error(this.getCtx(), TAG, "fileStatusCheckAsync", this.getAtCorpusPath(), CdmLogCode.ErrDocAdapterNotFound, this.getInDocument().getName());
return;
}
// get a list of all corpusPaths under the root
fileInfoList = adapter.fetchAllFilesMetadataAsync(pathTuple.getRight()).join();
} catch (Exception e) {
Logger.warning(this.getCtx(), TAG, "fileStatusCheckAsync", rootCorpus, CdmLogCode.WarnPartitionFileFetchFailed, rootCorpus, e.getMessage());
}
// update modified times
setLastFileStatusCheckTime(OffsetDateTime.now(ZoneOffset.UTC));
if (fileInfoList == null) {
Logger.error(this.getCtx(), TAG, "fileStatusCheckAsync", rootCorpus, CdmLogCode.ErrFetchingFileMetadataNull, nameSpace);
return;
}
if (nameSpace != null) {
HashMap<String, CdmFileMetadata> cleanedFileList = new HashMap<String, CdmFileMetadata>();
// remove root of the search from the beginning of all paths so anything in the root is not found by regex
for (final Map.Entry<String, CdmFileMetadata> entry : fileInfoList.entrySet()) {
cleanedFileList.put(StringUtils.slice(nameSpace + ":" + entry.getKey(), rootCorpus.length()), entry.getValue());
}
if (this.getOwner() instanceof CdmLocalEntityDeclarationDefinition) {
final CdmLocalEntityDeclarationDefinition localEntDecDefOwner = (CdmLocalEntityDeclarationDefinition) this.getOwner();
// if both are present log warning and use glob pattern, otherwise use regularExpression
if (!StringUtils.isNullOrTrimEmpty(this.getGlobPattern()) && !StringUtils.isNullOrTrimEmpty(this.getRegularExpression())) {
Logger.warning(this.getCtx(), TAG,
"fileStatusCheckAsync",
rootCorpus, CdmLogCode.WarnPartitionGlobAndRegexPresent,
this.getGlobPattern(), this.getRegularExpression());
}
String regularExpression = !StringUtils.isNullOrTrimEmpty(this.globPattern) ? this.globPatternToRegex(this.globPattern) : this.regularExpression;
Pattern regexPattern = null;
try {
regexPattern = Pattern.compile(regularExpression);
} catch (final PatternSyntaxException e) {
Logger.error(this.getCtx(), TAG,
"fileStatusCheckAsync",
rootCorpus, CdmLogCode.ErrValdnInvalidExpression, !StringUtils.isNullOrTrimEmpty(this.globPattern) ? "glob pattern" : "regular expression",
!StringUtils.isNullOrTrimEmpty(this.globPattern) ? this.globPattern : this.regularExpression, e.getMessage());
if (nameSpace != null) {
HashMap<String, CdmFileMetadata> cleanedFileList = new HashMap<String, CdmFileMetadata>();
// remove root of the search from the beginning of all paths so anything in the root is not found by regex
for (final Map.Entry<String, CdmFileMetadata> entry : fileInfoList.entrySet()) {
cleanedFileList.put(StringUtils.slice(nameSpace + ":" + entry.getKey(), rootCorpus.length()), entry.getValue());
}
if (regexPattern != null) {
// a hashset to check if the data partition exists
HashSet<String> dataPartitionPathHashSet = new HashSet<>();
if (localEntDecDefOwner.getDataPartitions() != null) {
for (final CdmDataPartitionDefinition dataPartition : localEntDecDefOwner.getDataPartitions()) {
final String fullPath = this.getCtx().getCorpus().getStorage().createAbsoluteCorpusPath(dataPartition.getLocation(), this.getInDocument());
dataPartitionPathHashSet.add(fullPath);
}
if (this.getOwner() instanceof CdmLocalEntityDeclarationDefinition) {
final CdmLocalEntityDeclarationDefinition localEntDecDefOwner = (CdmLocalEntityDeclarationDefinition) this.getOwner();
// if both are present log warning and use glob pattern, otherwise use regularExpression
if (!StringUtils.isNullOrTrimEmpty(this.getGlobPattern()) && !StringUtils.isNullOrTrimEmpty(this.getRegularExpression())) {
Logger.warning(this.getCtx(), TAG,
"fileStatusCheckAsync",
rootCorpus, CdmLogCode.WarnPartitionGlobAndRegexPresent,
this.getGlobPattern(), this.getRegularExpression());
}
String regularExpression = !StringUtils.isNullOrTrimEmpty(this.globPattern) ? this.globPatternToRegex(this.globPattern) : this.regularExpression;
Pattern regexPattern = null;
try {
regexPattern = Pattern.compile(regularExpression);
} catch (final PatternSyntaxException e) {
Logger.error(this.getCtx(), TAG,
"fileStatusCheckAsync",
rootCorpus, CdmLogCode.ErrValdnInvalidExpression, !StringUtils.isNullOrTrimEmpty(this.globPattern) ? "glob pattern" : "regular expression",
!StringUtils.isNullOrTrimEmpty(this.globPattern) ? this.globPattern : this.regularExpression, e.getMessage());
}
HashSet<String> incrementalPartitionPathHashSet = new HashSet<>();
if (localEntDecDefOwner.getIncrementalPartitions() != null) {
for (final CdmDataPartitionDefinition incrementalPartition : localEntDecDefOwner.getIncrementalPartitions()) {
final String fullPath = this.getCtx().getCorpus().getStorage().createAbsoluteCorpusPath(incrementalPartition.getLocation(), this.getInDocument());
incrementalPartitionPathHashSet.add(fullPath);
}
}
for (final Map.Entry<String, CdmFileMetadata> fi : cleanedFileList.entrySet()) {
final String fileName = fi.getKey();
final CdmFileMetadata partitionMetadata = fi.getValue();
final Matcher m = regexPattern.matcher(fileName);
if (m.matches() && m.group().equals(fileName)) {
// create a map of arguments out of capture groups
final Map<String, List<String>> args = new LinkedHashMap<>();
// For each capture group, save the matching substring into the parameter.
for (int i = 0; i < m.groupCount(); i++) {
if (this.getParameters() != null && i < this.getParameters().size()) {
final String currentParam = this.getParameters().get(i);
if (!args.containsKey(currentParam)) {
args.put(currentParam, new ArrayList<>());
}
args.get(currentParam).add(m.group(i + 1));
}
}
// put the original but cleaned up root back onto the matched doc as the location stored in the partition
final String locationCorpusPath = rootCleaned + fileName;
final String fullPath = rootCorpus + fileName;
// Remove namespace from path
final Pair<String, String> pathTuple = StorageUtils.splitNamespacePath(fullPath);
if (pathTuple == null) {
Logger.error(this.getCtx(), TAG, "fileStatusCheckAsync", rootCorpus, CdmLogCode.ErrStorageNullCorpusPath, this.getAtCorpusPath());
return;
}
CdmTraitCollection exhibitsTraits = this.getExhibitsTraits();
if (fileStatusCheckOptions != null && fileStatusCheckOptions.getIncludeDataPartitionSize() && partitionMetadata != null) {
exhibitsTraits = new CdmTraitCollection(this.getCtx(), this);
for (final CdmTraitReferenceBase trait : this.getExhibitsTraits()) {
exhibitsTraits.add(trait);
}
exhibitsTraits.add("is.partition.size", new ArrayList<>(Collections.singletonList(new ImmutablePair<>("value", partitionMetadata.getSize()))));
}
final OffsetDateTime lastModifiedTime =
adapter.computeLastModifiedTimeAsync(pathTuple.getRight()).join();
if (this.isIncremental() && !incrementalPartitionPathHashSet.contains(fullPath)) {
localEntDecDefOwner.createDataPartitionFromPattern(locationCorpusPath, exhibitsTraits, args, this.getSpecializedSchema(), lastModifiedTime, true, this.getName());
incrementalPartitionPathHashSet.add(fullPath);
} else if (!this.isIncremental() && !dataPartitionPathHashSet.contains(fullPath)) {
localEntDecDefOwner.createDataPartitionFromPattern(
locationCorpusPath, exhibitsTraits, args, getSpecializedSchema(), lastModifiedTime);
if (regexPattern != null) {
// a hashset to check if the data partition exists
HashSet<String> dataPartitionPathHashSet = new HashSet<>();
if (localEntDecDefOwner.getDataPartitions() != null) {
for (final CdmDataPartitionDefinition dataPartition : localEntDecDefOwner.getDataPartitions()) {
final String fullPath = this.getCtx().getCorpus().getStorage().createAbsoluteCorpusPath(dataPartition.getLocation(), this.getInDocument());
dataPartitionPathHashSet.add(fullPath);
}
}
HashSet<String> incrementalPartitionPathHashSet = new HashSet<>();
if (localEntDecDefOwner.getIncrementalPartitions() != null) {
for (final CdmDataPartitionDefinition incrementalPartition : localEntDecDefOwner.getIncrementalPartitions()) {
final String fullPath = this.getCtx().getCorpus().getStorage().createAbsoluteCorpusPath(incrementalPartition.getLocation(), this.getInDocument());
incrementalPartitionPathHashSet.add(fullPath);
}
}
for (final Map.Entry<String, CdmFileMetadata> fi : cleanedFileList.entrySet()) {
final String fileName = fi.getKey();
final CdmFileMetadata partitionMetadata = fi.getValue();
final Matcher m = regexPattern.matcher(fileName);
if (m.matches() && m.group().equals(fileName)) {
// create a map of arguments out of capture groups
final Map<String, List<String>> args = new LinkedHashMap<>();
// For each capture group, save the matching substring into the parameter.
for (int i = 0; i < m.groupCount(); i++) {
if (this.getParameters() != null && i < this.getParameters().size()) {
final String currentParam = this.getParameters().get(i);
if (!args.containsKey(currentParam)) {
args.put(currentParam, new ArrayList<>());
}
args.get(currentParam).add(m.group(i + 1));
}
}
// put the original but cleaned up root back onto the matched doc as the location stored in the partition
final String locationCorpusPath = rootCleaned + fileName;
final String fullPath = rootCorpus + fileName;
// Remove namespace from path
final Pair<String, String> pathTuple = StorageUtils.splitNamespacePath(fullPath);
if (pathTuple == null) {
Logger.error(this.getCtx(), TAG, "fileStatusCheckAsync", rootCorpus, CdmLogCode.ErrStorageNullCorpusPath, this.getAtCorpusPath());
return;
}
CdmTraitCollection exhibitsTraits = this.getExhibitsTraits();
if (fileStatusCheckOptions != null && fileStatusCheckOptions.getIncludeDataPartitionSize() && partitionMetadata != null) {
exhibitsTraits = new CdmTraitCollection(this.getCtx(), this);
for (final CdmTraitReferenceBase trait : this.getExhibitsTraits()) {
exhibitsTraits.add(trait);
}
exhibitsTraits.add("is.partition.size", new ArrayList<>(Collections.singletonList(new ImmutablePair<>("value", partitionMetadata.getSize()))));
}
final OffsetDateTime lastModifiedTime =
adapter.computeLastModifiedTimeAsync(pathTuple.getRight()).join();
if (this.isIncremental() && !incrementalPartitionPathHashSet.contains(fullPath)) {
localEntDecDefOwner.createDataPartitionFromPattern(locationCorpusPath, exhibitsTraits, args, this.getSpecializedSchema(), lastModifiedTime, true, this.getName());
incrementalPartitionPathHashSet.add(fullPath);
} else if (!this.isIncremental() && !dataPartitionPathHashSet.contains(fullPath)) {
localEntDecDefOwner.createDataPartitionFromPattern(
locationCorpusPath, exhibitsTraits, args, getSpecializedSchema(), lastModifiedTime);
dataPartitionPathHashSet.add(fullPath);
}
}
}
}
}
}
}
}
});
});
}
@Override
public CompletableFuture<Void> fileStatusCheckAsync() {
public CompletableFuture<Void> fileStatusCheckAsync() throws CdmReadPartitionFromPatternException {
return this.fileStatusCheckAsync(null);
}

5
objectModel/Java/objectmodel/src/main/java/com/microsoft/commondatamodel/objectmodel/cdm/CdmEntityAttributeDefinition.java
Просмотреть файл

@ -354,7 +354,9 @@ public class CdmEntityAttributeDefinition extends CdmAttribute {
// A Projection
// if the max depth is exceeded it should not try to execute the projection
if (!resOpt.depthInfo.getMaxDepthExceeded()) {
if (resOpt.depthInfo.getMaxDepthExceeded()) {
Logger.warning(this.getCtx(), TAG, "constructResolvedAttributes", this.getAtCorpusPath(), CdmLogCode.WarnMaxDepthExceeded, resOpt.depthInfo.getMaxDepth() != null ? resOpt.depthInfo.getMaxDepth().toString() : "", this.entity.fetchObjectDefinitionName());
} else {
CdmProjection projDef = this.getEntity().fetchObjectDefinition(resOpt);
ProjectionDirective projDirective = new ProjectionDirective(resOpt, this, this.getEntity());
ProjectionContext projCtx = projDef.constructProjectionContext(projDirective, under);
@ -432,6 +434,7 @@ public class CdmEntityAttributeDefinition extends CdmAttribute {
// if we got here because of the max depth, need to impose the directives to make the trait work as expected
if (resOpt.depthInfo.getMaxDepthExceeded()) {
Logger.warning(this.getCtx(), TAG, "constructResolvedAttributes", this.getAtCorpusPath(), CdmLogCode.WarnMaxDepthExceeded, resOpt.depthInfo.getMaxDepth() != null ? resOpt.depthInfo.getMaxDepth().toString() : "", this.entity.fetchObjectDefinitionName());
if (arc.getResOpt().getDirectives() == null) {
arc.getResOpt().setDirectives(new AttributeResolutionDirectiveSet());
}

4
objectModel/Java/objectmodel/src/main/java/com/microsoft/commondatamodel/objectmodel/cdm/CdmFileStatus.java
Просмотреть файл

@ -6,6 +6,8 @@ package com.microsoft.commondatamodel.objectmodel.cdm;
import java.time.OffsetDateTime;
import java.util.concurrent.CompletableFuture;
import com.microsoft.commondatamodel.objectmodel.utilities.exceptions.CdmReadPartitionFromPatternException;
public interface CdmFileStatus extends CdmObject {
/**
@ -36,7 +38,7 @@ public interface CdmFileStatus extends CdmObject {
* Updates the object and any children with changes made in the document file where it came from.
* @return OffsetDateTime
*/
CompletableFuture<Void> fileStatusCheckAsync();
CompletableFuture<Void> fileStatusCheckAsync() throws CdmReadPartitionFromPatternException;
/**
* Report most recent modified time (of current or children objects) to the parent object.

21
objectModel/Java/objectmodel/src/main/java/com/microsoft/commondatamodel/objectmodel/cdm/CdmLocalEntityDeclarationDefinition.java
Просмотреть файл

@ -12,7 +12,9 @@ import com.microsoft.commondatamodel.objectmodel.utilities.ResolveOptions;
import com.microsoft.commondatamodel.objectmodel.utilities.StringUtils;
import com.microsoft.commondatamodel.objectmodel.utilities.TimeUtils;
import com.microsoft.commondatamodel.objectmodel.utilities.VisitCallback;
import com.microsoft.commondatamodel.objectmodel.utilities.exceptions.CdmReadPartitionFromPatternException;
import com.microsoft.commondatamodel.objectmodel.utilities.logger.Logger;
import jdk.jshell.spi.ExecutionControlProvider;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
@ -21,6 +23,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.stream.Collectors;
public class CdmLocalEntityDeclarationDefinition extends CdmObjectDefinitionBase implements
@ -252,23 +255,23 @@ public class CdmLocalEntityDeclarationDefinition extends CdmObjectDefinitionBase
}
@Override
public CompletableFuture<Void> fileStatusCheckAsync() {
public CompletableFuture<Void> fileStatusCheckAsync() throws CdmReadPartitionFromPatternException {
return fileStatusCheckAsync(PartitionFileStatusCheckType.Full);
}
public CompletableFuture<Void> fileStatusCheckAsync(final PartitionFileStatusCheckType partitionFileStatusCheckType) {
public CompletableFuture<Void> fileStatusCheckAsync(final PartitionFileStatusCheckType partitionFileStatusCheckType) throws CdmReadPartitionFromPatternException {
return fileStatusCheckAsync(partitionFileStatusCheckType, CdmIncrementalPartitionType.None);
}
public CompletableFuture<Void> fileStatusCheckAsync(final PartitionFileStatusCheckType partitionFileStatusCheckType, final CdmIncrementalPartitionType incrementalType) {
public CompletableFuture<Void> fileStatusCheckAsync(final PartitionFileStatusCheckType partitionFileStatusCheckType, final CdmIncrementalPartitionType incrementalType) throws CdmReadPartitionFromPatternException {
return fileStatusCheckAsync(partitionFileStatusCheckType, incrementalType, null);
}
public CompletableFuture<Void> fileStatusCheckAsync(final PartitionFileStatusCheckType partitionFileStatusCheckType, final CdmIncrementalPartitionType incrementalType, final FileStatusCheckOptions fileStatusCheckOptions) {
public CompletableFuture<Void> fileStatusCheckAsync(final PartitionFileStatusCheckType partitionFileStatusCheckType, final CdmIncrementalPartitionType incrementalType, final FileStatusCheckOptions fileStatusCheckOptions) throws CdmReadPartitionFromPatternException {
return CompletableFuture.runAsync(() -> {
StorageAdapterBase adapter = this.getCtx().getCorpus().getStorage().fetchAdapter(this.getInDocument().getNamespace());
StorageAdapterBase.CacheContext cacheContext = null;
if(adapter != null) {
if (adapter != null) {
cacheContext = adapter.createFileQueryCacheContext();
}
try {
@ -295,7 +298,7 @@ public class CdmLocalEntityDeclarationDefinition extends CdmObjectDefinitionBase
Logger.error(partition.getCtx(), TAG, "fileStatusCheckAsync", partition.getAtCorpusPath(), CdmLogCode.ErrUnexpectedIncrementalPartitionTrait,
partition.getClass().getSimpleName(), partition.fetchObjectDefinitionName(), Constants.IncrementalTraitName, "dataPartitions");
} else {
partition.fileStatusCheckAsync().join();
partition.fileStatusCheckAsync(fileStatusCheckOptions).join();
}
}
}
@ -318,10 +321,8 @@ public class CdmLocalEntityDeclarationDefinition extends CdmObjectDefinitionBase
setLastFileModifiedTime(TimeUtils.maxTime(modifiedTime, getLastFileModifiedTime()));
reportMostRecentTimeAsync(getLastFileModifiedTime()).join();
}
finally {
if(cacheContext != null)
{
} finally {
if (cacheContext != null) {
cacheContext.dispose();
}
}

45
objectModel/Java/objectmodel/src/main/java/com/microsoft/commondatamodel/objectmodel/cdm/CdmManifestDefinition.java
Просмотреть файл

@ -12,6 +12,7 @@ import com.microsoft.commondatamodel.objectmodel.enums.ImportsLoadStrategy;
import com.microsoft.commondatamodel.objectmodel.enums.CdmIncrementalPartitionType;
import com.microsoft.commondatamodel.objectmodel.enums.PartitionFileStatusCheckType;
import com.microsoft.commondatamodel.objectmodel.utilities.*;
import com.microsoft.commondatamodel.objectmodel.utilities.exceptions.CdmReadPartitionFromPatternException;
import com.microsoft.commondatamodel.objectmodel.utilities.logger.Logger;
import java.time.OffsetDateTime;
@ -23,6 +24,7 @@ import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.lang3.ObjectUtils;
@ -456,19 +458,19 @@ public class CdmManifestDefinition extends CdmDocumentDefinition implements CdmO
}
@Override
public CompletableFuture<Void> fileStatusCheckAsync() {
public CompletableFuture<Void> fileStatusCheckAsync() throws CdmReadPartitionFromPatternException {
return fileStatusCheckAsync(PartitionFileStatusCheckType.Full);
}
public CompletableFuture<Void> fileStatusCheckAsync(PartitionFileStatusCheckType partitionFileStatusCheckType) {
public CompletableFuture<Void> fileStatusCheckAsync(PartitionFileStatusCheckType partitionFileStatusCheckType) throws CdmReadPartitionFromPatternException {
return fileStatusCheckAsync(partitionFileStatusCheckType, CdmIncrementalPartitionType.None);
}
public CompletableFuture<Void> fileStatusCheckAsync(PartitionFileStatusCheckType partitionFileStatusCheckType, final CdmIncrementalPartitionType incrementalType) {
public CompletableFuture<Void> fileStatusCheckAsync(PartitionFileStatusCheckType partitionFileStatusCheckType, final CdmIncrementalPartitionType incrementalType) throws CdmReadPartitionFromPatternException {
return fileStatusCheckAsync(partitionFileStatusCheckType, incrementalType, null);
}
public CompletableFuture<Void> fileStatusCheckAsync(final PartitionFileStatusCheckType partitionFileStatusCheckType, final CdmIncrementalPartitionType incrementalType, final FileStatusCheckOptions fileStatusCheckOptions) {
public CompletableFuture<Void> fileStatusCheckAsync(final PartitionFileStatusCheckType partitionFileStatusCheckType, final CdmIncrementalPartitionType incrementalType, final FileStatusCheckOptions fileStatusCheckOptions) throws CdmReadPartitionFromPatternException {
return CompletableFuture.runAsync(() -> {
try (Logger.LoggerScope logScope = Logger.enterScope(CdmManifestDefinition.class.getSimpleName(), getCtx(), "fileStatusCheckAsync")) {
StorageAdapterBase adapter = this.getCtx().getCorpus().getStorage().fetchAdapter(this.getInDocument().getNamespace());
@ -603,6 +605,23 @@ public class CdmManifestDefinition extends CdmDocumentDefinition implements CdmO
return copy;
}
/**
* Creates a resolved copy of the manifest.
* newEntityDocumentNameFormat specifies a pattern to use when creating documents for resolved entities.
* The default is "{f}resolved/{n}.cdm.json" to avoid a document name conflict with documents in the same folder as the manifest.
* Every instance of the string {n} is replaced with the entity name from the source manifest.
* Every instance of the string {f} is replaced with the folder path from the source manifest to the source entity
* (if there is one that is possible as a relative location, else nothing).
* @param newManifestName name string
* @param newEntityDocumentNameFormat format string
* @return CompletableFuture
*/
public CompletableFuture<CdmManifestDefinition> createResolvedManifestAsync(
String newManifestName,
String newEntityDocumentNameFormat) {
return createResolvedManifestAsync(newManifestName, newEntityDocumentNameFormat, null);
}
/**
* Creates a resolved copy of the manifest.
* newEntityDocumentNameFormat specifies a pattern to use when creating documents for resolved entities.
@ -616,8 +635,9 @@ public class CdmManifestDefinition extends CdmDocumentDefinition implements CdmO
*/
public CompletableFuture<CdmManifestDefinition> createResolvedManifestAsync(
String newManifestName,
String newEntityDocumentNameFormat) {
return createResolvedManifestAsync(newManifestName, newEntityDocumentNameFormat, null);
String newEntityDocumentNameFormat,
final AttributeResolutionDirectiveSet directives) {
return createResolvedManifestAsync(newManifestName, newEntityDocumentNameFormat, null, null);
}
/**
@ -635,7 +655,8 @@ public class CdmManifestDefinition extends CdmDocumentDefinition implements CdmO
public CompletableFuture<CdmManifestDefinition> createResolvedManifestAsync(
final String newManifestName,
final String newEntityDocumentNameFormat,
final AttributeResolutionDirectiveSet directives) {
final AttributeResolutionDirectiveSet directives,
final ResolveOptions resOpt) {
return CompletableFuture.supplyAsync(() -> {
try (Logger.LoggerScope logScope = Logger.enterScope(CdmManifestDefinition.class.getSimpleName(), getCtx(), "createResolvedManifestAsync")) {
@ -778,18 +799,20 @@ public class CdmManifestDefinition extends CdmDocumentDefinition implements CdmO
// Next create the resolved entity.
AttributeResolutionDirectiveSet withDirectives =
directives != null ? directives : this.getCtx().getCorpus().getDefaultResolutionDirectives();
final ResolveOptions resOpt = new ResolveOptions(entDef.getInDocument(), withDirectives != null ? withDirectives.copy() : null);
final ResolveOptions resOptEntity = resOpt != null ? resOpt.copy() : new ResolveOptions();
resOptEntity.setWrtDoc(entDef.getInDocument());
resOptEntity.setDirectives(withDirectives != null ? withDirectives.copy() : null);
Logger.debug(this.getCtx(), TAG, "createResolvedManifestAsync", this.getAtCorpusPath(), Logger.format("resolving entity {0} to document {1}", sourceEntityFullPath, newDocumentFullPath));
final CdmEntityDefinition resolvedEntity = entDef
.createResolvedEntityAsync(entDef.getEntityName(), resOpt, folder, newDocumentName).join();
.createResolvedEntityAsync(entDef.getEntityName(), resOptEntity, folder, newDocumentName).join();
if (null == resolvedEntity) {
// Fail all resolution, if any one entity resolution fails.
return null;
}
CdmEntityDeclarationDefinition result = (CdmEntityDeclarationDefinition) entity.copy(resOpt);
CdmEntityDeclarationDefinition result = (CdmEntityDeclarationDefinition) entity.copy(resOptEntity);
if (result.getObjectType() == CdmObjectType.LocalEntityDeclarationDef) {
result.setEntityPath(
ObjectUtils.firstNonNull(
@ -806,7 +829,7 @@ public class CdmManifestDefinition extends CdmDocumentDefinition implements CdmO
Logger.debug(this.getCtx(), TAG, "createResolvedManifestAsync", this.getAtCorpusPath(), "calculating relationships");
// Calculate the entity graph for just this folio and any subManifests.
this.getCtx().getCorpus().calculateEntityGraphAsync(resolvedManifest).join();
this.getCtx().getCorpus().calculateEntityGraphAsync(resolvedManifest, resOpt).join();
// Stick results into the relationships list for the manifest.
// Only put in relationships that are between the entities that are used in the manifest.
resolvedManifest.populateManifestRelationshipsAsync(

4
objectModel/Java/objectmodel/src/main/java/com/microsoft/commondatamodel/objectmodel/cdm/CdmObjectBase.java
Просмотреть файл

@ -3,6 +3,7 @@
package com.microsoft.commondatamodel.objectmodel.cdm;
import com.microsoft.commondatamodel.objectmodel.enums.CdmLogCode;
import com.microsoft.commondatamodel.objectmodel.enums.CdmObjectType;
import com.microsoft.commondatamodel.objectmodel.persistence.PersistenceLayer;
import com.microsoft.commondatamodel.objectmodel.resolvedmodel.ResolveContext;
@ -19,10 +20,12 @@ import com.microsoft.commondatamodel.objectmodel.utilities.ResolveOptions;
import com.microsoft.commondatamodel.objectmodel.utilities.StringUtils;
import com.microsoft.commondatamodel.objectmodel.utilities.SymbolSet;
import com.microsoft.commondatamodel.objectmodel.utilities.VisitCallback;
import com.microsoft.commondatamodel.objectmodel.utilities.logger.Logger;
import java.util.*;
public abstract class CdmObjectBase implements CdmObject {
private static final String TAG = CdmEntityAttributeDefinition.class.getSimpleName();
/**
* The minimum json semantic versions that can be loaded by this ObjectModel version.
@ -622,6 +625,7 @@ public abstract class CdmObjectBase implements CdmObject {
// if using the cache passes the maxDepth, we cannot use it
if (rasbCache != null && resOpt.depthInfo.getMaxDepth() != null
&& resOpt.depthInfo.getCurrentDepth() + rasbCache.getResolvedAttributeSet().getDepthTraveled() > resOpt.depthInfo.getMaxDepth()) {
Logger.warning(this.getCtx(), TAG, "fetchResolvedAttributes", this.getAtCorpusPath(), CdmLogCode.WarnMaxDepthExceeded, resOpt.depthInfo.getMaxDepth() != null ? resOpt.depthInfo.getMaxDepth().toString() : "", this.fetchObjectDefinitionName());
rasbCache = null;
}

2
objectModel/Java/objectmodel/src/main/java/com/microsoft/commondatamodel/objectmodel/cdm/CdmTraitDefinition.java
Просмотреть файл

@ -12,7 +12,7 @@ import com.microsoft.commondatamodel.objectmodel.resolvedmodel.ResolvedAttribute
import com.microsoft.commondatamodel.objectmodel.resolvedmodel.ResolvedTrait;
import com.microsoft.commondatamodel.objectmodel.resolvedmodel.ResolvedTraitSet;
import com.microsoft.commondatamodel.objectmodel.resolvedmodel.ResolvedTraitSetBuilder;
import com.microsoft.commondatamodel.objectmodel.utilities.CdmException;
import com.microsoft.commondatamodel.objectmodel.utilities.exceptions.CdmException;
import com.microsoft.commondatamodel.objectmodel.utilities.CopyOptions;
import com.microsoft.commondatamodel.objectmodel.utilities.ResolveOptions;
import com.microsoft.commondatamodel.objectmodel.utilities.StringUtils;

2
objectModel/Java/objectmodel/src/main/java/com/microsoft/commondatamodel/objectmodel/cdm/CdmTraitReference.java
Просмотреть файл

@ -10,7 +10,7 @@ import com.microsoft.commondatamodel.objectmodel.resolvedmodel.ResolvedAttribute
import com.microsoft.commondatamodel.objectmodel.resolvedmodel.ResolvedTrait;
import com.microsoft.commondatamodel.objectmodel.resolvedmodel.ResolvedTraitSet;
import com.microsoft.commondatamodel.objectmodel.resolvedmodel.ResolvedTraitSetBuilder;
import com.microsoft.commondatamodel.objectmodel.utilities.CdmException;
import com.microsoft.commondatamodel.objectmodel.utilities.exceptions.CdmException;
import com.microsoft.commondatamodel.objectmodel.utilities.CopyOptions;
import com.microsoft.commondatamodel.objectmodel.utilities.ResolveOptions;
import com.microsoft.commondatamodel.objectmodel.utilities.StringUtils;

2
objectModel/Java/objectmodel/src/main/java/com/microsoft/commondatamodel/objectmodel/enums/CdmLogCode.java
Просмотреть файл

@ -30,6 +30,7 @@ public enum CdmLogCode
ErrManifestFileModTimeFailure,
ErrMissingIncrementalPartitionTrait,
ErrObjectWithoutOwnerFound,
ErrPartitionFileMetadataFailure,
ErrPartitionFileModTimeFailure,
ErrPathIsDuplicate,
ErrPathNullObjectPath,
@ -134,6 +135,7 @@ public enum CdmLogCode
WarnDeprecatedResolutionGuidance,
WarnDocChangesDiscarded,
WarnDocImportNotLoaded,
WarnMaxDepthExceeded,
WarnPartitionFileFetchFailed,
WarnLinkEntIdentArgsNotSupported,
WarnPartitionInvalidArguments,

2
objectModel/Java/objectmodel/src/main/java/com/microsoft/commondatamodel/objectmodel/resolvedmodel/ParameterCollection.java
Просмотреть файл

@ -4,7 +4,7 @@
package com.microsoft.commondatamodel.objectmodel.resolvedmodel;
import com.microsoft.commondatamodel.objectmodel.cdm.CdmParameterDefinition;
import com.microsoft.commondatamodel.objectmodel.utilities.CdmException;
import com.microsoft.commondatamodel.objectmodel.utilities.exceptions.CdmException;
import com.microsoft.commondatamodel.objectmodel.utilities.StringUtils;
import java.util.ArrayList;

35
objectModel/Java/objectmodel/src/main/java/com/microsoft/commondatamodel/objectmodel/storage/AdlsAdapter.java
Просмотреть файл

@ -99,7 +99,7 @@ public class AdlsAdapter extends NetworkAdapter {
*/
private int httpMaxResults = 5000;
private Map<String, OffsetDateTime> fileModifiedTimeCache = new LinkedHashMap<String, OffsetDateTime>();
private Map<String, CdmFileMetadata> fileMetadataCache = new LinkedHashMap<String, CdmFileMetadata>();
private AdlsAdapterAuthenticator adlsAdapterAuthenticator;
@ -273,18 +273,28 @@ public class AdlsAdapter extends NetworkAdapter {
@Override
public void clearCache() {
this.fileModifiedTimeCache.clear();
this.fileMetadataCache.clear();
}
@Override
public CompletableFuture<OffsetDateTime> computeLastModifiedTimeAsync(final String corpusPath) {
public CompletableFuture<OffsetDateTime> computeLastModifiedTimeAsync(final String corpusPath) {
final CdmFileMetadata fileMetadata = this.fetchFileMetadataAsync(corpusPath).join();
if (fileMetadata == null) {
return CompletableFuture.completedFuture(null);
}
return CompletableFuture.completedFuture(fileMetadata.getLastModifiedTime());
}
@Override
public CompletableFuture<CdmFileMetadata> fetchFileMetadataAsync(final String corpusPath) {
return CompletableFuture.supplyAsync(() -> {
OffsetDateTime cachedValue = this.getIsCacheEnabled() ? this.fileModifiedTimeCache.get(corpusPath) : null;
CdmFileMetadata cachedValue = this.getIsCacheEnabled() ? this.fileMetadataCache.get(corpusPath) : null;
if(cachedValue != null)
{
return cachedValue;
}
else{
} else{
String url = this.createFormattedAdapterPath(corpusPath);
final CdmHttpRequest request = this.buildRequest(url, "HEAD");
@ -295,10 +305,12 @@ public class AdlsAdapter extends NetworkAdapter {
DateUtils.parseDate(cdmResponse.getResponseHeaders().get("Last-Modified"))
.toInstant()
.atOffset(ZoneOffset.UTC);
int fileSize = Integer.parseInt(cdmResponse.getResponseHeaders().get("Content-Length"));
CdmFileMetadata fileMetadata = new CdmFileMetadata(lastTime, fileSize);
if(this.getIsCacheEnabled()) {
this.fileModifiedTimeCache.put(corpusPath, lastTime);
this.fileMetadataCache.put(corpusPath, fileMetadata);
}
return lastTime;
return fileMetadata;
}
return null;
@ -384,14 +396,15 @@ public class AdlsAdapter extends NetworkAdapter {
String filepath = this.formatCorpusPath(nameWithoutSubPath);
final JsonNode contentLength = path.get("contentLength");
result.put(filepath, new CdmFileMetadata(contentLength.asLong()));
OffsetDateTime lastTime = DateUtils.parseDate(path.get("lastModified").asText())
.toInstant()
.atOffset(ZoneOffset.UTC);
final CdmFileMetadata fileMetadata = new CdmFileMetadata(lastTime, contentLength.asLong());
result.put(filepath, fileMetadata);
if (this.getIsCacheEnabled()) {
this.fileModifiedTimeCache.put(filepath, lastTime);
this.fileMetadataCache.put(filepath, fileMetadata);
}
}
}

4
objectModel/Java/objectmodel/src/main/java/com/microsoft/commondatamodel/objectmodel/storage/CdmStandardsAdapter.java
Просмотреть файл

@ -16,4 +16,8 @@ public class CdmStandardsAdapter extends CdmCustomPackageAdapter {
public CdmStandardsAdapter() throws ClassNotFoundException {
super("com.microsoft.commondatamodel.cdmstandards.CdmStandards");
}
public String fetchConfig() {
return "{\"config\":{},\"type\":\"cdm-standards\"}";
}
}

18
objectModel/Java/objectmodel/src/main/java/com/microsoft/commondatamodel/objectmodel/storage/LocalAdapter.java
Просмотреть файл

@ -166,13 +166,25 @@ public class LocalAdapter extends StorageAdapterBase {
@Override
public CompletableFuture<OffsetDateTime> computeLastModifiedTimeAsync(final String corpusPath) {
final CdmFileMetadata fileMetadata = this.fetchFileMetadataAsync(corpusPath).join();
if (fileMetadata == null) {
return CompletableFuture.completedFuture(null);
}
return CompletableFuture.completedFuture(fileMetadata.getLastModifiedTime());
}
@Override
public CompletableFuture<CdmFileMetadata> fetchFileMetadataAsync(final String corpusPath) {
return CompletableFuture.supplyAsync(() -> {
final Path adapterPath = Paths.get(this.createAdapterPath(corpusPath));
if (Files.exists(adapterPath)) {
try {
return OffsetDateTime
final OffsetDateTime lastTime = OffsetDateTime
.ofInstant(Files.getLastModifiedTime(adapterPath).toInstant(), ZoneOffset.UTC);
return new CdmFileMetadata(lastTime, Files.size(adapterPath));
} catch (final IOException e) {
throw new StorageAdapterException(
"Failed to get last modified time of file at adapter path " + corpusPath, e);
@ -225,7 +237,9 @@ public class LocalAdapter extends StorageAdapterBase {
Path path = Paths.get(this.createAdapterPath(fileName));
if (Files.exists(path)) {
try {
fileMetadatas.put(fileName, new CdmFileMetadata(Files.size(path)));
final OffsetDateTime lastTime = OffsetDateTime
.ofInstant(Files.getLastModifiedTime(path).toInstant(), ZoneOffset.UTC);
fileMetadatas.put(fileName, new CdmFileMetadata(lastTime, Files.size(path)));
} catch (IOException e) {
}
}

12
objectModel/Java/objectmodel/src/main/java/com/microsoft/commondatamodel/objectmodel/storage/StorageAdapterBase.java
Просмотреть файл

@ -5,6 +5,7 @@ package com.microsoft.commondatamodel.objectmodel.storage;
import com.microsoft.commondatamodel.objectmodel.cdm.CdmCorpusContext;
import com.microsoft.commondatamodel.objectmodel.utilities.CdmFileMetadata;
import com.microsoft.commondatamodel.objectmodel.utilities.exceptions.CdmReadPartitionFromPatternException;
import java.io.IOException;
import java.time.OffsetDateTime;
@ -127,6 +128,15 @@ public abstract class StorageAdapterBase {
return CompletableFuture.completedFuture(OffsetDateTime.now());
}
/**
* Returns the file metadata info about the specified document.
* @param corpusPath The path to the document.
* @return the Cdm File Metadata of the document
*/
public CompletableFuture<CdmFileMetadata> fetchFileMetadataAsync(String corpusPath) {
return CompletableFuture.completedFuture(null);
}
/**
* @deprecated Deprecated in favor of fetchAllFilesMetadataAsync
* meant to be called externally at all. Please refrain from using it.
@ -146,7 +156,7 @@ public abstract class StorageAdapterBase {
* @param folderCorpusPath Path to the folder to scan
* @return Dictionary with list of corpus paths as keys and CdmFileMetadata info for each as the value
*/
public CompletableFuture<HashMap<String, CdmFileMetadata>> fetchAllFilesMetadataAsync(String folderCorpusPath) {
public CompletableFuture<HashMap<String, CdmFileMetadata>> fetchAllFilesMetadataAsync(String folderCorpusPath) throws CdmReadPartitionFromPatternException {
List<String> allFiles = this.fetchAllFilesAsync(folderCorpusPath).join();
HashMap<String, CdmFileMetadata> filesMetadata = new HashMap<String, CdmFileMetadata>();

10
objectModel/Java/objectmodel/src/main/java/com/microsoft/commondatamodel/objectmodel/utilities/CdmFileMetadata.java
Просмотреть файл

@ -3,10 +3,18 @@
package com.microsoft.commondatamodel.objectmodel.utilities;
import java.time.OffsetDateTime;
public class CdmFileMetadata {
private long size;
private OffsetDateTime lastModifiedTime;
public CdmFileMetadata(long size) { this.size = size; }
public CdmFileMetadata(OffsetDateTime lastModifiedTime, long size) {
this.lastModifiedTime = lastModifiedTime;
this.size = size;
}
public final OffsetDateTime getLastModifiedTime() { return this.lastModifiedTime; }
public final long getSize() { return this.size; }
}

14
objectModel/Java/objectmodel/src/main/java/com/microsoft/commondatamodel/objectmodel/utilities/FileStatusCheckOptions.java
Просмотреть файл

@ -6,7 +6,19 @@ package com.microsoft.commondatamodel.objectmodel.utilities;
public class FileStatusCheckOptions {
private boolean includeDataPartitionSize;
public FileStatusCheckOptions(boolean includeDataPartitionSize) { this.includeDataPartitionSize = includeDataPartitionSize; }
public boolean throwOnPartitionError;
public FileStatusCheckOptions(boolean includeDataPartitionSize) {
this.includeDataPartitionSize = includeDataPartitionSize;
this.throwOnPartitionError = false;
}
public FileStatusCheckOptions(boolean includeDataPartitionSize, boolean throwOnPartitionError) {
this.includeDataPartitionSize = includeDataPartitionSize;
this.throwOnPartitionError = throwOnPartitionError;
}
public boolean getIncludeDataPartitionSize() { return this.includeDataPartitionSize; }
public boolean getThrowOnPartitionError() { return this.throwOnPartitionError; }
}

1
objectModel/Java/objectmodel/src/main/java/com/microsoft/commondatamodel/objectmodel/utilities/ResolveOptions.java
Просмотреть файл

@ -302,6 +302,7 @@ public class ResolveOptions {
final ResolveOptions resOptCopy = new ResolveOptions();
resOptCopy.wrtDoc = this.wrtDoc;
resOptCopy.depthInfo = this.depthInfo.copy();
resOptCopy.maxDepth = this.maxDepth;
resOptCopy.localizeReferencesFor = this.localizeReferencesFor;
resOptCopy.indexingDoc = this.indexingDoc;
resOptCopy.shallowValidation = this.shallowValidation;

2
objectModel/Java/objectmodel/src/main/java/com/microsoft/commondatamodel/objectmodel/utilities/CdmException.java → objectModel/Java/objectmodel/src/main/java/com/microsoft/commondatamodel/objectmodel/utilities/exceptions/CdmException.java
Просмотреть файл

@ -1,7 +1,7 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.
package com.microsoft.commondatamodel.objectmodel.utilities;
package com.microsoft.commondatamodel.objectmodel.utilities.exceptions;
public class CdmException extends Exception {
private static final long serialVersionUID = -2327313620869310516L;

17
objectModel/Java/objectmodel/src/main/java/com/microsoft/commondatamodel/objectmodel/utilities/exceptions/CdmReadPartitionFromPatternException.java Normal file
Просмотреть файл

@ -0,0 +1,17 @@
package com.microsoft.commondatamodel.objectmodel.utilities.exceptions;
public class CdmReadPartitionFromPatternException extends RuntimeException {
private static final long serialVersionUID = 1;
public CdmReadPartitionFromPatternException() {
super();
}
public CdmReadPartitionFromPatternException(final String message) {
super(message);
}
public CdmReadPartitionFromPatternException(final String message, final Throwable throwable) {
super(message, throwable);
}
}

5
objectModel/Java/objectmodel/src/main/java/com/microsoft/commondatamodel/objectmodel/utilities/logger/Logger.java
Просмотреть файл

@ -241,7 +241,10 @@ public class Logger {
int i = 0;
for (String x : args) {
String from = "{" + i + "}";
builder = builder.replace(builder.indexOf(from), builder.indexOf(from) + from.length(), x == null ? "" : x);
int index = builder.indexOf(from);
if (index > -1) {
builder = builder.replace(index, index + from.length(), x == null ? "" : x);
}
i++;
}
return builder.toString();

26
objectModel/Java/objectmodel/src/test/java/com/microsoft/commondatamodel/objectmodel/cdm/CorpusTest.java
Просмотреть файл

@ -187,4 +187,30 @@ public class CorpusTest {
corpus.calculateEntityGraphAsync(manifest).join();
TestHelper.assertCdmLogCodeEquality(corpus, CdmLogCode.ErrInvalidCast, true);
}
/**
* Test warning correctly logged when max depth is exceeded for Resolution Guidance
*/
@Test
public void testMaxDepthExceededResolutionGuidance() throws InterruptedException {
final HashSet<CdmLogCode> expectedLogCodes = new HashSet<>(Arrays.asList(CdmLogCode.WarnMaxDepthExceeded ));
final CdmCorpusDefinition corpus = TestHelper.getLocalCorpus(TESTS_SUBPATH, "TestMaxDepthExceededResolutionGuidance", null, false, expectedLogCodes);
final CdmEntityDefinition entity = corpus.<CdmEntityDefinition>fetchObjectAsync("local:/firstEntity.cdm.json/firstEntity").join();
entity.createResolvedEntityAsync("resFirstEntity").join();
TestHelper.assertCdmLogCodeEquality(corpus, CdmLogCode.WarnMaxDepthExceeded, true);
}
/**
* Test warning correctly logged when max depth is exceeded for Projections
*/
@Test
public void testMaxDepthExceededProjections() throws InterruptedException {
final HashSet<CdmLogCode> expectedLogCodes = new HashSet<>(Arrays.asList(CdmLogCode.WarnMaxDepthExceeded ));
final CdmCorpusDefinition corpus = TestHelper.getLocalCorpus(TESTS_SUBPATH, "TestMaxDepthExceededProjections", null, false, expectedLogCodes);
final CdmEntityDefinition entity = corpus.<CdmEntityDefinition>fetchObjectAsync("local:/A.cdm.json/A").join();
entity.createResolvedEntityAsync("resA").join();
TestHelper.assertCdmLogCodeEquality(corpus, CdmLogCode.WarnMaxDepthExceeded, true);
}
}

5
objectModel/Java/objectmodel/src/test/java/com/microsoft/commondatamodel/objectmodel/cdm/ManifestDefinitionTest.java
Просмотреть файл

@ -5,13 +5,14 @@ package com.microsoft.commondatamodel.objectmodel.cdm;
import java.io.File;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import com.microsoft.commondatamodel.objectmodel.ModelJsonUnitTestLocalAdapter;
import com.microsoft.commondatamodel.objectmodel.TestHelper;
import com.microsoft.commondatamodel.objectmodel.persistence.PersistenceLayer;
import com.microsoft.commondatamodel.objectmodel.storage.LocalAdapter;
import com.microsoft.commondatamodel.objectmodel.utilities.exceptions.CdmReadPartitionFromPatternException;
import org.testng.Assert;
import org.testng.annotations.Test;
@ -83,7 +84,7 @@ public class ManifestDefinitionTest {
* Tests if FileStatusCheckAsync() works properly for manifest loaded from model.json
*/
@Test
public void testModelJsonManifestFileStatusCheckAsync() throws InterruptedException {
public void testModelJsonManifestFileStatusCheckAsync() throws CdmReadPartitionFromPatternException, InterruptedException {
CdmCorpusDefinition corpus = TestHelper.getLocalCorpus(TESTS_SUBPATH, "testModelJsonManifestFileStatusCheckAsync");
ModelJsonUnitTestLocalAdapter modeljsonAdapter = new ModelJsonUnitTestLocalAdapter(((LocalAdapter)corpus.getStorage().getNamespaceAdapters().get("local")).getRoot());
corpus.getStorage().mount("modeljson", modeljsonAdapter);

40
objectModel/Java/objectmodel/src/test/java/com/microsoft/commondatamodel/objectmodel/cdm/datapartition/DataPartitionTest.java Normal file
Просмотреть файл

@ -0,0 +1,40 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.
package com.microsoft.commondatamodel.objectmodel.cdm.datapartition;
import com.microsoft.commondatamodel.objectmodel.TestHelper;
import com.microsoft.commondatamodel.objectmodel.cdm.*;
import com.microsoft.commondatamodel.objectmodel.enums.CdmIncrementalPartitionType;
import com.microsoft.commondatamodel.objectmodel.enums.PartitionFileStatusCheckType;
import com.microsoft.commondatamodel.objectmodel.utilities.FileStatusCheckOptions;
import com.microsoft.commondatamodel.objectmodel.utilities.exceptions.CdmReadPartitionFromPatternException;
import org.testng.annotations.Test;
import java.io.File;
import org.testng.Assert;
public class DataPartitionTest {
private static final String CDM = "cdm";
private static final String TESTS_SUBPATH = new File(CDM, "dataPartition").toString();
@Test
public void testRefreshesDataPartition() throws CdmReadPartitionFromPatternException, InterruptedException {
final CdmCorpusDefinition cdmCorpus = TestHelper.getLocalCorpus(TESTS_SUBPATH, "testRefreshesDataPartition");
final CdmManifestDefinition cdmManifest = cdmCorpus.<CdmManifestDefinition>fetchObjectAsync("local:/partitions.manifest.cdm.json").join();
final FileStatusCheckOptions fileStatusCheckOptions = new FileStatusCheckOptions(true);
final CdmEntityDeclarationDefinition partitionEntity = cdmManifest.getEntities().get(0);
Assert.assertEquals(partitionEntity.getDataPartitions().size(), 1);
final CdmDataPartitionDefinition partition = partitionEntity.getDataPartitions().get(0);
cdmManifest.fileStatusCheckAsync(PartitionFileStatusCheckType.Full, CdmIncrementalPartitionType.None, fileStatusCheckOptions).join();
int localTraitIndex = partition.getExhibitsTraits().indexOf("is.partition.size");
Assert.assertNotEquals(localTraitIndex, -1);
final CdmTraitReference localTrait = (CdmTraitReference)partition.getExhibitsTraits().get(localTraitIndex);
Assert.assertEquals(localTrait.getNamedReference(), "is.partition.size");
Assert.assertEquals(localTrait.getArguments().get(0).getValue(), (long)2);
}
}

91
objectModel/Java/objectmodel/src/test/java/com/microsoft/commondatamodel/objectmodel/cdm/datapartitionpattern/DataPartitionPatternTest.java
Просмотреть файл

@ -8,6 +8,7 @@ import java.io.IOException;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.*;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
@ -37,12 +38,14 @@ import com.microsoft.commondatamodel.objectmodel.persistence.cdmfolder.types.Man
import com.microsoft.commondatamodel.objectmodel.resolvedmodel.ResolveContext;
import com.microsoft.commondatamodel.objectmodel.storage.LocalAdapter;
import com.microsoft.commondatamodel.objectmodel.storage.testAdapters.FetchAllMetadataNullAdapter;
import com.microsoft.commondatamodel.objectmodel.storage.testAdapters.FetchAllMetadataThrowErrorAdapter;
import com.microsoft.commondatamodel.objectmodel.storage.testAdapters.NoOverrideAdapter;
import com.microsoft.commondatamodel.objectmodel.storage.testAdapters.OverrideFetchAllFilesAdapter;
import com.microsoft.commondatamodel.objectmodel.utilities.Constants;
import com.microsoft.commondatamodel.objectmodel.utilities.FileStatusCheckOptions;
import com.microsoft.commondatamodel.objectmodel.utilities.JMapper;
import com.microsoft.commondatamodel.objectmodel.utilities.exceptions.CdmReadPartitionFromPatternException;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.testng.Assert;
import org.testng.AssertJUnit;
@ -56,7 +59,7 @@ public class DataPartitionPatternTest {
* Tests refreshing files that match the regular expression
*/
@Test
public void testRefreshesDataPartitionPatterns() throws InterruptedException {
public void testRefreshesDataPartitionPatterns() throws CdmReadPartitionFromPatternException, InterruptedException {
final CdmCorpusDefinition cdmCorpus = TestHelper.getLocalCorpus(TESTS_SUBPATH, "testRefreshDataPartitionPatterns");
final CdmManifestDefinition cdmManifest = cdmCorpus.<CdmManifestDefinition>fetchObjectAsync("local:/patternManifest.manifest.cdm.json").join();
@ -138,7 +141,7 @@ public class DataPartitionPatternTest {
* Tests data partition objects created by a partition pattern do not share the same trait with the partition pattern
*/
@Test
public void testRefreshesDataPartitionPatternsWithTrait() throws InterruptedException {
public void testRefreshesDataPartitionPatternsWithTrait() throws CdmReadPartitionFromPatternException, InterruptedException {
final CdmCorpusDefinition corpus = TestHelper.getLocalCorpus(TESTS_SUBPATH, "testRefreshesDataPartitionPatternsWithTrait");
final CdmManifestDefinition manifest = corpus.<CdmManifestDefinition>fetchObjectAsync("local:/patternManifest.manifest.cdm.json").join();
@ -175,7 +178,7 @@ public class DataPartitionPatternTest {
* Tests refreshing incremental partition files that match the regular expression
*/
@Test
public void testIncrementalPatternsRefreshesFullAndIncremental() throws InterruptedException {
public void testIncrementalPatternsRefreshesFullAndIncremental() throws CdmReadPartitionFromPatternException, InterruptedException {
final CdmCorpusDefinition corpus = TestHelper.getLocalCorpus(TESTS_SUBPATH, "testIncrementalPatternsRefreshesFullAndIncremental");
final CdmManifestDefinition manifest = corpus.<CdmManifestDefinition>fetchObjectAsync("local:/pattern.manifest.cdm.json").join();
@ -250,7 +253,7 @@ public class DataPartitionPatternTest {
* Tests refreshing incremental partition files that match the regular expression
*/
@Test
public void testIncrementalPatternsRefreshesDeleteIncremental() throws InterruptedException {
public void testIncrementalPatternsRefreshesDeleteIncremental() throws CdmReadPartitionFromPatternException, InterruptedException {
final CdmCorpusDefinition corpus = TestHelper.getLocalCorpus(TESTS_SUBPATH, "testIncrementalPatternsRefreshesDeleteIncremental");
final CdmManifestDefinition manifest = corpus.<CdmManifestDefinition>fetchObjectAsync("local:/pattern.manifest.cdm.json").join();
@ -317,7 +320,7 @@ public class DataPartitionPatternTest {
* Tests refreshing partition pattern with invalid incremental partition trait and invalid arguments.
*/
@Test
public void testPatternRefreshesWithInvalidTraitAndArgument() throws InterruptedException {
public void testPatternRefreshesWithInvalidTraitAndArgument() throws CdmReadPartitionFromPatternException, InterruptedException {
// providing invalid enum value of CdmIncrementalPartitionType in string
// "traitReference": "is.partition.incremental", "arguments": [{"name": "type","value": "typo"}]
@ -406,7 +409,7 @@ public class DataPartitionPatternTest {
* Tests refreshing partition with invalid incremental partition trait and invalid arguments.
*/
@Test
public void testPartitionRefreshesWithInvalidTraitAndArgument() throws InterruptedException {
public void testPartitionRefreshesWithInvalidTraitAndArgument() throws CdmReadPartitionFromPatternException, InterruptedException {
// providing invalid enum value of CdmIncrementalPartitionType in string
// "traitReference": "is.partition.incremental", "arguments": [{"name": "type","value": "typo"}]
@ -495,7 +498,7 @@ public class DataPartitionPatternTest {
* Tests fileStatusCheckAsync(), fileStatusCheckAsync(PartitionFileStatusCheckType.Full), and fileStatusCheckAsync(PartitionFileStatusCheckType.None).
*/
@Test
public void testPartitionFileRefreshTypeFullOrNone() throws InterruptedException {
public void testPartitionFileRefreshTypeFullOrNone() throws CdmReadPartitionFromPatternException, InterruptedException {
final CdmCorpusDefinition corpus = TestHelper.getLocalCorpus(TESTS_SUBPATH, "testPartitionFileRefreshTypeFullOrNone");
final CdmManifestDefinition manifest = corpus.<CdmManifestDefinition>fetchObjectAsync("local:/pattern.manifest.cdm.json").join();
@ -552,7 +555,7 @@ public class DataPartitionPatternTest {
* Testing that error is handled when partition pattern contains a folder that does not exist
*/
@Test
public void testPatternWithNonExistingFolder() throws IOException, InterruptedException {
public void testPatternWithNonExistingFolder() throws CdmReadPartitionFromPatternException, InterruptedException, IOException {
final CdmCorpusDefinition corpus = TestHelper.getLocalCorpus(TESTS_SUBPATH, "testPatternWithNonExistingFolder");
final String content = TestHelper.getInputFileContent(TESTS_SUBPATH, "testPatternWithNonExistingFolder", "entities.manifest.cdm.json");
final CdmManifestDefinition cdmManifest = ManifestPersistence.fromObject(new ResolveContext(corpus), "entities", "local", "/", JMapper.MAP.readValue(content, ManifestContent.class));
@ -575,7 +578,7 @@ public class DataPartitionPatternTest {
* Testing that partition is correctly found when namespace of pattern differs from namespace of the manifest
*/
@Test
public void TestPatternWithDifferentNamespace() throws IOException, InterruptedException {
public void TestPatternWithDifferentNamespace() throws CdmReadPartitionFromPatternException, InterruptedException {
final String testName = "TestPatternWithDifferentNamespace";
final CdmCorpusDefinition cdmCorpus = TestHelper.getLocalCorpus(TESTS_SUBPATH, testName);
LocalAdapter localAdapter = (LocalAdapter)cdmCorpus.getStorage().fetchAdapter("local");
@ -592,7 +595,7 @@ public class DataPartitionPatternTest {
* Testing that patterns behave correctly with variations to rootLocation
*/
@Test
public void testVariationsInRootLocation() throws IOException, InterruptedException {
public void testVariationsInRootLocation() throws CdmReadPartitionFromPatternException, InterruptedException {
CdmCorpusDefinition corpus = TestHelper.getLocalCorpus(TESTS_SUBPATH, "TestVariationsInRootLocation");
CdmManifestDefinition manifest = corpus.<CdmManifestDefinition>fetchObjectAsync("pattern.manifest.cdm.json").join();
manifest.fileStatusCheckAsync().join();
@ -617,7 +620,7 @@ public class DataPartitionPatternTest {
* Testing data partition patterns that use glob patterns
*/
@Test
public void testPartitionPatternWithGlob() throws InterruptedException {
public void testPartitionPatternWithGlob() throws CdmReadPartitionFromPatternException, InterruptedException {
final CdmCorpusDefinition corpus = TestHelper.getLocalCorpus(TESTS_SUBPATH, "testPartitionPatternWithGlob");
HashMap<String, String> patternWithGlobAndRegex = new HashMap<>();
@ -786,7 +789,7 @@ public class DataPartitionPatternTest {
* Testing data partition patterns that use glob patterns with variations in path style
*/
@Test
public void testGlobPathVariation() throws InterruptedException {
public void testGlobPathVariation() throws CdmReadPartitionFromPatternException, InterruptedException {
final CdmCorpusDefinition corpus = TestHelper.getLocalCorpus(TESTS_SUBPATH, "testGlobPathVariation");
final CdmManifestDefinition manifest = corpus.<CdmManifestDefinition>fetchObjectAsync("pattern.manifest.cdm.json").join();
@ -843,13 +846,13 @@ public class DataPartitionPatternTest {
* @throws ExecutionException
*/
@Test
public void testFileStatusCheckOnNullLocation() throws InterruptedException, ExecutionException {
public void testFileStatusCheckOnNullLocation() throws CdmReadPartitionFromPatternException, InterruptedException {
CdmCorpusDefinition corpus = TestHelper.getLocalCorpus(TESTS_SUBPATH, "testFileStatusCheckOnNullLocation");
corpus.setEventCallback((level, message) -> {
Assert.assertEquals(level, CdmStatusLevel.Error, "Error level message should have been reported");
Assert.assertTrue(
message.equals("StorageManager | The object path cannot be null or empty. | createAbsoluteCorpusPath") ||
message.equals("CdmCorpusDefinition | The object path cannot be null or empty. | getLastModifiedTimeFromPartitionPathAsync"),
message.equals("CdmCorpusDefinition | The object path cannot be null or empty. | getFileMetadataFromPartitionPathAsync"),
"Unexpected error message received");
}, CdmStatusLevel.Warning);
@ -879,7 +882,7 @@ public class DataPartitionPatternTest {
* @throws InterruptedException
*/
@Test
public void testFetchAllFilesMetadata() throws InterruptedException {
public void testFetchAllFilesMetadata() throws CdmReadPartitionFromPatternException, InterruptedException {
final HashSet<CdmLogCode> expectedLogCodes = new HashSet<>(Arrays.asList(CdmLogCode.ErrFetchingFileMetadataNull));
final CdmCorpusDefinition corpus = TestHelper.getLocalCorpus(TESTS_SUBPATH, "testFetchAllFilesMetadata", null, false, expectedLogCodes);
final FileStatusCheckOptions fileStatusCheckOptions = new FileStatusCheckOptions(true);
@ -937,4 +940,62 @@ public class DataPartitionPatternTest {
final CdmManifestDefinition fetchNullManifest = corpus.<CdmManifestDefinition>fetchObjectAsync("fetchNull:/manifest.manifest.cdm.json").join();
fetchNullManifest.fileStatusCheckAsync(PartitionFileStatusCheckType.Full, CdmIncrementalPartitionType.None, fileStatusCheckOptions).join();
}
/**
* Test that error is thrown when FileStatusCheckOption is set
*/
@Test
public void testThrowOnPartitionError() throws InterruptedException {
final HashSet<CdmLogCode> expectedLogCodes = new HashSet<>(Arrays.asList(CdmLogCode.WarnPartitionFileFetchFailed));
CdmCorpusDefinition corpus = TestHelper.getLocalCorpus(TESTS_SUBPATH, "TestFetchAllFilesMetadata", null, false, expectedLogCodes);
final LocalAdapter testLocalAdapter = (LocalAdapter)corpus.getStorage().getNamespaceAdapters().get(corpus.getStorage().getDefaultNamespace());
corpus.getStorage().mount("error", new FetchAllMetadataThrowErrorAdapter(testLocalAdapter));
final FileStatusCheckOptions fileStatusCheckOptions = new FileStatusCheckOptions(false, true);
boolean manifestThrowsError = false;
boolean entityDecThrowsError = false;
boolean partitionPatternThrowsError = false;
final CdmManifestDefinition manifest = corpus.<CdmManifestDefinition>fetchObjectAsync("error:/manifest.manifest.cdm.json").join();
try {
manifest.fileStatusCheckAsync(PartitionFileStatusCheckType.Full, CdmIncrementalPartitionType.None, fileStatusCheckOptions).join();
} catch (CompletionException e) {
Throwable cdmReadPartitionException = e.getCause();
Assert.assertTrue(cdmReadPartitionException instanceof CdmReadPartitionFromPatternException);
Throwable innerException = cdmReadPartitionException.getCause();
Assert.assertNotNull(innerException);
Assert.assertEquals("Some test error message.", innerException.getMessage());
manifestThrowsError = true;
}
final CdmLocalEntityDeclarationDefinition entityDec = (CdmLocalEntityDeclarationDefinition) manifest.getEntities().get(0);
try {
entityDec.fileStatusCheckAsync(PartitionFileStatusCheckType.Full, CdmIncrementalPartitionType.None, fileStatusCheckOptions).join();
} catch (CompletionException e) {
Throwable cdmReadPartitionException = e.getCause();
Assert.assertTrue(cdmReadPartitionException instanceof CdmReadPartitionFromPatternException);
Throwable innerException = cdmReadPartitionException.getCause();
Assert.assertNotNull(innerException);
Assert.assertEquals("Some test error message.", innerException.getMessage());
entityDecThrowsError = true;
}
final CdmDataPartitionPatternDefinition partitionPattern = (manifest.getEntities().get(0)).getDataPartitionPatterns().get(0);
try {
partitionPattern.fileStatusCheckAsync(fileStatusCheckOptions).join();
} catch (CompletionException e) {
Throwable cdmReadPartitionException = e.getCause();
Assert.assertTrue(cdmReadPartitionException instanceof CdmReadPartitionFromPatternException);
Throwable innerException = cdmReadPartitionException.getCause();
Assert.assertNotNull(innerException);
Assert.assertEquals("Some test error message.", innerException.getMessage());
partitionPatternThrowsError = true;
}
Assert.assertTrue(manifestThrowsError && entityDecThrowsError && partitionPatternThrowsError);
}
}

6
objectModel/Java/objectmodel/src/test/java/com/microsoft/commondatamodel/objectmodel/cdm/projection/ProjectionMiscellaneousTest.java
Просмотреть файл

@ -7,6 +7,7 @@ import com.microsoft.commondatamodel.objectmodel.TestHelper;
import com.microsoft.commondatamodel.objectmodel.cdm.*;
import com.microsoft.commondatamodel.objectmodel.cdm.projections.CardinalitySettings;
import com.microsoft.commondatamodel.objectmodel.cdm.projections.CdmProjection;
import com.microsoft.commondatamodel.objectmodel.enums.CdmLogCode;
import com.microsoft.commondatamodel.objectmodel.enums.CdmObjectType;
import com.microsoft.commondatamodel.objectmodel.enums.CdmStatusLevel;
import com.microsoft.commondatamodel.objectmodel.utilities.ProjectionTestUtils;
@ -189,7 +190,8 @@ public class ProjectionMiscellaneousTest {
String testName = "testMaxDepthOnPolymorphicEntity";
String entityName = "A";
CdmCorpusDefinition corpus = TestHelper.getLocalCorpus(TESTS_SUBPATH, testName);
final HashSet<CdmLogCode> expectedLogCodes = new HashSet<>(Arrays.asList(CdmLogCode.WarnMaxDepthExceeded ));
CdmCorpusDefinition corpus = TestHelper.getLocalCorpus(TESTS_SUBPATH, testName, null, false, expectedLogCodes);
CdmEntityDefinition entity = corpus.<CdmEntityDefinition>fetchObjectAsync(entityName + ".cdm.json/" + entityName).join();
@ -198,7 +200,7 @@ public class ProjectionMiscellaneousTest {
CdmEntityDefinition resEntity = entity.createResolvedEntityAsync("resolved-" + entityName, resOpt).join();
Assert.assertNotNull(resEntity);
Assert.assertEquals(resEntity.getAttributes().getCount(), 4);
Assert.assertEquals(resEntity.getAttributes().getCount(), 2);
}
/**

7
objectModel/Java/objectmodel/src/test/java/com/microsoft/commondatamodel/objectmodel/cdm/resolution/CacheTest.java
Просмотреть файл

@ -10,6 +10,7 @@ import com.microsoft.commondatamodel.objectmodel.cdm.CdmAttributeGroupReference;
import com.microsoft.commondatamodel.objectmodel.cdm.CdmCorpusDefinition;
import com.microsoft.commondatamodel.objectmodel.cdm.CdmEntityDefinition;
import com.microsoft.commondatamodel.objectmodel.cdm.CdmTypeAttributeDefinition;
import com.microsoft.commondatamodel.objectmodel.enums.CdmLogCode;
import com.microsoft.commondatamodel.objectmodel.utilities.AttributeResolutionDirectiveSet;
import com.microsoft.commondatamodel.objectmodel.utilities.ResolveOptions;
@ -25,7 +26,8 @@ public class CacheTest {
*/
@Test
public void testMaxDepthCached() throws InterruptedException {
CdmCorpusDefinition cdmCorpus = TestHelper.getLocalCorpus(TESTS_SUBPATH, "testMaxDepth");
final HashSet<CdmLogCode> expectedLogCodes = new HashSet<>(Arrays.asList(CdmLogCode.WarnMaxDepthExceeded ));
CdmCorpusDefinition cdmCorpus = TestHelper.getLocalCorpus(TESTS_SUBPATH, "testMaxDepth", null, false, expectedLogCodes);
CdmEntityDefinition aEnt = cdmCorpus.<CdmEntityDefinition>fetchObjectAsync("A.cdm.json/A").join();
CdmEntityDefinition bEnt = cdmCorpus.<CdmEntityDefinition>fetchObjectAsync("B.cdm.json/B").join();
CdmEntityDefinition cEnt = cdmCorpus.<CdmEntityDefinition>fetchObjectAsync("C.cdm.json/C").join();
@ -68,7 +70,8 @@ public class CacheTest {
*/
@Test
public void testNonMaxDepthCached() throws InterruptedException {
CdmCorpusDefinition cdmCorpus = TestHelper.getLocalCorpus(TESTS_SUBPATH, "testMaxDepth");
final HashSet<CdmLogCode> expectedLogCodes = new HashSet<>(Arrays.asList(CdmLogCode.WarnMaxDepthExceeded ));
CdmCorpusDefinition cdmCorpus = TestHelper.getLocalCorpus(TESTS_SUBPATH, "testMaxDepth", null, false, expectedLogCodes);
CdmEntityDefinition aEnt = cdmCorpus.<CdmEntityDefinition>fetchObjectAsync("A.cdm.json/A").join();
CdmEntityDefinition bEnt = cdmCorpus.<CdmEntityDefinition>fetchObjectAsync("B.cdm.json/B").join();
CdmEntityDefinition cEnt = cdmCorpus.<CdmEntityDefinition>fetchObjectAsync("C.cdm.json/C").join();

5
objectModel/Java/objectmodel/src/test/java/com/microsoft/commondatamodel/objectmodel/cdm/resolution/ManifestResolutionTest.java
Просмотреть файл

@ -14,6 +14,7 @@ import com.microsoft.commondatamodel.objectmodel.enums.CdmObjectType;
import com.microsoft.commondatamodel.objectmodel.enums.CdmStatusLevel;
import com.microsoft.commondatamodel.objectmodel.persistence.CdmConstants;
import com.microsoft.commondatamodel.objectmodel.storage.LocalAdapter;
import com.microsoft.commondatamodel.objectmodel.utilities.ResolveOptions;
import org.testng.Assert;
import org.testng.annotations.Test;
@ -147,7 +148,9 @@ public class ManifestResolutionTest {
try {
final CdmCorpusDefinition corpus = TestHelper.getLocalCorpus(TESTS_SUBPATH, "testResolveManifestWithInterdependentPolymorphicSource");
final CdmManifestDefinition manifest = (CdmManifestDefinition) corpus.fetchObjectAsync("local:/Input.manifest.cdm.json").join();
final CdmManifestDefinition resolvedManifest = manifest.createResolvedManifestAsync("resolved", null).join();
final ResolveOptions resOpt = new ResolveOptions();
resOpt.setMaxDepth(3);
final CdmManifestDefinition resolvedManifest = manifest.createResolvedManifestAsync("resolved", null, null, resOpt).join();
Assert.assertEquals(2, resolvedManifest.getEntities().getCount());
Assert.assertEquals("resolved/group.cdm.json/group", resolvedManifest.getEntities().get(0).getEntityPath().toLowerCase());

4
objectModel/Java/objectmodel/src/test/java/com/microsoft/commondatamodel/objectmodel/persistence/cdmfolder/manifest/ManifestImplTest.java
Просмотреть файл

@ -13,6 +13,8 @@ import com.microsoft.commondatamodel.objectmodel.resolvedmodel.ResolveContext;
import com.microsoft.commondatamodel.objectmodel.storage.LocalAdapter;
import com.microsoft.commondatamodel.objectmodel.utilities.JMapper;
import com.microsoft.commondatamodel.objectmodel.utilities.TimeUtils;
import com.microsoft.commondatamodel.objectmodel.utilities.exceptions.CdmReadPartitionFromPatternException;
import org.testng.Assert;
import org.testng.annotations.Test;
@ -154,7 +156,7 @@ public class ManifestImplTest {
* Test modified times for manifest and files beneath it
*/
@Test
public void testLoadsAndSetsTimesCorrectly() throws InterruptedException {
public void testLoadsAndSetsTimesCorrectly() throws CdmReadPartitionFromPatternException, InterruptedException {
String inputPath = TestHelper.getInputFolderPath(TESTS_SUBPATH, "testLoadsAndSetsTimesCorrectly");
final OffsetDateTime timeBeforeLoad = OffsetDateTime.now();

8
objectModel/Java/objectmodel/src/test/java/com/microsoft/commondatamodel/objectmodel/samples/SearchPartitionPatternTest.java
Просмотреть файл

@ -15,6 +15,8 @@ import com.microsoft.commondatamodel.objectmodel.enums.CdmIncrementalPartitionTy
import com.microsoft.commondatamodel.objectmodel.enums.CdmObjectType;
import com.microsoft.commondatamodel.objectmodel.enums.PartitionFileStatusCheckType;
import com.microsoft.commondatamodel.objectmodel.storage.LocalAdapter;
import com.microsoft.commondatamodel.objectmodel.utilities.exceptions.CdmReadPartitionFromPatternException;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.testng.annotations.Test;
@ -26,11 +28,11 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
public class SearchPartitionPatternTest extends SampleTestBase{
public class SearchPartitionPatternTest extends SampleTestBase {
private static final String TEST_NAME = "TestSearchPartitionPattern";
@Test
public void testSearchPartitionPattern() throws InterruptedException, IOException {
public void testSearchPartitionPattern() throws CdmReadPartitionFromPatternException, InterruptedException, IOException {
this.checkSampleRunTestsFlag();
TestHelper.deleteFilesFromActualOutput(TestHelper.getActualOutputFolderPath(TESTS_SUBPATH, TEST_NAME));
@ -65,7 +67,7 @@ public class SearchPartitionPatternTest extends SampleTestBase{
return cdmCorpus;
}
private void searchPartitionPattern(CdmCorpusDefinition cdmCorpus) {
private void searchPartitionPattern(CdmCorpusDefinition cdmCorpus) throws CdmReadPartitionFromPatternException {
String sampleEntityName = "Account";
System.out.println("Make placeholder manifest.");

34
objectModel/Java/objectmodel/src/test/java/com/microsoft/commondatamodel/objectmodel/storage/AdlsAdapterTest.java
Просмотреть файл

@ -10,15 +10,15 @@ import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.microsoft.commondatamodel.objectmodel.AdlsTestHelper;
import com.microsoft.commondatamodel.objectmodel.TestHelper;
import com.microsoft.commondatamodel.objectmodel.cdm.CdmCorpusDefinition;
import com.microsoft.commondatamodel.objectmodel.cdm.CdmDocumentDefinition;
import com.microsoft.commondatamodel.objectmodel.cdm.CdmFolderDefinition;
import com.microsoft.commondatamodel.objectmodel.cdm.CdmManifestDefinition;
import com.microsoft.commondatamodel.objectmodel.cdm.*;
import com.microsoft.commondatamodel.objectmodel.enums.AzureCloudEndpoint;
import com.microsoft.commondatamodel.objectmodel.enums.CdmIncrementalPartitionType;
import com.microsoft.commondatamodel.objectmodel.enums.CdmStatusLevel;
import com.microsoft.commondatamodel.objectmodel.enums.PartitionFileStatusCheckType;
import com.microsoft.commondatamodel.objectmodel.resolvedmodel.ResolveContext;
import com.microsoft.commondatamodel.objectmodel.storage.StorageAdapterBase.CacheContext;
import com.microsoft.commondatamodel.objectmodel.storage.testAdapters.MockAdlsAdapter;
import com.microsoft.commondatamodel.objectmodel.utilities.FileStatusCheckOptions;
import com.microsoft.commondatamodel.objectmodel.utilities.JMapper;
import com.microsoft.commondatamodel.objectmodel.utilities.network.CdmHttpClient;
import com.microsoft.commondatamodel.objectmodel.utilities.network.CdmHttpResponse;
@ -649,4 +649,30 @@ public class AdlsAdapterTest {
someDoc.saveAsAsync("someDoc.cdm.json").join();
assertTrue(noFlushErrorHit.get());
}
/**
* Tests refreshing data partition gets file size in ADLS
*/
@Test
public void testADLSRefreshesDataPartition() throws Exception {
AdlsTestHelper.checkADLSEnvironment();
final AdlsAdapter adlsAdapter = AdlsTestHelper.createAdapterWithSharedKey();
final CdmCorpusDefinition corpus = new CdmCorpusDefinition();
corpus.getStorage().mount("adls", adlsAdapter);
final CdmManifestDefinition cdmManifest = corpus.<CdmManifestDefinition>fetchObjectAsync("adls:/TestPartitionMetadata/partitions.manifest.cdm.json").join();
final FileStatusCheckOptions fileStatusCheckOptions = new FileStatusCheckOptions(true);
final CdmEntityDeclarationDefinition partitionEntity = cdmManifest.getEntities().get(0);
Assert.assertEquals(partitionEntity.getDataPartitions().size(), 1);
final CdmDataPartitionDefinition partition = partitionEntity.getDataPartitions().get(0);
cdmManifest.fileStatusCheckAsync(PartitionFileStatusCheckType.Full, CdmIncrementalPartitionType.None, fileStatusCheckOptions).join();
int localTraitIndex = partition.getExhibitsTraits().indexOf("is.partition.size");
Assert.assertNotEquals(localTraitIndex, -1);
final CdmTraitReference localTrait = (CdmTraitReference) partition.getExhibitsTraits().get(localTraitIndex);
Assert.assertEquals(localTrait.getNamedReference(), "is.partition.size");
Assert.assertEquals(localTrait.getArguments().get(0).getValue(), (long) 2);
}
}

23
objectModel/Java/objectmodel/src/test/java/com/microsoft/commondatamodel/objectmodel/storage/CdmCustomPackageAdapterTest.java
Просмотреть файл

@ -3,6 +3,11 @@
package com.microsoft.commondatamodel.objectmodel.storage;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.microsoft.commondatamodel.objectmodel.cdm.CdmCorpusDefinition;
import com.microsoft.commondatamodel.objectmodel.enums.CdmStatusLevel;
import com.microsoft.commondatamodel.objectmodel.utilities.JMapper;
import org.testng.Assert;
import org.testng.annotations.Test;
@ -94,4 +99,22 @@ public class CdmCustomPackageAdapterTest {
String foundations = adapter.readAsync(FOUNDATIONS_FILE_PATH).join();
Assert.assertNotNull(foundations);
}
/**
* Test mounting CdmStandards adapter from config does not cause an error
*/
@Test
public void testCdmStandardsMountFromConfig() throws ClassNotFoundException {
CdmCorpusDefinition corpus = new CdmCorpusDefinition();
corpus.setEventCallback((CdmStatusLevel level, String message) -> {
Assert.fail(String.format("Unexpected error: %s", message));
}, CdmStatusLevel.Warning);
corpus.getStorage().mountFromConfig("{\"adapters\": [{\"config\": {\"locationHint\": \"\", \"maximumTimeout\": 20000, \"numberOfRetries\": 2, \"root\": \"/logical\", \"timeout\": 5000}, \"namespace\": \"cdm\", \"type\": \"cdm-standards\"}], \"defaultNamespace\": \"local\"}");
corpus.getStorage().mount("cdm", new CdmStandardsAdapter());
String config = corpus.getStorage().fetchConfig();
corpus.getStorage().mountFromConfig(config);
}
}

21
objectModel/Java/objectmodel/src/test/java/com/microsoft/commondatamodel/objectmodel/storage/testAdapters/FetchAllMetadataThrowErrorAdapter.java Normal file
Просмотреть файл

@ -0,0 +1,21 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.
package com.microsoft.commondatamodel.objectmodel.storage.testAdapters;
import com.microsoft.commondatamodel.objectmodel.storage.LocalAdapter;
import com.microsoft.commondatamodel.objectmodel.utilities.CdmFileMetadata;
import java.util.HashMap;
import java.util.concurrent.CompletableFuture;
public class FetchAllMetadataThrowErrorAdapter extends NoOverrideAdapter {
public FetchAllMetadataThrowErrorAdapter(final LocalAdapter localAdapter) {
super(localAdapter);
}
@Override
public CompletableFuture<HashMap<String, CdmFileMetadata>> fetchAllFilesMetadataAsync(final String folderCorpusPath) {
throw new RuntimeException("Some test error message.");
}
}

2
objectModel/Java/objectmodel/src/test/java/com/microsoft/commondatamodel/objectmodel/utilities/EventListTest.java
Просмотреть файл

@ -36,7 +36,7 @@ public class EventListTest {
* @throws InterruptedException
*/
@Test
public void testWithoutNesting() throws InterruptedException {
public void testWithoutNesting() throws Exception {
CdmCorpusDefinition corpus = TestHelper.getLocalCorpus(TESTS_SUBPATH, "TestEventList");
corpus.setEventCallback(eventCallback, CdmStatusLevel.Warning);
corpus.getCtx().setCorrelationId(DUMMY_CORRELATION_ID);

2
objectModel/Java/pom.xml
Просмотреть файл

@ -5,7 +5,7 @@
<properties>
<!-- Increment version for each official release of the SDK -->
<revision>1.7.5</revision>
<revision>1.7.6</revision>
<cdmstandards-revision>2.8.0</cdmstandards-revision>
</properties>

2
objectModel/Python/cdm/enums/cdm_log_code.py
Просмотреть файл

@ -28,6 +28,7 @@ class CdmLogCode(AutoNumber):
ERR_MANIFEST_FILE_MOD_TIME_FAILURE = ()
ERR_MISSING_INCREMENTAL_PARTITION_TRAIT = ()
ERR_OBJECT_WITHOUT_OWNER_FOUND = ()
ERR_PARTITION_FILE_METADATA_FAILURE = ()
ERR_PARTITION_FILE_MOD_TIME_FAILURE = ()
ERR_PATH_IS_DUPLICATE = ()
ERR_PATH_NULL_OBJECT_PATH = ()
@ -155,6 +156,7 @@ class CdmLogCode(AutoNumber):
WARN_DEPRECATED_RESOLUTION_GUIDANCE = ()
WARN_DOC_CHANGES_DISCARDED = ()
WARN_DOC_IMPORT_NOT_LOADED = ()
WARN_MAX_DEPTH_EXCEEDED = ()
WARN_PARTITION_FILE_FETCH_FAILED = ()
WARN_LINK_ENT_IDENT_ARGS_NOT_SUPPORTED = ()
WARN_PARTITION_INVALID_ARGUMENTS = ()

43
objectModel/Python/cdm/objectmodel/cdm_corpus_def.py
Просмотреть файл

@ -15,6 +15,7 @@ from cdm.enums import CdmAttributeContextType, CdmObjectType, CdmStatusLevel, Cd
from cdm.objectmodel import CdmContainerDefinition
from cdm.utilities import AttributeResolutionDirectiveSet, DocsResult, ImportInfo, ResolveOptions, \
StorageUtils, SymbolSet, logger
from cdm.utilities.cdm_file_metadata import CdmFileMetadata
from .cdm_attribute_ref import CdmAttributeReference
from .cdm_corpus_context import CdmCorpusContext
@ -115,7 +116,7 @@ class CdmCorpusDefinition:
return doc
async def calculate_entity_graph_async(self, curr_manifest: 'CdmManifestDefinition') -> None:
async def calculate_entity_graph_async(self, curr_manifest: 'CdmManifestDefinition', res_opt: Optional['ResolveOptions'] = None) -> None:
"""Calculate the entity to entity relationships for all the entities present in the folder and its sub folder."""
with logger._enter_scope(self._TAG, self.ctx, self.calculate_entity_graph_async.__name__):
for entity_dec in curr_manifest.entities:
@ -132,21 +133,22 @@ class CdmCorpusDefinition:
res_entity = None # type: Optional[CdmEntityDefinition]
# make options wrt this entity document and "relational" always
res_opt = ResolveOptions(entity.in_document,
AttributeResolutionDirectiveSet({'normalized', 'referenceOnly'}))
res_opt_copy = res_opt.copy() if res_opt is not None else ResolveOptions()
res_opt_copy.wrt_doc = entity.in_document
res_opt_copy.directives = AttributeResolutionDirectiveSet({'normalized', 'referenceOnly'})
is_resolved_entity = entity._is_resolved
# only create a resolved entity if the entity passed in was not a resolved entity
if not is_resolved_entity:
# first get the resolved entity so that all of the references are present
res_entity = await entity.create_resolved_entity_async('wrtSelf_' + entity.entity_name, res_opt)
res_entity = await entity.create_resolved_entity_async('wrtSelf_' + entity.entity_name, res_opt_copy)
else:
res_entity = entity
# find outgoing entity relationships using attribute context
new_outgoing_relationships = self._find_outgoing_relationships(
res_opt, res_entity, res_entity.attribute_context, is_resolved_entity) # type: List[CdmE2ERelationship]
res_opt_copy, res_entity, res_entity.attribute_context, is_resolved_entity) # type: List[CdmE2ERelationship]
old_outgoing_relationships = self._outgoing_relationships.get(entity.at_corpus_path) # type: List[CdmE2ERelationship]
@ -195,7 +197,7 @@ class CdmCorpusDefinition:
logger.error(self.ctx, self._TAG, self.calculate_entity_graph_async.__name__, curr_manifest.at_corpus_path,
CdmLogCode.ERR_INVALID_CAST, sub_manifest_def.definition, 'CdmManifestDefinition')
continue
await self.calculate_entity_graph_async(sub_manifest)
await self.calculate_entity_graph_async(sub_manifest, res_opt)
async def create_root_manifest_async(self, corpus_path: str) -> Optional['CdmManifestDefinition']:
if self._is_path_manifest_document(corpus_path):
@ -1226,25 +1228,34 @@ class CdmCorpusDefinition:
async def _get_last_modified_time_from_partition_path_async(self, corpus_path: str) -> datetime:
"""Return last modified time of a partition object."""
file_metadata = await self._get_file_metadata_from_partition_path_async(corpus_path)
if file_metadata is None:
return None
return file_metadata['last_modified_time']
async def _get_file_metadata_from_partition_path_async(self, corpus_path: 'CdmObject') -> CdmFileMetadata:
"""Gets the file metadata of the partition without trying to read the file itself."""
# We do not want to load partitions from file, just check the modified times.
path_tuple = StorageUtils.split_namespace_path(corpus_path)
if not path_tuple:
logger.error(self.ctx, self._TAG, self._get_last_modified_time_from_partition_path_async.__name__, corpus_path,
logger.error(self.ctx, self._TAG, self._get_file_metadata_from_partition_path_async.__name__, corpus_path,
CdmLogCode.ERR_PATH_NULL_OBJECT_PATH)
return None
namespace = path_tuple[0]
if namespace:
adapter = self.storage.fetch_adapter(namespace)
if not adapter:
logger.error(self.ctx, self._TAG, self._get_last_modified_time_from_partition_path_async.__name__, corpus_path,
CdmLogCode.ERR_ADAPTER_NOT_FOUND, namespace)
return None
try:
return await adapter.compute_last_modified_time_async(path_tuple[1])
except Exception as e:
logger.error(self.ctx, self._TAG, self._get_last_modified_time_from_partition_path_async.__name__, corpus_path,
CdmLogCode.ERR_PARTITION_FILE_MOD_TIME_FAILURE, path_tuple, e)
if not adapter:
logger.error(self.ctx, self._TAG, self._get_file_metadata_from_partition_path_async.__name__, corpus_path,
CdmLogCode.ERR_ADAPTER_NOT_FOUND, namespace)
return None
try:
return await adapter.fetch_file_metadata_async(path_tuple[1])
except Exception as e:
logger.error(self.ctx, self._TAG, self._get_file_metadata_from_partition_path_async.__name__, corpus_path,
CdmLogCode.ERR_PARTITION_FILE_MOD_TIME_FAILURE, path_tuple, e)
return None
@staticmethod

8
objectModel/Python/cdm/objectmodel/cdm_data_partition_def.py
Просмотреть файл

@ -99,15 +99,17 @@ class CdmDataPartitionDefinition(CdmObjectDefinition, CdmFileStatus):
return copy
async def file_status_check_async(self) -> None:
async def file_status_check_async(self, file_status_check_options = None) -> None:
"""Check the modified time for this object and any children."""
with logger._enter_scope(self._TAG, self.ctx, self.file_status_check_async.__name__):
full_path = self.ctx.corpus.storage.create_absolute_corpus_path(self.location, self.in_document)
modified_time = await self.ctx.corpus._get_last_modified_time_from_partition_path_async(full_path)
partition_metadata = await self.ctx.corpus._get_file_metadata_from_partition_path_async(full_path)
# Update modified times.
self.last_file_status_check_time = datetime.now(timezone.utc)
self.last_file_modified_time = time_utils._max_time(modified_time, self.last_file_modified_time)
self.last_file_modified_time = time_utils._max_time(partition_metadata['last_modified_time'], self.last_file_modified_time) if partition_metadata and partition_metadata['last_modified_time'] else self.last_file_modified_time
if file_status_check_options and file_status_check_options['include_data_partition_size'] and partition_metadata and partition_metadata['file_size_bytes']:
self.exhibits_traits.append('is.partition.size', [['value', partition_metadata['file_size_bytes']]])
await self.report_most_recent_time_async(self.last_file_modified_time)

5
objectModel/Python/cdm/objectmodel/cdm_data_partition_pattern_def.py
Просмотреть файл

@ -8,6 +8,7 @@ import regex
from cdm.enums import CdmLogCode, CdmObjectType
from cdm.utilities import FileStatusCheckOptions, logger, ResolveOptions, StorageUtils, TraitToPropertyMap
from cdm.utilities.exceptions.cdm_read_partition_from_pattern_exception import CdmReadPartitionFromPatternException
from .cdm_file_status import CdmFileStatus
from .cdm_local_entity_declaration_def import CdmLocalEntityDeclarationDefinition
@ -126,6 +127,10 @@ class CdmDataPartitionPatternDefinition(CdmObjectDefinition, CdmFileStatus):
logger.warning(self.ctx, self._TAG, CdmDataPartitionPatternDefinition.file_status_check_async.__name__, self.at_corpus_path,
CdmLogCode.WARN_PARTITION_FILE_FETCH_FAILED, root_corpus, e)
if file_status_check_options is not None and file_status_check_options['throw_on_partition_error']:
message = 'There was an error fetching partitions from \'{}\', see the inner exception.'.format(root_corpus)
raise CdmReadPartitionFromPatternException(message) from e
# update modified times.
self.last_file_status_check_time = datetime.now(timezone.utc)

6
objectModel/Python/cdm/objectmodel/cdm_entity_attribute_def.py
Просмотреть файл

@ -114,7 +114,9 @@ class CdmEntityAttributeDefinition(CdmAttribute):
# A Projection
# if the max depth is exceeded it should not try to execute the projection
if not res_opt._depth_info.max_depth_exceeded:
if res_opt._depth_info.max_depth_exceeded:
logger.warning(self.ctx, self._TAG, self._construct_resolved_attributes.__name__, self.at_corpus_path, CdmLogCode.WARN_MAX_DEPTH_EXCEEDED, res_opt._depth_info.max_depth, self.entity.fetch_object_definition_name())
else:
proj_def = self.entity.fetch_object_definition(res_opt)
proj_directive = ProjectionDirective(res_opt, self, self.entity)
proj_ctx = proj_def._construct_projection_context(proj_directive, under)
@ -179,6 +181,8 @@ class CdmEntityAttributeDefinition(CdmAttribute):
# if we got here because of the max depth, need to impose the directives to make the trait work as expected
if res_opt._depth_info.max_depth_exceeded:
logger.warning(self.ctx, self._TAG, self._construct_resolved_attributes.__name__, self.at_corpus_path, CdmLogCode.WARN_MAX_DEPTH_EXCEEDED, res_opt._depth_info.max_depth, self.entity.fetch_object_definition_name())
if not arc.res_opt.directives:
arc.res_opt.directives = AttributeResolutionDirectiveSet()
arc.res_opt.directives.add('referenceOnly')

2
objectModel/Python/cdm/objectmodel/cdm_local_entity_declaration_def.py
Просмотреть файл

@ -138,7 +138,7 @@ class CdmLocalEntityDeclarationDefinition(CdmEntityDeclarationDefinition):
Constants._INCREMENTAL_TRAIT_NAME,
CdmLocalEntityDeclarationDefinition.data_partitions.fget.__name__)
else:
await partition.file_status_check_async()
await partition.file_status_check_async(file_status_check_options)
if partition_file_status_check_type == partition_file_status_check_type.INCREMENTAL or partition_file_status_check_type == PartitionFileStatusCheckType.FULL_AND_INCREMENTAL:
for pattern in self.incremental_partition_patterns:

12
objectModel/Python/cdm/objectmodel/cdm_manifest_def.py
Просмотреть файл

@ -113,7 +113,7 @@ class CdmManifestDefinition(CdmDocumentDefinition, CdmObjectDefinition, CdmFileS
return copy
async def create_resolved_manifest_async(self, new_manifest_name: str, new_entity_document_name_format: Optional[str], directives: Optional[AttributeResolutionDirectiveSet] = None) -> Optional['CdmManifestDefinition']:
async def create_resolved_manifest_async(self, new_manifest_name: str, new_entity_document_name_format: Optional[str], directives: Optional[AttributeResolutionDirectiveSet] = None, res_opt: Optional[ResolveOptions] = None) -> Optional['CdmManifestDefinition']:
"""Creates a resolved copy of the manifest.
new_entity_document_name_format specifies a pattern to use when creating documents for resolved entities.
The default is "resolved/{n}.cdm.json" to avoid a document name conflict with documents in the same folder as
@ -229,18 +229,20 @@ class CdmManifestDefinition(CdmDocumentDefinition, CdmObjectDefinition, CdmFileS
# next create the resolved entity.
with_directives = directives if directives is not None else self.ctx.corpus.default_resolution_directives
res_opt = ResolveOptions(ent_def.in_document, with_directives.copy())
entity_res_opt = res_opt.copy() if res_opt is not None else ResolveOptions()
entity_res_opt.wrt_doc = ent_def.in_document
entity_res_opt.directives = with_directives.copy()
logger.debug(self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path,
'resolving entity {} to document {}'.format(source_entity_full_path,
new_document_full_path))
resolved_entity = await ent_def.create_resolved_entity_async(ent_def.entity_name, res_opt, folder, new_document_name)
resolved_entity = await ent_def.create_resolved_entity_async(ent_def.entity_name, entity_res_opt, folder, new_document_name)
if not resolved_entity:
# fail all resolution, if any one entity resolution fails
return None
result = entity.copy(res_opt)
result = entity.copy(entity_res_opt)
if result.object_type == CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF:
relative_entity_path = self.ctx.corpus.storage.create_relative_corpus_path(resolved_entity.at_corpus_path, resolved_manifest)
result.entity_path = relative_entity_path or result.at_corpus_path
@ -250,7 +252,7 @@ class CdmManifestDefinition(CdmDocumentDefinition, CdmObjectDefinition, CdmFileS
logger.debug(self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path,
'calculating relationships')
# Calculate the entity graph for just this manifest.
await self.ctx.corpus.calculate_entity_graph_async(resolved_manifest)
await self.ctx.corpus.calculate_entity_graph_async(resolved_manifest, res_opt)
# Stick results into the relationships list for the manifest.
await resolved_manifest.populate_manifest_relationships_async(CdmRelationshipDiscoveryStyle.EXCLUSIVE)

3
objectModel/Python/cdm/objectmodel/cdm_object.py
Просмотреть файл

@ -5,9 +5,11 @@ import math
import abc
from threading import Lock
from typing import List, cast, Dict, Iterable, Optional, TYPE_CHECKING
from cdm.enums import CdmLogCode
from cdm.resolvedmodel import TraitProfile, TraitProfileCache
from cdm.enums import CdmObjectType
from cdm.utilities import logger
if TYPE_CHECKING:
from cdm.objectmodel import CdmCorpusContext, CdmDocumentDefinition, CdmEntityAttributeDefinition, \
@ -224,6 +226,7 @@ class CdmObject(abc.ABC):
if rasb_cache \
and res_opt._depth_info.max_depth \
and res_opt._depth_info.current_depth + rasb_cache._resolved_attribute_set._depth_traveled > res_opt._depth_info.max_depth:
logger.warning(self.ctx, self._TAG, self._fetch_resolved_attributes.__name__, self.at_corpus_path, CdmLogCode.WARN_MAX_DEPTH_EXCEEDED, res_opt._depth_info.max_depth, self.fetch_object_definition_name())
rasb_cache = None
if not rasb_cache:

2
objectModel/Python/cdm/resx/logmessages.txt
Просмотреть файл

@ -19,6 +19,7 @@ ERR_INVALID_PATH: Invalid path '{0}'
ERR_MANIFEST_FILE_MOD_TIME_FAILURE: Failed to compute last modified time for manifest file {0}. Exception {1}
ERR_MISSING_INCREMENTAL_PARTITION_TRAIT: This '{0}' object '{1}' does not contain the trait '{2}', so it should not be in the collection '{3}'.
ERR_OBJECT_WITHOUT_OWNER_FOUND: Found object without owner when calculating relationships.
ERR_PARTITION_FILE_METADATA_FAILURE: Failed to get file metadata for partition file {0}. Exception: {1}
ERR_PARTITION_FILE_MOD_TIME_FAILURE: Failed to compute last modified time for partition file {0}. Exception {1}
ERR_PATH_IS_DUPLICATE: Duplicate declaration for item '{0}'
ERR_PATH_NULL_OBJECT_PATH: The object path cannot be null or empty.
@ -145,6 +146,7 @@ ERR_REL_UNDEFINED: Trying to create relationship to an entity not defined
WARN_DEPRECATED_RESOLUTION_GUIDANCE: Resolution guidance is being deprecated in favor of Projections. https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#projection-overview.
WARN_PARTITION_GLOB_AND_REGEX_PRESENT: The Data Partition Pattern contains both a glob pattern ({0}) and a regular expression ({1}) set, the glob pattern will be used.
WARN_DOC_IMPORT_NOT_LOADED: Import document {0} not loaded. This may cause unexpected behavior of the model.
WARN_MAX_DEPTH_EXCEEDED: The entity max depth of {0} has been exceeded. Entity {1} will not be resolved any further.
WARN_PARTITION_FILE_FETCH_FAILED: Failed to fetch all files in the folder location '{0}' described by a partition pattern. Exception: {1}
WARN_LINK_ENT_IDENT_ARGS_NOT_SUPPORTED: Trait 'is.linkedEntity.identifier' is not accepting arguments, skipping relationship recording for attribute '{0}' in entity '{1}'.
WARN_PARTITION_INVALID_ARGUMENTS: Invalid set of arguments provided for data partition corresponding to location {0}

32
objectModel/Python/cdm/storage/adls.py
Просмотреть файл

@ -57,7 +57,7 @@ class ADLSAdapter(NetworkAdapter, StorageAdapterBase):
self._sas_token = None
self._unescaped_root_sub_path = None # type: Optional[str]
self._escaped_root_sub_path = None # type: Optional[str]
self._file_modified_time_cache = {} # type: Dict[str, datetime]
self._file_metadata_cache = {} # type: Dict[str, CdmFileMetadata]
self.http_max_results = self.HTTP_DEFAULT_MAX_RESULTS # type: int
self.timeout = self.ADLS_DEFAULT_TIMEOUT # type: int
@ -123,12 +123,20 @@ class ADLSAdapter(NetworkAdapter, StorageAdapterBase):
return True
def clear_cache(self) -> None:
self._file_modified_time_cache.clear()
self._file_metadata_cache.clear()
async def compute_last_modified_time_async(self, corpus_path: str) -> Optional[datetime]:
file_metadata = await self.fetch_file_metadata_async(corpus_path)
if file_metadata is None:
return None
return file_metadata['last_modified_time']
async def fetch_file_metadata_async(self, corpus_path: str) -> Optional[CdmFileMetadata]:
cachedValue = None
if self._is_cache_enabled:
cachedValue = self._file_modified_time_cache.get(corpus_path)
cachedValue = self._file_metadata_cache.get(corpus_path)
if cachedValue is not None:
return cachedValue
@ -139,10 +147,12 @@ class ADLSAdapter(NetworkAdapter, StorageAdapterBase):
cdm_response = await self._http_client._send_async(request, self.wait_time_callback, self.ctx)
if cdm_response.status_code == HTTPStatus.OK:
lastTime = dateutil.parser.parse(typing.cast(str, cdm_response.response_headers['Last-Modified']))
if lastTime is not None and self._is_cache_enabled:
self._file_modified_time_cache[corpus_path] = lastTime
return lastTime
file_size = int(cdm_response.response_headers['Content-Length'])
last_time = dateutil.parser.parse(typing.cast(str, cdm_response.response_headers['Last-Modified']))
file_metadata = { 'last_modified_time': last_time, 'file_size_bytes': file_size }
if self._is_cache_enabled:
self._file_metadata_cache[corpus_path] = file_metadata
return file_metadata
return None
@ -232,11 +242,13 @@ class ADLSAdapter(NetworkAdapter, StorageAdapterBase):
filepath = self._format_corpus_path(name_without_root_sub_path)
content_length = int(path['contentLength'])
results[filepath] = { 'file_size_bytes': content_length }
lastTimeString = path.get('lastModified')
file_metadata = { 'last_modified_time': dateutil.parser.parse(lastTimeString), 'file_size_bytes': content_length }
results[filepath] = file_metadata
if lastTimeString is not None and self._is_cache_enabled:
self._file_modified_time_cache[filepath] = dateutil.parser.parse(lastTimeString)
self._file_metadata_cache[filepath] = file_metadata
if continuation_token is None:
break

10
objectModel/Python/cdm/storage/base.py
Просмотреть файл

@ -64,6 +64,10 @@ class StorageAdapterBase:
"""Return last modified time of specified document."""
return datetime.datetime.now()
async def fetch_file_metadata_async(self, corpus_path: str) -> Optional[CdmFileMetadata]:
"""Returns the file metadata info about the specified document"""
return None
async def fetch_all_files_async(self, folder_corpus_path: str) -> List[str]:
"""Deprecated: Deprecated in favor of fetch_all_files_metadata_async. Return list of corpus paths to all files and folders under the specified corpus path to
a folder.
@ -101,3 +105,9 @@ class StorageAdapterBase:
self.clear_cache()
cache_context.dispose = dispose
return cache_context
def fetch_config(self):
return None
def update_config(self, config: str) -> None:
return

3
objectModel/Python/cdm/storage/cdm_standards.py
Просмотреть файл

@ -16,3 +16,6 @@ class CdmStandardsAdapter(CdmCustomPackageAdapter):
# --- internal ---
self._type = 'cdm-standards'
def fetch_config(self):
return '{"config":{},"type":"cdm-standards"}'

13
objectModel/Python/cdm/storage/local.py
Просмотреть файл

@ -63,10 +63,19 @@ class LocalAdapter(StorageAdapterBase):
return None
async def compute_last_modified_time_async(self, corpus_path: str) -> Optional[datetime]:
file_metadata = await self.fetch_file_metadata_async(corpus_path)
if file_metadata is None:
return None
return file_metadata['last_modified_time']
async def fetch_file_metadata_async(self, corpus_path: str) -> Optional[CdmFileMetadata]:
adapter_path = self.create_adapter_path(corpus_path)
if os.path.exists(adapter_path):
file_size = os.path.getsize(adapter_path)
modified_time = datetime.fromtimestamp(os.path.getmtime(adapter_path))
return modified_time.replace(tzinfo=timezone.utc)
return { 'last_modified_time': modified_time.replace(tzinfo=timezone.utc), 'file_size_bytes': file_size }
return None
async def fetch_all_files_async(self, folder_corpus_path: str) -> List[str]:
@ -83,7 +92,7 @@ class LocalAdapter(StorageAdapterBase):
for file_name in file_names:
path = self.create_adapter_path(file_name)
if os.path.exists(path):
file_metadatas[file_name] = {'file_size_bytes': os.path.getsize(path) if os.path.isfile else None}
file_metadatas[file_name] = {'file_size_bytes': os.path.getsize(path) if os.path.isfile else None, 'last_modified_time': datetime.fromtimestamp(os.path.getmtime(path))}
return file_metadatas

4
objectModel/Python/cdm/storage/storage_manager.py
Просмотреть файл

@ -292,13 +292,13 @@ class StorageManager:
configs = None
# Check whether the config exists.
if item.get('config'):
if item.get('config') is not None:
configs = item['config']
else:
logger.error(self._ctx, self._TAG, StorageManager.mount_from_config.__name__, None, CdmLogCode.ERR_STORAGE_MISSING_JSON_CONFIG, namespace)
continue
if not item.get('type'):
if item.get('type') is None:
logger.error(self._ctx, self._TAG, StorageManager.mount_from_config.__name__, None, CdmLogCode.ERR_STORAGE_MISSING_TYPE_JSON_CONFIG, namespace)
continue

2
objectModel/Python/cdm/utilities/__init__.py
Просмотреть файл

@ -11,13 +11,13 @@ from .applier_state import ApplierState
from .attribute_context_parameters import AttributeContextParameters
from .attribute_resolution_applier import AttributeResolutionApplier
from .attribute_resolution_directive_set import AttributeResolutionDirectiveSet
from .exceptions.cdm_error import CdmError
from .cdm_file_metadata import CdmFileMetadata
from .copy_options import CopyOptions
from .constants import Constants
from .depth_info import DepthInfo
from .docs_result import DocsResult
from .event_callback import EventCallback
from .exceptions import CdmError
from .file_status_check_options import FileStatusCheckOptions
from .friendly_format_node import FriendlyFormatNode
from .identifier_ref import IdentifierRef

6
objectModel/Python/cdm/utilities/cdm_file_metadata.py
Просмотреть файл

@ -1,9 +1,13 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.
from typing import Optional
from typing import Optional, TYPE_CHECKING
if TYPE_CHECKING:
from datetime import datetime
class CdmFileMetadata:
def __init__(self) -> None:
self.file_size_bytes = None # type: Optional[int]
self.last_modified_time = None # type: Optional[datetime]

2
objectModel/Python/cdm/utilities/exceptions/__init__.py Normal file
Просмотреть файл

@ -0,0 +1,2 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.

0
objectModel/Python/cdm/utilities/exceptions.py → objectModel/Python/cdm/utilities/exceptions/cdm_error.py
Просмотреть файл

5
objectModel/Python/cdm/utilities/exceptions/cdm_read_partition_from_pattern_exception.py Normal file
Просмотреть файл

@ -0,0 +1,5 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.
class CdmReadPartitionFromPatternException(Exception):
"""CDM exception when partitions is not read properly from pattern"""

2
objectModel/Python/cdm/utilities/file_status_check_options.py
Просмотреть файл

@ -9,3 +9,5 @@ class FileStatusCheckOptions:
self.include_data_partition_size = None # type: Optional[bool]
self.regex_timeout_seconds = None # type: Optional[float]
self.throw_on_partition_error = None # type: Optional[bool]

1
objectModel/Python/cdm/utilities/resolve_options.py
Просмотреть файл

@ -114,6 +114,7 @@ class ResolveOptions:
def copy(self) -> 'ResolveOptions':
res_opt_copy = ResolveOptions()
res_opt_copy.wrt_doc = self.wrt_doc
res_opt_copy.max_depth = self.max_depth
res_opt_copy._depth_info = self._depth_info._copy()
res_opt_copy._in_circular_reference = self._in_circular_reference
res_opt_copy._localize_references_for = self._localize_references_for

2
objectModel/Python/setup.py
Просмотреть файл

@ -77,7 +77,7 @@ class CopyResourcesCommand(distutils.cmd.Command):
setuptools.setup(
name='commondatamodel-objectmodel',
version='1.7.5',
version='1.7.6',
author='Microsoft',
description='Common Data Model Object Model library for Python',
url='https://github.com/microsoft/CDM/tree/master/objectModel/Python',

2
objectModel/Python/tests/cdm/data_partition/__init__.py Normal file
Просмотреть файл

@ -0,0 +1,2 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.

30
objectModel/Python/tests/cdm/data_partition/test_data_partition.py Normal file
Просмотреть файл

@ -0,0 +1,30 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.
import os
import unittest
from tests.common import async_test, TestHelper
class DataPartitionTest(unittest.TestCase):
test_subpath = os.path.join('Cdm', 'DataPartition')
@async_test
async def test_refreshes_data_partition(self):
"""Tests refreshing files that match the regular expression"""
test_name = 'test_refreshes_data_partition'
cdm_corpus = TestHelper.get_local_corpus(self.test_subpath, test_name)
cdmManifest = await cdm_corpus.fetch_object_async('local:/partitions.manifest.cdm.json')
file_status_check_options = {'include_data_partition_size': True}
partition_entity = cdmManifest.entities[0]
self.assertEqual(len(partition_entity.data_partitions), 1)
partition = partition_entity.data_partitions[0]
await cdmManifest.file_status_check_async(file_status_check_options=file_status_check_options)
local_trait_index = partition.exhibits_traits.index('is.partition.size')
self.assertNotEqual(local_trait_index, -1)
localTrait = partition.exhibits_traits[local_trait_index]
self.assertEqual(localTrait.named_reference, 'is.partition.size')
self.assertEqual(localTrait.arguments[0].value, 2)

45
objectModel/Python/tests/cdm/data_partition_pattern/test_data_partition_pattern.py
Просмотреть файл

@ -5,6 +5,7 @@ from cdm.storage.local import LocalAdapter
from datetime import datetime, timedelta, timezone
import os
import time
import traceback
import unittest
from cdm.objectmodel import CdmManifestDefinition, CdmLocalEntityDeclarationDefinition, CdmTraitDefinition, CdmParameterDefinition, CdmTraitReference
@ -15,9 +16,11 @@ from cdm.persistence.cdmfolder.types import ManifestContent
from cdm.objectmodel.cdm_corpus_context import CdmCorpusContext
from cdm.enums import CdmStatusLevel, PartitionFileStatusCheckType, CdmIncrementalPartitionType, CdmObjectType, \
CdmLogCode
from cdm.utilities.exceptions.cdm_read_partition_from_pattern_exception import CdmReadPartitionFromPatternException
from tests.adls_test_helper import AdlsTestHelper
from tests.common import async_test, TestHelper
from tests.storage.testAdapters.fetch_all_metadata import FetchAllMetadataNullAdapter
from tests.storage.testAdapters.fetch_all_metadata_throw_error import FetchAllMetadataThrowErrorAdapter
from tests.storage.testAdapters.no_override import NoOverride
from tests.storage.testAdapters.override_fetch_all_files import OverrideFetchAllFiles
@ -770,4 +773,44 @@ class DataPartitionPatternTest(unittest.TestCase):
file_status_check__options = {'regex_timeout_seconds': 0}
manifest = await corpus.fetch_object_async('manifest.manifest.cdm.json')
await manifest.file_status_check_async(file_status_check_options=file_status_check__options)
await manifest.file_status_check_async(file_status_check_options=file_status_check__options)
@async_test
async def test_throw_on_partition_error(self):
expected_log_codes = {CdmLogCode.WARN_PARTITION_FILE_FETCH_FAILED}
corpus = TestHelper.get_local_corpus(self.test_subpath, 'TestFetchAllFilesMetadata', expected_codes=expected_log_codes)
test_local_adapter = corpus.storage.namespace_adapters[corpus.storage.default_namespace]
corpus.storage.mount('error', FetchAllMetadataThrowErrorAdapter(test_local_adapter))
file_status_check_options = { 'throw_on_partition_error': True }
manifestThrowsError = False
entityDecThrowsError = False
partitionPatternThrowsError = False
manifest = await corpus.fetch_object_async('error:/manifest.manifest.cdm.json')
try:
await manifest.file_status_check_async(file_status_check_options=file_status_check_options)
except CdmReadPartitionFromPatternException as e:
error_message = traceback.format_exc()
self.assertTrue('Some test error message.' in error_message)
manifestThrowsError = True
entity_dec = manifest.entities[0]
try:
await entity_dec.file_status_check_async(file_status_check_options=file_status_check_options)
except CdmReadPartitionFromPatternException as e:
error_message = traceback.format_exc()
self.assertTrue('Some test error message.' in error_message)
entityDecThrowsError = True
partition_pattern = manifest.entities[0].data_partition_patterns[0]
try:
await partition_pattern.file_status_check_async(file_status_check_options=file_status_check_options)
except CdmReadPartitionFromPatternException as e:
error_message = traceback.format_exc()
self.assertTrue('Some test error message.' in error_message)
partitionPatternThrowsError = True
self.assertTrue(manifestThrowsError and entityDecThrowsError and partitionPatternThrowsError)

5
objectModel/Python/tests/cdm/projection/test_projection_miscellaneous.py
Просмотреть файл

@ -5,6 +5,7 @@ import os
import unittest
from cdm.enums import CdmStatusLevel, CdmObjectType
from cdm.enums import CdmLogCode
from cdm.objectmodel import CdmCorpusDefinition, CdmEntityAttributeDefinition, CdmEntityDefinition, CdmEntityReference, CdmProjection, CdmTypeAttributeDefinition
from cdm.objectmodel.projections.cardinality_settings import CardinalitySettings
from cdm.utilities import ResolveOptions, AttributeResolutionDirectiveSet
@ -169,7 +170,7 @@ class ProjectionMiscellaneousTest(unittest.TestCase):
test_name = 'test_max_depth_on_polymorphic_entity'
entity_name = 'A'
corpus = ProjectionTestUtils.get_local_corpus(self.tests_subpath, test_name) # type: CdmCorpusDefinition
corpus = ProjectionTestUtils.get_local_corpus(self.tests_subpath, test_name, [CdmLogCode.WARN_MAX_DEPTH_EXCEEDED]) # type: CdmCorpusDefinition
entity = await corpus.fetch_object_async('{0}.cdm.json/{0}'.format(entity_name)) # type: CdmEntityDefinition
@ -178,7 +179,7 @@ class ProjectionMiscellaneousTest(unittest.TestCase):
res_entity = await entity.create_resolved_entity_async('resolved-{}'.format(entity_name), res_opt) # type: CdmEntityDefinition
self.assertIsNotNone(res_entity)
self.assertEqual(4, len(res_entity.attributes))
self.assertEqual(2, len(res_entity.attributes))
def test_type_attribute_source(self):
"""Tests if setting the projection "source" on a type attribute triggers an error log"""

5
objectModel/Python/tests/cdm/resolution/test_cache.py
Просмотреть файл

@ -3,6 +3,7 @@
import os
import unittest
from cdm.enums import CdmLogCode
from tests.common import async_test, TestHelper
from cdm.utilities import AttributeResolutionDirectiveSet, ResolveOptions
@ -13,7 +14,7 @@ class Cache(unittest.TestCase):
@async_test
async def test_max_depth_cached(self):
cdm_corpus = TestHelper.get_local_corpus(self.tests_subpath, self.test_name_path)
cdm_corpus = TestHelper.get_local_corpus(self.tests_subpath, self.test_name_path, expected_codes=[CdmLogCode.WARN_MAX_DEPTH_EXCEEDED])
a_ent = await cdm_corpus .fetch_object_async('A.cdm.json/A')
b_ent = await cdm_corpus .fetch_object_async('B.cdm.json/B')
c_ent = await cdm_corpus .fetch_object_async('C.cdm.json/C')
@ -46,7 +47,7 @@ class Cache(unittest.TestCase):
@async_test
async def test_non_max_depth_cached(self):
cdm_corpus = TestHelper.get_local_corpus(self.tests_subpath, self.test_name_path)
cdm_corpus = TestHelper.get_local_corpus(self.tests_subpath, self.test_name_path, expected_codes=[CdmLogCode.WARN_MAX_DEPTH_EXCEEDED])
a_ent = await cdm_corpus.fetch_object_async('A.cdm.json/A')
b_ent = await cdm_corpus.fetch_object_async('B.cdm.json/B')
c_ent = await cdm_corpus.fetch_object_async('C.cdm.json/C')

11
objectModel/Python/tests/cdm/resolution/test_manifest_resolution.py
Просмотреть файл

@ -9,6 +9,7 @@ from cdm.enums import CdmStatusLevel, CdmLogCode, CdmObjectType
from cdm.objectmodel import CdmCorpusDefinition, CdmManifestDefinition, CdmReferencedEntityDeclarationDefinition
from cdm.persistence import PersistenceLayer
from cdm.storage import LocalAdapter
from cdm.utilities import resolve_options
from tests.common import async_test, TestHelper
@ -101,16 +102,16 @@ class ManifestResolution(unittest.TestCase):
"""
Test that manifest containing entities having dependency on each other for polymorphic sources resolves.
"""
cdm_corpus = CdmCorpusDefinition()
cdm_corpus = TestHelper.get_local_corpus(self.tests_subpath, 'test_resolve_manifest_with_interdependent_polymorphic_source')
cdm_corpus = TestHelper.get_local_corpus(self.tests_subpath, 'test_resolve_manifest_with_interdependent_polymorphic_source', expected_codes=[CdmLogCode.WARN_MAX_DEPTH_EXCEEDED])
manifest = await cdm_corpus.fetch_object_async('local:/Input.manifest.cdm.json')
resolved_manifest = await manifest.create_resolved_manifest_async('resolved', None)
res_opt = resolve_options.ResolveOptions()
res_opt.max_depth = 3
resolved_manifest = await manifest.create_resolved_manifest_async('resolved', None, None, res_opt)
self.assertEqual(2, len(resolved_manifest.entities))
self.assertEqual('resolved/group.cdm.json/group',
resolved_manifest.entities[0].entity_path.lower())
self.assertEqual('resolved/groupmember.cdm.json/groupmember',
resolved_manifest.entities[1].entity_path.lower())
resolved_manifest.entities[1].entity_path.lower())

19
objectModel/Python/tests/cdm/test_corpus.py
Просмотреть файл

@ -144,6 +144,7 @@ class CorpusTests(unittest.TestCase):
wrt_entity = await corpus.fetch_object_async('local:/wrtConstEntity.cdm.json/wrtConstEntity') # type: CdmEntityDefinition
res_opt = ResolveOptions(wrt_entity, AttributeResolutionDirectiveSet())
await wrt_entity.create_resolved_entity_async('NewEntity', res_opt)
@async_test
async def test_incorrect_cast_on_fetch(self):
"""Tests that errors when trying to cast objects after fetching is handled correctly."""
@ -154,3 +155,21 @@ class CorpusTests(unittest.TestCase):
await corpus.calculate_entity_graph_async(manifest)
TestHelper.assert_cdm_log_code_equality(corpus, CdmLogCode.ERR_INVALID_CAST, True, self)
@async_test
async def test_max_depth_exceeded_resolution_guidance(self):
"""Test warning correctly logged when max depth is exceeded for Resolution Guidance"""
corpus = TestHelper.get_local_corpus(self.tests_subpath, 'test_max_depth_exceeded_resolution_guidance', expected_codes=[CdmLogCode.WARN_MAX_DEPTH_EXCEEDED])
entity = await corpus.fetch_object_async('local:/firstEntity.cdm.json/firstEntity')
await entity.create_resolved_entity_async('resFirstEntity')
TestHelper.assert_cdm_log_code_equality(corpus, CdmLogCode.WARN_MAX_DEPTH_EXCEEDED, True, self)
@async_test
async def test_max_depth_exceeded_projections(self):
"""Test warning correctly logged when max depth is exceeded for Projections"""
corpus = TestHelper.get_local_corpus(self.tests_subpath, 'test_max_depth_exceeded_projections', expected_codes=[CdmLogCode.WARN_MAX_DEPTH_EXCEEDED])
entity = await corpus.fetch_object_async('local:/A.cdm.json/A')
await entity.create_resolved_entity_async('resA')
TestHelper.assert_cdm_log_code_equality(corpus, CdmLogCode.WARN_MAX_DEPTH_EXCEEDED, True, self)

2
objectModel/Python/tests/persistence/cdmfolder/data_partition_pattern/test_data_partition_pattern.py
Просмотреть файл

@ -79,7 +79,7 @@ class DataPartitionPatternTest(unittest.TestCase):
self.assertEqual(status_level, CdmStatusLevel.ERROR, 'Error level message should have been reported')
self.assertTrue(
(message == 'StorageManager | The object path cannot be null or empty. | create_absolute_corpus_path') or \
(message == 'CdmCorpusDefinition | The object path cannot be null or empty. | _get_last_modified_time_from_partition_path_async'),
(message == 'CdmCorpusDefinition | The object path cannot be null or empty. | _get_file_metadata_from_partition_path_async'),
"Unexpected error message received"
)

14
objectModel/Python/tests/storage/testAdapters/fetch_all_metadata_throw_error.py Normal file
Просмотреть файл

@ -0,0 +1,14 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.
from typing import List
from .no_override import NoOverride
class FetchAllMetadataThrowErrorAdapter(NoOverride):
def __init__(self, base_adapter) -> None:
super().__init__(base_adapter)
async def fetch_all_files_metadata_async(self, folder_corpus_path: str) -> List[str]:
raise Exception('Some test error message.')

22
objectModel/Python/tests/storage/test_adls.py
Просмотреть файл

@ -568,6 +568,28 @@ class AdlsStorageAdapterTestCase(unittest.TestCase):
await some_doc.save_as_async('someDoc.cdm.json')
self.assertTrue(uploaded_data_not_accepted_error)
@async_test
@unittest.skipIf(not AdlsTestHelper.is_adls_env_enabled(), "ADLS environment variables not set up")
async def test_adls_refreshes_data_partition(self):
adls_adapter = AdlsTestHelper.create_adapter_with_shared_key()
corpus = CdmCorpusDefinition()
corpus.storage.mount('adls', adls_adapter)
cdmManifest = await corpus.fetch_object_async('adls:/TestPartitionMetadata/partitions.manifest.cdm.json');
file_status_check_options = {'include_data_partition_size': True}
partitionEntity = cdmManifest.entities[0]
self.assertEqual(len(partitionEntity.data_partitions), 1)
partition = partitionEntity.data_partitions[0]
await cdmManifest.file_status_check_async(file_status_check_options=file_status_check_options)
local_trait_index = partition.exhibits_traits.index('is.partition.size')
self.assertNotEqual(local_trait_index, -1)
local_trait = partition.exhibits_traits[local_trait_index]
self.assertEqual(local_trait.named_reference, 'is.partition.size')
self.assertEqual(local_trait.arguments[0].value, 2)
if __name__ == '__main__':
unittest.main()

17
objectModel/Python/tests/storage/test_cdm_custom_package.py
Просмотреть файл

@ -5,6 +5,8 @@ import os
import unittest
from tests.common import async_test
from cdm.enums import CdmStatusLevel
from cdm.objectmodel import CdmCorpusDefinition
from cdm.storage import CdmCustomPackageAdapter
from cdm.storage import CdmStandardsAdapter
import commondatamodel_objectmodel_cdmstandards as cdmstandards
@ -72,3 +74,18 @@ class CdmCustomPackageAdapterTests(unittest.TestCase):
"""Tests if the CdmCustomPackageAdapter works when assembly is passed in the constructor"""
adapter = CdmCustomPackageAdapter(cdmstandards, 'schema_documents')
self.assertIsNotNone(await adapter.read_async(self.FOUNDATIONS_FILE_PATH))
@async_test
async def test_cdmstandards_mount_from_config(self):
"""Test mounting CdmStandards adapter from config does not cause an error"""
corpus = CdmCorpusDefinition()
def callback(level: CdmStatusLevel, message: str):
self.fail()
corpus.set_event_callback(callback, CdmStatusLevel.WARNING)
corpus.storage.mount_from_config("{\"adapters\": [{\"config\": {\"locationHint\": \"\", \"maximumTimeout\": 20000, \"numberOfRetries\": 2, \"root\": \"/logical\", \"timeout\": 5000}, \"namespace\": \"cdm\", \"type\": \"cdm-standards\"}], \"defaultNamespace\": \"local\"}")
corpus.storage.mount('cdm', CdmStandardsAdapter())
config = corpus.storage.fetch_config()
corpus.storage.mount_from_config(config)

6
objectModel/Python/tests/utilities/projection_test_utils.py
Просмотреть файл

@ -82,13 +82,15 @@ class ProjectionTestUtils:
return resolved_entity
@staticmethod
def get_local_corpus(tests_subpath: str, test_name: str) -> 'CdmCorpusDefinition':
def get_local_corpus(tests_subpath: str, test_name: str, expected_codes: Optional[set] = None) -> 'CdmCorpusDefinition':
"""Creates a corpus"""
corpus = TestHelper.get_local_corpus(tests_subpath, test_name)
corpus = TestHelper.get_local_corpus(tests_subpath, test_name, expected_codes=expected_codes)
def callback(level: CdmStatusLevel, message: str):
last_event = corpus.ctx.events[-1]
if not last_event.get('code') or last_event['code'] not in ProjectionTestUtils.allowed_logs:
if expected_codes is not None and CdmLogCode[last_event['code']] in expected_codes:
return
raise Exception(message)
corpus.set_event_callback(callback, CdmStatusLevel.WARNING)

Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше