microsoft
/
bc2adls
зеркало из https://github.com/microsoft/bc2adls.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Add TableCaption/Entity to Telemetry Logs (#66) 

* Add TableCaption/Entity to Telemetry Logs

At the moment the Table Caption is only used in the Start-Export Event. 
We have no solution to see when the export for a given table started and stopped. With this change you can:
1. See when the export for a given Table started
2. See how many datasets to export are found
3.. See when the export for a given Table finished

This way you can monitor the mean export time per Table and can react to slower exports or find missing SystemModifiedAt keys.

* Add more detailed Log to ExportTableDeletes

* Increase App version

* Fix EntityCount Rec in ExportTableDeletes
This commit is contained in:
JulianSchmidtke 2022-12-09 11:23:37 +01:00 коммит произвёл GitHub
Родитель 30b1d45145
Коммит f80f504716
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
2 изменённых файлов: 38 добавлений и 13 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

2
businessCentral/app.json
Просмотреть файл

@ -4,7 +4,7 @@
"publisher": "The bc2adls team, Microsoft Denmark",
"brief": "Sync data from Business Central to the Azure storage",
"description": "Exports data in chosen tables to the Azure Data Lake and keeps it in sync by incremental updates. Before you use this tool, please read the SUPPORT.md file at https://github.com/microsoft/bc2adls.",
"version": "1.3.2.1",
"version": "1.3.2.2",
"privacyStatement": "https://go.microsoft.com/fwlink/?LinkId=724009",
"EULA": "https://go.microsoft.com/fwlink/?linkid=2009120",
"help": "https://go.microsoft.com/fwlink/?LinkId=724011",

49
businessCentral/src/ADLSEExecute.Codeunit.al
Просмотреть файл

@ -15,6 +15,7 @@ codeunit 82561 "ADLSE Execute"
ADLSEExecution: Codeunit "ADLSE Execution";
ADLSEUtil: Codeunit "ADLSE Util";
CustomDimensions: Dictionary of [Text, Text];
TableCaption: Text;
UpdatedLastTimestamp: BigInteger;
DeletedLastEntryNo: BigInteger;
OldUpdatedLastTimestamp: BigInteger;
@ -25,8 +26,10 @@ codeunit 82561 "ADLSE Execute"
ADLSESetup.GetSingleton();
EmitTelemetry := ADLSESetup."Emit telemetry";
CDMDataFormat := ADLSESetup.DataFormat;
if EmitTelemetry then begin
CustomDimensions.Add('Entity', ADLSEUtil.GetTableCaption(Rec."Table ID"));
TableCaption := ADLSEUtil.GetTableCaption(Rec."Table ID");
CustomDimensions.Add('Entity', TableCaption);
ADLSEExecution.Log('ADLSE-017', 'Starting the export for table', Verbosity::Normal, CustomDimensions);
end;
@ -41,7 +44,6 @@ codeunit 82561 "ADLSE Execute"
DeletedLastEntryNo := ADLSETableLastTimestamp.GetDeletedLastEntryNo(Rec."Table ID");
if EmitTelemetry then begin
Clear(CustomDimensions);
CustomDimensions.Add('Old Updated Last time stamp', Format(UpdatedLastTimestamp));
CustomDimensions.Add('Old Deleted Last entry no.', Format(DeletedLastEntryNo));
ADLSEExecution.Log('ADLSE-004', 'Exporting with parameters', Verbosity::Verbose, CustomDimensions);
@ -56,6 +58,7 @@ codeunit 82561 "ADLSE Execute"
end;
if EmitTelemetry then begin
Clear(CustomDimensions);
CustomDimensions.Add('Entity', TableCaption);
CustomDimensions.Add('Updated Last time stamp', Format(UpdatedLastTimestamp));
CustomDimensions.Add('Deleted Last entry no.', Format(DeletedLastEntryNo));
CustomDimensions.Add('Entity Json needs update', Format(EntityJsonNeedsUpdate));
@ -74,8 +77,11 @@ codeunit 82561 "ADLSE Execute"
SetStateFinished(Rec);
exit;
end;
if EmitTelemetry then
ADLSEExecution.Log('ADLSE-006', 'Saved the timestamps into the database', Verbosity::Normal);
if EmitTelemetry then begin
Clear(CustomDimensions);
CustomDimensions.Add('Entity', TableCaption);
ADLSEExecution.Log('ADLSE-006', 'Saved the timestamps into the database', Verbosity::Normal, CustomDimensions);
end;
Commit(); // to save the last time stamps into the database.
end;
@ -85,12 +91,12 @@ codeunit 82561 "ADLSE Execute"
exit;
end;
if EmitTelemetry then
ADLSEExecution.Log('ADLSE-007', 'Jsons have been updated', Verbosity::Normal);
ADLSEExecution.Log('ADLSE-007', 'Jsons have been updated', Verbosity::Normal, CustomDimensions);
// Finalize
SetStateFinished(Rec);
if EmitTelemetry then
ADLSEExecution.Log('ADLSE-005', 'Export completed without error', Verbosity::Normal);
ADLSEExecution.Log('ADLSE-005', 'Export completed without error', Verbosity::Normal, CustomDimensions);
end;
var
@ -144,6 +150,9 @@ codeunit 82561 "ADLSE Execute"
ADLSEExecution: Codeunit "ADLSE Execution";
Rec: RecordRef;
TimeStampField: FieldRef;
CustomDimensions: Dictionary of [Text, Text];
TableCaption: Text;
EntityCount: Text;
FlushedTimeStamp: BigInteger;
FieldId: Integer;
begin
@ -156,8 +165,14 @@ codeunit 82561 "ADLSE Execute"
Error(InsufficientReadPermErr);
if ADLSESeekData.FindRecords(Rec) then begin
if EmitTelemetry then
ADLSEExecution.Log('ADLSE-021', 'Updated records found', Verbosity::Verbose);
if EmitTelemetry then begin
TableCaption := Rec.Caption();
EntityCount := Format(Rec.Count());
CustomDimensions.Add('Entity', TableCaption);
CustomDimensions.Add('Entity Count', EntityCount);
ADLSEExecution.Log('ADLSE-021', 'Updated records found', Verbosity::Normal, CustomDimensions);
end;
repeat
if ADLSECommunication.TryCollectAndSendRecord(Rec, TimeStampField.Value(), FlushedTimeStamp) then
UpdatedLastTimeStamp := FlushedTimeStamp
@ -197,14 +212,24 @@ codeunit 82561 "ADLSE Execute"
ADLSEUtil: Codeunit "ADLSE Util";
ADLSEExecution: Codeunit "ADLSE Execution";
Rec: RecordRef;
CustomDimensions: Dictionary of [Text, Text];
TableCaption: Text;
EntityCount: Text;
FlushedTimeStamp: BigInteger;
begin
SetFilterForDeletes(TableID, DeletedLastEntryNo, ADLSEDeletedRecord);
if ADLSESeekData.FindRecords(ADLSEDeletedRecord) then begin
if EmitTelemetry then
ADLSEExecution.Log('ADLSE-010', 'Deleted records found', Verbosity::Verbose);
Rec.Open(ADLSEDeletedRecord."Table ID");
if EmitTelemetry then begin
TableCaption := Rec.Caption();
EntityCount := Format(ADLSEDeletedRecord.Count());
CustomDimensions.Add('Entity', TableCaption);
CustomDimensions.Add('Entity Count', EntityCount);
ADLSEExecution.Log('ADLSE-010', 'Deleted records found', Verbosity::Normal, CustomDimensions);
end;
repeat
ADLSEUtil.CreateFakeRecordForDeletedAction(ADLSEDeletedRecord, Rec);
if ADLSECommunication.TryCollectAndSendRecord(Rec, ADLSEDeletedRecord."Entry No.", FlushedTimeStamp) then
@ -219,7 +244,7 @@ codeunit 82561 "ADLSE Execute"
Error('%1%2', GetLastErrorText(), GetLastErrorCallStack());
end;
if EmitTelemetry then
ADLSEExecution.Log('ADLSE-011', 'Deleted records exported', Verbosity::Verbose);
ADLSEExecution.Log('ADLSE-011', 'Deleted records exported', Verbosity::Normal, CustomDimensions);
end;
local procedure CreateFieldListForTable(TableID: Integer) FieldIdList: List of [Integer]
@ -267,4 +292,4 @@ codeunit 82561 "ADLSE Execute"
ADLSERun.RegisterEnded(ADLSETableIDRunning, EmitTelemetry);
ADLSECurrentSession.Stop(ADLSETableIDRunning);
end;
}
}