microsoft
/
fhir-server
зеркало из https://github.com/microsoft/fhir-server.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Remove +1 assumption for input versions in $import (#3685) 

* Remove +1 assumption for input versions in $import

* spelling

* order

* Abilty to add meta

* time generation

* better time

* var name

* _ => x
This commit is contained in:
SergeyGaluzo 2024-01-30 13:48:25 -08:00 коммит произвёл GitHub
Родитель 53f5153126
Коммит 3046bd0c0f
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: B5690EEEBB952194
12 изменённых файлов: 5942 добавлений и 16 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

415
src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/74.diff.sql Normal file
Просмотреть файл

@ -0,0 +1,415 @@
ALTER PROCEDURE dbo.MergeResources
-- This stored procedure can be used for:
-- 1. Ordinary put with single version per resource in input
-- 2. Put with history preservation (multiple input versions per resource)
-- 3. Copy from one gen2 store to another with ResourceSurrogateId preserved.
@AffectedRows int = 0 OUT
,@RaiseExceptionOnConflict bit = 1
,@IsResourceChangeCaptureEnabled bit = 0
,@TransactionId bigint = NULL
,@SingleTransaction bit = 1
,@Resources dbo.ResourceList READONLY
,@ResourceWriteClaims dbo.ResourceWriteClaimList READONLY
,@ReferenceSearchParams dbo.ReferenceSearchParamList READONLY
,@TokenSearchParams dbo.TokenSearchParamList READONLY
,@TokenTexts dbo.TokenTextList READONLY
,@StringSearchParams dbo.StringSearchParamList READONLY
,@UriSearchParams dbo.UriSearchParamList READONLY
,@NumberSearchParams dbo.NumberSearchParamList READONLY
,@QuantitySearchParams dbo.QuantitySearchParamList READONLY
,@DateTimeSearchParms dbo.DateTimeSearchParamList READONLY
,@ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY
,@TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY
,@TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY
,@TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY
,@TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY
,@TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY
AS
set nocount on
DECLARE @st datetime = getUTCdate()
,@SP varchar(100) = object_name(@@procid)
,@DummyTop bigint = 9223372036854775807
,@InitialTranCount int = @@trancount
,@IsRetry bit = 0
DECLARE @Mode varchar(200) = isnull((SELECT 'RT=['+convert(varchar,min(ResourceTypeId))+','+convert(varchar,max(ResourceTypeId))+'] Sur=['+convert(varchar,min(ResourceSurrogateId))+','+convert(varchar,max(ResourceSurrogateId))+'] V='+convert(varchar,max(Version))+' Rows='+convert(varchar,count(*)) FROM @Resources),'Input=Empty')
SET @Mode += ' E='+convert(varchar,@RaiseExceptionOnConflict)+' CC='+convert(varchar,@IsResourceChangeCaptureEnabled)+' IT='+convert(varchar,@InitialTranCount)+' T='+isnull(convert(varchar,@TransactionId),'NULL')
SET @AffectedRows = 0
BEGIN TRY
DECLARE @Existing AS TABLE (ResourceTypeId smallint NOT NULL, SurrogateId bigint NOT NULL PRIMARY KEY (ResourceTypeId, SurrogateId))
DECLARE @ResourceInfos AS TABLE
(
ResourceTypeId smallint NOT NULL
,SurrogateId bigint NOT NULL
,Version int NOT NULL
,KeepHistory bit NOT NULL
,PreviousVersion int NULL
,PreviousSurrogateId bigint NULL
PRIMARY KEY (ResourceTypeId, SurrogateId)
)
DECLARE @PreviousSurrogateIds AS TABLE (TypeId smallint NOT NULL, SurrogateId bigint NOT NULL PRIMARY KEY (TypeId, SurrogateId), KeepHistory bit)
IF @SingleTransaction = 0 AND isnull((SELECT Number FROM dbo.Parameters WHERE Id = 'MergeResources.NoTransaction.IsEnabled'),0) = 0
SET @SingleTransaction = 1
SET @Mode += ' ST='+convert(varchar,@SingleTransaction)
-- perform retry check in transaction to hold locks
IF @InitialTranCount = 0
BEGIN
IF EXISTS (SELECT * -- This extra statement avoids putting range locks when we don't need them
FROM @Resources A JOIN dbo.Resource B ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId
--WHERE B.IsHistory = 0 -- With this clause wrong plans are created on empty/small database. Commented until resource separation is in place.
)
BEGIN
BEGIN TRANSACTION
INSERT INTO @Existing
( ResourceTypeId, SurrogateId )
SELECT B.ResourceTypeId, B.ResourceSurrogateId
FROM (SELECT TOP (@DummyTop) * FROM @Resources) A
JOIN dbo.Resource B WITH (ROWLOCK, HOLDLOCK) ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId
WHERE B.IsHistory = 0
OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
IF @@rowcount > 0 SET @IsRetry = 1
IF @IsRetry = 0 COMMIT TRANSACTION -- commit check transaction
END
END
SET @Mode += ' R='+convert(varchar,@IsRetry)
IF @SingleTransaction = 1 AND @@trancount = 0 BEGIN TRANSACTION
IF @IsRetry = 0
BEGIN
INSERT INTO @ResourceInfos
( ResourceTypeId, SurrogateId, Version, KeepHistory, PreviousVersion, PreviousSurrogateId )
SELECT A.ResourceTypeId, A.ResourceSurrogateId, A.Version, A.KeepHistory, B.Version, B.ResourceSurrogateId
FROM (SELECT TOP (@DummyTop) * FROM @Resources WHERE HasVersionToCompare = 1) A
LEFT OUTER JOIN dbo.Resource B -- WITH (UPDLOCK, HOLDLOCK) These locking hints cause deadlocks and are not needed. Racing might lead to tries to insert dups in unique index (with version key), but it will fail anyway, and in no case this will cause incorrect data saved.
ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceId = A.ResourceId AND B.IsHistory = 0
OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
IF @RaiseExceptionOnConflict = 1 AND EXISTS (SELECT * FROM @ResourceInfos WHERE PreviousVersion IS NOT NULL AND Version <= PreviousVersion)
THROW 50409, 'Resource has been recently updated or added, please compare the resource content in code for any duplicate updates', 1
INSERT INTO @PreviousSurrogateIds
SELECT ResourceTypeId, PreviousSurrogateId, KeepHistory
FROM @ResourceInfos
WHERE PreviousSurrogateId IS NOT NULL
IF @@rowcount > 0
BEGIN
UPDATE dbo.Resource
SET IsHistory = 1
WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId AND KeepHistory = 1)
SET @AffectedRows += @@rowcount
IF @IsResourceChangeCaptureEnabled = 1 AND NOT EXISTS (SELECT * FROM dbo.Parameters WHERE Id = 'InvisibleHistory.IsEnabled' AND Number = 0)
UPDATE dbo.Resource
SET IsHistory = 1
,RawResource = 0xF -- "invisible" value
,SearchParamHash = NULL
,HistoryTransactionId = @TransactionId
WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId AND KeepHistory = 0)
ELSE
DELETE FROM dbo.Resource WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId AND KeepHistory = 0)
SET @AffectedRows += @@rowcount
DELETE FROM dbo.ResourceWriteClaim WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE SurrogateId = ResourceSurrogateId)
SET @AffectedRows += @@rowcount
DELETE FROM dbo.ReferenceSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId)
SET @AffectedRows += @@rowcount
DELETE FROM dbo.TokenSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId)
SET @AffectedRows += @@rowcount
DELETE FROM dbo.TokenText WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId)
SET @AffectedRows += @@rowcount
DELETE FROM dbo.StringSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId)
SET @AffectedRows += @@rowcount
DELETE FROM dbo.UriSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId)
SET @AffectedRows += @@rowcount
DELETE FROM dbo.NumberSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId)
SET @AffectedRows += @@rowcount
DELETE FROM dbo.QuantitySearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId)
SET @AffectedRows += @@rowcount
DELETE FROM dbo.DateTimeSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId)
SET @AffectedRows += @@rowcount
DELETE FROM dbo.ReferenceTokenCompositeSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId)
SET @AffectedRows += @@rowcount
DELETE FROM dbo.TokenTokenCompositeSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId)
SET @AffectedRows += @@rowcount
DELETE FROM dbo.TokenDateTimeCompositeSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId)
SET @AffectedRows += @@rowcount
DELETE FROM dbo.TokenQuantityCompositeSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId)
SET @AffectedRows += @@rowcount
DELETE FROM dbo.TokenStringCompositeSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId)
SET @AffectedRows += @@rowcount
DELETE FROM dbo.TokenNumberNumberCompositeSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId)
SET @AffectedRows += @@rowcount
--EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Info',@Start=@st,@Rows=@AffectedRows,@Text='Old rows'
END
INSERT INTO dbo.Resource
( ResourceTypeId, ResourceId, Version, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, RawResource, IsRawResourceMetaSet, SearchParamHash, TransactionId )
SELECT ResourceTypeId, ResourceId, Version, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, RawResource, IsRawResourceMetaSet, SearchParamHash, @TransactionId
FROM @Resources
SET @AffectedRows += @@rowcount
INSERT INTO dbo.ResourceWriteClaim
( ResourceSurrogateId, ClaimTypeId, ClaimValue )
SELECT ResourceSurrogateId, ClaimTypeId, ClaimValue
FROM @ResourceWriteClaims
SET @AffectedRows += @@rowcount
INSERT INTO dbo.ReferenceSearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion
FROM @ReferenceSearchParams
SET @AffectedRows += @@rowcount
INSERT INTO dbo.TokenSearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow
FROM @TokenSearchParams
SET @AffectedRows += @@rowcount
INSERT INTO dbo.TokenText
( ResourceTypeId, ResourceSurrogateId, SearchParamId, Text )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, Text
FROM @TokenTexts
SET @AffectedRows += @@rowcount
INSERT INTO dbo.StringSearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax
FROM @StringSearchParams
SET @AffectedRows += @@rowcount
INSERT INTO dbo.UriSearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri
FROM @UriSearchParams
SET @AffectedRows += @@rowcount
INSERT INTO dbo.NumberSearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue
FROM @NumberSearchParams
SET @AffectedRows += @@rowcount
INSERT INTO dbo.QuantitySearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue
FROM @QuantitySearchParams
SET @AffectedRows += @@rowcount
INSERT INTO dbo.DateTimeSearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax
FROM @DateTimeSearchParms
SET @AffectedRows += @@rowcount
INSERT INTO dbo.ReferenceTokenCompositeSearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2 )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2
FROM @ReferenceTokenCompositeSearchParams
SET @AffectedRows += @@rowcount
INSERT INTO dbo.TokenTokenCompositeSearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2 )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2
FROM @TokenTokenCompositeSearchParams
SET @AffectedRows += @@rowcount
INSERT INTO dbo.TokenDateTimeCompositeSearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2 )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2
FROM @TokenDateTimeCompositeSearchParams
SET @AffectedRows += @@rowcount
INSERT INTO dbo.TokenQuantityCompositeSearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2 )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2
FROM @TokenQuantityCompositeSearchParams
SET @AffectedRows += @@rowcount
INSERT INTO dbo.TokenStringCompositeSearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2 )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2
FROM @TokenStringCompositeSearchParams
SET @AffectedRows += @@rowcount
INSERT INTO dbo.TokenNumberNumberCompositeSearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange
FROM @TokenNumberNumberCompositeSearchParams
SET @AffectedRows += @@rowcount
END -- @IsRetry = 0
ELSE
BEGIN -- @IsRetry = 1
INSERT INTO dbo.ResourceWriteClaim
( ResourceSurrogateId, ClaimTypeId, ClaimValue )
SELECT ResourceSurrogateId, ClaimTypeId, ClaimValue
FROM (SELECT TOP (@DummyTop) * FROM @ResourceWriteClaims) A
WHERE EXISTS (SELECT * FROM @Existing B WHERE B.SurrogateId = A.ResourceSurrogateId)
AND NOT EXISTS (SELECT * FROM dbo.ResourceWriteClaim C WHERE C.ResourceSurrogateId = A.ResourceSurrogateId)
OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
SET @AffectedRows += @@rowcount
INSERT INTO dbo.ReferenceSearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion
FROM (SELECT TOP (@DummyTop) * FROM @ReferenceSearchParams) A
WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId)
AND NOT EXISTS (SELECT * FROM dbo.ReferenceSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId)
OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
SET @AffectedRows += @@rowcount
INSERT INTO dbo.TokenSearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow
FROM (SELECT TOP (@DummyTop) * FROM @TokenSearchParams) A
WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId)
AND NOT EXISTS (SELECT * FROM dbo.TokenSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId)
OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
SET @AffectedRows += @@rowcount
INSERT INTO dbo.TokenText
( ResourceTypeId, ResourceSurrogateId, SearchParamId, Text )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, Text
FROM (SELECT TOP (@DummyTop) * FROM @TokenTexts) A
WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId)
AND NOT EXISTS (SELECT * FROM dbo.TokenSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId)
OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
SET @AffectedRows += @@rowcount
INSERT INTO dbo.StringSearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax
FROM (SELECT TOP (@DummyTop) * FROM @StringSearchParams) A
WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId)
AND NOT EXISTS (SELECT * FROM dbo.TokenText C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId)
OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
SET @AffectedRows += @@rowcount
INSERT INTO dbo.UriSearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri
FROM (SELECT TOP (@DummyTop) * FROM @UriSearchParams) A
WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId)
AND NOT EXISTS (SELECT * FROM dbo.UriSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId)
OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
SET @AffectedRows += @@rowcount
INSERT INTO dbo.NumberSearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue
FROM (SELECT TOP (@DummyTop) * FROM @NumberSearchParams) A
WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId)
AND NOT EXISTS (SELECT * FROM dbo.NumberSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId)
OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
SET @AffectedRows += @@rowcount
INSERT INTO dbo.QuantitySearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue
FROM (SELECT TOP (@DummyTop) * FROM @QuantitySearchParams) A
WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId)
AND NOT EXISTS (SELECT * FROM dbo.QuantitySearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId)
OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
SET @AffectedRows += @@rowcount
INSERT INTO dbo.DateTimeSearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax
FROM (SELECT TOP (@DummyTop) * FROM @DateTimeSearchParms) A
WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId)
AND NOT EXISTS (SELECT * FROM dbo.TokenSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId)
OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
SET @AffectedRows += @@rowcount
INSERT INTO dbo.ReferenceTokenCompositeSearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2 )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2
FROM (SELECT TOP (@DummyTop) * FROM @ReferenceTokenCompositeSearchParams) A
WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId)
AND NOT EXISTS (SELECT * FROM dbo.DateTimeSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId)
OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
SET @AffectedRows += @@rowcount
INSERT INTO dbo.TokenTokenCompositeSearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2 )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2
FROM (SELECT TOP (@DummyTop) * FROM @TokenTokenCompositeSearchParams) A
WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId)
AND NOT EXISTS (SELECT * FROM dbo.TokenTokenCompositeSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId)
OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
SET @AffectedRows += @@rowcount
INSERT INTO dbo.TokenDateTimeCompositeSearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2 )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2
FROM (SELECT TOP (@DummyTop) * FROM @TokenDateTimeCompositeSearchParams) A
WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId)
AND NOT EXISTS (SELECT * FROM dbo.TokenDateTimeCompositeSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId)
OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
SET @AffectedRows += @@rowcount
INSERT INTO dbo.TokenQuantityCompositeSearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2 )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2
FROM (SELECT TOP (@DummyTop) * FROM @TokenQuantityCompositeSearchParams) A
WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId)
AND NOT EXISTS (SELECT * FROM dbo.TokenQuantityCompositeSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId)
OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
SET @AffectedRows += @@rowcount
INSERT INTO dbo.TokenStringCompositeSearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2 )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2
FROM (SELECT TOP (@DummyTop) * FROM @TokenStringCompositeSearchParams) A
WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId)
AND NOT EXISTS (SELECT * FROM dbo.TokenStringCompositeSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId)
OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
SET @AffectedRows += @@rowcount
INSERT INTO dbo.TokenNumberNumberCompositeSearchParam
( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange )
SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange
FROM (SELECT TOP (@DummyTop) * FROM @TokenNumberNumberCompositeSearchParams) A
WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId)
AND NOT EXISTS (SELECT * FROM dbo.TokenNumberNumberCompositeSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId)
OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
SET @AffectedRows += @@rowcount
END
IF @IsResourceChangeCaptureEnabled = 1 --If the resource change capture feature is enabled, to execute a stored procedure called CaptureResourceChanges to insert resource change data.
EXECUTE dbo.CaptureResourceIdsForChanges @Resources
IF @TransactionId IS NOT NULL
EXECUTE dbo.MergeResourcesCommitTransaction @TransactionId
IF @InitialTranCount = 0 AND @@trancount > 0 COMMIT TRANSACTION
EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@st,@Rows=@AffectedRows
END TRY
BEGIN CATCH
IF @InitialTranCount = 0 AND @@trancount > 0 ROLLBACK TRANSACTION
IF error_number() = 1750 THROW -- Real error is before 1750, cannot trap in SQL.
EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Error',@Start=@st;
IF @RaiseExceptionOnConflict = 1 AND error_number() IN (2601, 2627) AND error_message() LIKE '%''dbo.Resource''%'
THROW 50409, 'Resource has been recently updated or added, please compare the resource content in code for any duplicate updates', 1;
ELSE
THROW
END CATCH
GO

5440
src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/74.sql Normal file
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

1
src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs
Просмотреть файл

@ -83,5 +83,6 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Schema
V71 = 71,
V72 = 72,
V73 = 73,
V74 = 74,
}
}

2
src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs
Просмотреть файл

@ -8,7 +8,7 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Schema
public static class SchemaVersionConstants
{
public const int Min = (int)SchemaVersion.V69;
public const int Max = (int)SchemaVersion.V73;
public const int Max = (int)SchemaVersion.V74;
public const int MinForUpgrade = (int)SchemaVersion.V69; // this is used for upgrade tests only
public const int SearchParameterStatusSchemaVersion = (int)SchemaVersion.V6;
public const int SupportForReferencesWithMissingTypeVersion = (int)SchemaVersion.V7;

2
src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql
Просмотреть файл

@ -19,6 +19,6 @@ Go
INSERT INTO dbo.SchemaVersion
VALUES
(73, 'started')
(74, 'started')
Go

2
src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/MergeResources.sql
Просмотреть файл

@ -99,7 +99,7 @@ BEGIN TRY
ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceId = A.ResourceId AND B.IsHistory = 0
OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
IF @RaiseExceptionOnConflict = 1 AND EXISTS (SELECT * FROM @ResourceInfos WHERE PreviousVersion IS NOT NULL AND Version <> PreviousVersion + 1)
IF @RaiseExceptionOnConflict = 1 AND EXISTS (SELECT * FROM @ResourceInfos WHERE PreviousVersion IS NOT NULL AND Version <= PreviousVersion)
THROW 50409, 'Resource has been recently updated or added, please compare the resource content in code for any duplicate updates', 1
INSERT INTO @PreviousSurrogateIds

38
src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs
Просмотреть файл

@ -164,12 +164,12 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Storage
(var transactionId, var minSequenceId) = await StoreClient.MergeResourcesBeginTransactionAsync(resources.Count, cancellationToken);
var index = 0;
var mergeWrappers = new List<MergeResourceWrapper>();
var mergeWrappersWithVersions = new List<(MergeResourceWrapper Wrapper, bool KeepVersion, int ResourceVersion, int? ExistingVersion)>();
var prevResourceId = string.Empty;
var singleTransaction = enlistInTransaction;
foreach (var resourceExt in resources) // if list contains more that one version per resource it must be sorted by id and last updated desc.
foreach (var resourceExt in resources) // if list contains more that one version per resource it must be sorted by id and last updated DESC.
{
var setAsHistory = prevResourceId == resourceExt.Wrapper.ResourceId;
var setAsHistory = prevResourceId == resourceExt.Wrapper.ResourceId; // this assumes that first resource version is the latest one
prevResourceId = resourceExt.Wrapper.ResourceId;
var weakETag = resourceExt.WeakETag;
int? eTag = weakETag == null
@ -178,9 +178,9 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Storage
var resource = resourceExt.Wrapper;
var identifier = resourceExt.GetIdentifier();
var resourceKey = resource.ToResourceKey(); // keep input version in the results to allow processing multiple versions per resource
existingResources.TryGetValue(resource.ToResourceKey(true), out var existingResource);
var hasVersionToCompare = false;
var existingVersion = 0;
// Check for any validation errors
if (existingResource != null && eTag.HasValue && !string.Equals(eTag.ToString(), existingResource.Version, StringComparison.Ordinal))
@ -271,7 +271,7 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Storage
}
}
var existingVersion = int.Parse(existingResource.Version);
existingVersion = int.Parse(existingResource.Version);
var versionPlusOne = (existingVersion + 1).ToString(CultureInfo.InvariantCulture);
if (!resourceExt.KeepVersion) // version is set on input
{
@ -310,12 +310,34 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Storage
singleTransaction = true;
}
mergeWrappers.Add(new MergeResourceWrapper(resource, resourceExt.KeepHistory, hasVersionToCompare));
mergeWrappersWithVersions.Add((new MergeResourceWrapper(resource, resourceExt.KeepHistory, hasVersionToCompare), resourceExt.KeepVersion, int.Parse(resource.Version), existingVersion));
index++;
results.Add(identifier, new DataStoreOperationOutcome(new UpsertOutcome(resource, resource.Version == InitialVersion ? SaveOutcomeType.Created : SaveOutcomeType.Updated)));
}
if (mergeWrappers.Count > 0) // Do not call DB with empty input
// Resources with input versions (keepVersion=true) might not have hasVersionToCompare set. Fix it here.
// Resources with keepVersion=true must be in separate call, and not mixed with keepVersion=false ones.
// Sort them in groups by resource id and order by version.
// In each group find the smallest version higher then existing
prevResourceId = string.Empty;
var notSetInResoureGroup = false;
foreach (var mergeWrapper in mergeWrappersWithVersions.Where(x => x.KeepVersion && x.ExistingVersion != 0).OrderBy(x => x.Wrapper.ResourceWrapper.ResourceId).ThenBy(x => x.ResourceVersion))
{
if (prevResourceId != mergeWrapper.Wrapper.ResourceWrapper.ResourceId) // this should reset flag on each resource id group including first.
{
notSetInResoureGroup = true;
}
prevResourceId = mergeWrapper.Wrapper.ResourceWrapper.ResourceId;
if (notSetInResoureGroup && mergeWrapper.ResourceVersion > mergeWrapper.ExistingVersion)
{
mergeWrapper.Wrapper.HasVersionToCompare = true;
notSetInResoureGroup = false;
}
}
if (mergeWrappersWithVersions.Count > 0) // Do not call DB with empty input
{
await using (new Timer(async _ => await _sqlStoreClient.MergeResourcesPutTransactionHeartbeatAsync(transactionId, MergeResourcesTransactionHeartbeatPeriod, cancellationToken), null, TimeSpan.FromSeconds(RandomNumberGenerator.GetInt32(100) / 100.0 * MergeResourcesTransactionHeartbeatPeriod.TotalSeconds), MergeResourcesTransactionHeartbeatPeriod))
{
@ -325,7 +347,7 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Storage
{
try
{
await MergeResourcesWrapperAsync(transactionId, singleTransaction, mergeWrappers, enlistInTransaction, timeoutRetries, cancellationToken);
await MergeResourcesWrapperAsync(transactionId, singleTransaction, mergeWrappersWithVersions.Select(_ => _.Wrapper).ToList(), enlistInTransaction, timeoutRetries, cancellationToken);
break;
}
catch (Exception e)

2
src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/Merge/MergeResourceWrapper.cs
Просмотреть файл

@ -29,6 +29,6 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Storage.TvpRowGeneration.Merg
/// <summary>
/// Flag indicating whether version in resource wrapper == (existing version in the database + 1)
/// </summary>
public bool HasVersionToCompare { get; private set; }
public bool HasVersionToCompare { get; internal set; }
}
}

2
src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj
Просмотреть файл

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<!-- Properties used by sql task to generate full script -->
<PropertyGroup>
<LatestSchemaVersion>73</LatestSchemaVersion>
<LatestSchemaVersion>74</LatestSchemaVersion>
<GeneratedFullScriptPath>Features\Schema\Migrations\$(LatestSchemaVersion).sql</GeneratedFullScriptPath>
</PropertyGroup>

35
test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportTests.cs
Просмотреть файл

@ -92,6 +92,41 @@ namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Import
Assert.Equal(GetLastUpdated("2002"), result.Resource.Meta.LastUpdated);
}
[Fact]
public async Task GivenIncrementalLoad_MultipleNonSequentialInputVersions_ResourceExisting()
{
var id = Guid.NewGuid().ToString("N");
// set existing
var ndJson = PrepareResource(id, "10000", "2000");
var location = (await ImportTestHelper.UploadFileAsync(ndJson, _fixture.StorageAccount)).location;
var request = CreateImportRequest(location, ImportMode.IncrementalLoad);
await ImportCheckAsync(request, null);
// set input. something before and something after existing
ndJson = PrepareResource(id, "9000", "1999");
var ndJson2 = PrepareResource(id, "10100", "2001");
var ndJson3 = PrepareResource(id, "10300", "2003");
// note order of records
location = (await ImportTestHelper.UploadFileAsync(ndJson2 + ndJson + ndJson3, _fixture.StorageAccount)).location;
request = CreateImportRequest(location, ImportMode.IncrementalLoad);
await ImportCheckAsync(request, null);
// check current
var result = await _client.ReadAsync<Patient>(ResourceType.Patient, id);
Assert.Equal("10300", result.Resource.Meta.VersionId);
Assert.Equal(GetLastUpdated("2003"), result.Resource.Meta.LastUpdated);
// check history
result = await _client.VReadAsync<Patient>(ResourceType.Patient, id, "9000");
Assert.Equal(GetLastUpdated("1999"), result.Resource.Meta.LastUpdated);
result = await _client.VReadAsync<Patient>(ResourceType.Patient, id, "10100");
Assert.Equal(GetLastUpdated("2001"), result.Resource.Meta.LastUpdated);
result = await _client.VReadAsync<Patient>(ResourceType.Patient, id, "10000");
Assert.Equal(GetLastUpdated("2000"), result.Resource.Meta.LastUpdated);
}
[Fact]
public async Task GivenIncrementalLoad_MultipleInputVersions_ResourceExisting_VersionConflict()
{

2
tools/BlobRewriter/App.config
Просмотреть файл

@ -21,5 +21,7 @@
<add key="SplitBySize" value="true" />
<!-- Filter on blob names. If empoty string - no filtering. -->
<add key="NameFilter" value="" />
<!-- If true tries to replace {"resourceType": by {"meta":{"versionId":"seconds from last updated","lastUpdated":"YYYY-MM-DDThh:mm:ss"},"resourceType": -->
<add key="AddMeta" value="false" />
</appSettings>
</configuration>

17
tools/BlobRewriter/Program.cs
Просмотреть файл

@ -33,6 +33,7 @@ namespace Microsoft.Health.Internal.Fhir.BlobRewriter
private static readonly bool WritesEnabled = bool.Parse(ConfigurationManager.AppSettings["WritesEnabled"]);
private static readonly bool SplitBySize = bool.Parse(ConfigurationManager.AppSettings["SplitBySize"]);
private static readonly string NameFilter = ConfigurationManager.AppSettings["NameFilter"];
private static readonly bool AddMeta = bool.Parse(ConfigurationManager.AppSettings["AddMeta"]);
public static void Main()
{
@ -60,7 +61,7 @@ namespace Microsoft.Health.Internal.Fhir.BlobRewriter
var blob = blobInt.Item2.First();
var lines = SplitBySize
? LinesPerBlob == 0 ? CopyBlob(sourceContainer, blob.Name, targetContainer, ref targetBlobs) : SplitBlobBySize(sourceContainer, blob.Name, targetContainer, ref targetBlobs)
? LinesPerBlob == 0 ? CopyBlob(sourceContainer, blob.Name, targetContainer, ref targetBlobs, blobIndex) : SplitBlobBySize(sourceContainer, blob.Name, targetContainer, ref targetBlobs)
: SplitBlobByResourceId(sourceContainer, blob.Name, targetContainer, ref targetBlobs);
Interlocked.Add(ref totalLines, lines);
Interlocked.Increment(ref sourceBlobs);
@ -148,7 +149,7 @@ namespace Microsoft.Health.Internal.Fhir.BlobRewriter
return lines;
}
private static long CopyBlob(BlobContainerClient sourceContainer, string blobName, BlobContainerClient targetContainer, ref long targetBlobs)
private static long CopyBlob(BlobContainerClient sourceContainer, string blobName, BlobContainerClient targetContainer, ref long targetBlobs, int blobIndex)
{
var lines = 0L;
using var stream = targetContainer.GetBlockBlobClient(blobName).OpenWrite(true);
@ -158,7 +159,17 @@ namespace Microsoft.Health.Internal.Fhir.BlobRewriter
lines++;
if (WritesEnabled)
{
writer.WriteLine(line);
if (AddMeta)
{
var date = DateTime.UtcNow.AddMinutes(-SourceBlobs).AddMinutes(blobIndex).AddMilliseconds(lines);
var seconds = ((int)(date - DateTime.Parse("1970-01-01")).TotalSeconds).ToString();
var lineWithMeta = line.Replace("{\"resourceType\":", "{\"meta\":{\"versionId\":\"" + seconds + "\",\"lastUpdated\":\"" + date.ToString("yyyy-MM-ddTHH:mm:ss.fff") + "\"},\"resourceType\":", StringComparison.OrdinalIgnoreCase);
writer.WriteLine(lineWithMeta);
}
else
{
writer.WriteLine(line);
}
}
}