зеркало из https://github.com/microsoft/BuildXL.git
Merged PR 810928: Support CAS on separate mount than outputs on Linux
This pull request addresses two bugs encountered when the Content Addressable Storage (CAS) is mounted separately from the outputs on Linux: 1. Temporary File Deletion: If TryInKernelFileCopyAsync fails, the temporary file is now deleted. Previously, this temp file would prevent CopyWithStreamAsync from copying the file. 2. Hardlink Creation: When creating a hardlink for content already in the cache, the process attempts to delete the original file and replace it with a new one that includes the hardlink. If this operation fails, the original file was being deleted. And the fallback to copying would return true because the content is already in the store, but the pip would fail in future operations due to the missing file. Now, if the file is deleted, the original file is re-created using the existing content from the cache. Related work items: #2216529, #2219745
This commit is contained in:
Qi Wang
Родитель
288310198b
Коммит
7e87663706
|
|
@ -821,45 +821,45 @@ namespace BuildXL.Cache.ContentStore.Stores
|
|||
content.Size,
|
||||
pinRequest?.PinContext,
|
||||
onContentAlreadyInCache: async (hashHandle, primaryPath, info) =>
|
||||
{
|
||||
// The content exists in the cache. Try to replace the file that is being put in
|
||||
// with a link to the file that is already in the cache. Release the handle to
|
||||
// allow for the hardlink to succeed.
|
||||
using var trace = _tracer.PutFileExistingHardLink();
|
||||
// ReSharper disable once AccessToDisposedClosure
|
||||
var result = await PlaceLinkFromCacheAsync(
|
||||
context,
|
||||
path,
|
||||
FileReplacementMode.ReplaceExisting,
|
||||
realizationMode,
|
||||
content.Hash,
|
||||
info);
|
||||
return result == CreateHardLinkResult.Success;
|
||||
},
|
||||
{
|
||||
// The content exists in the cache. Try to replace the file that is being put in
|
||||
// with a link to the file that is already in the cache. Release the handle to
|
||||
// allow for the hardlink to succeed.
|
||||
using var trace = _tracer.PutFileExistingHardLink();
|
||||
// ReSharper disable once AccessToDisposedClosure
|
||||
var result = await PlaceLinkFromCacheAsync(
|
||||
context,
|
||||
path,
|
||||
FileReplacementMode.ReplaceExisting,
|
||||
realizationMode,
|
||||
content.Hash,
|
||||
info);
|
||||
return result == CreateHardLinkResult.Success;
|
||||
},
|
||||
onContentNotInCache: primaryPath =>
|
||||
{
|
||||
using var trace = _tracer.PutFileNewHardLink();
|
||||
ApplyPermissions(context, path, FileAccessMode.ReadOnly);
|
||||
{
|
||||
using var trace = _tracer.PutFileNewHardLink();
|
||||
ApplyPermissions(context, path, FileAccessMode.ReadOnly);
|
||||
|
||||
var hardLinkResult = CreateHardLinkResult.Unknown;
|
||||
Func<bool> tryCreateHardlinkFunc = () => TryCreateHardlink(
|
||||
context,
|
||||
path,
|
||||
primaryPath,
|
||||
realizationMode,
|
||||
false,
|
||||
out hardLinkResult);
|
||||
var hardLinkResult = CreateHardLinkResult.Unknown;
|
||||
Func<bool> tryCreateHardlinkFunc = () => TryCreateHardlink(
|
||||
context,
|
||||
path,
|
||||
primaryPath,
|
||||
realizationMode,
|
||||
false,
|
||||
out hardLinkResult);
|
||||
|
||||
bool result = tryCreateHardlinkFunc();
|
||||
if (hardLinkResult == CreateHardLinkResult.FailedDestinationExists)
|
||||
{
|
||||
// Extraneous blobs on disk. Delete them and retry.
|
||||
RemoveAllReplicasFromDiskFor(context, content.Hash);
|
||||
result = tryCreateHardlinkFunc();
|
||||
}
|
||||
bool result = tryCreateHardlinkFunc();
|
||||
if (hardLinkResult == CreateHardLinkResult.FailedDestinationExists)
|
||||
{
|
||||
// Extraneous blobs on disk. Delete them and retry.
|
||||
RemoveAllReplicasFromDiskFor(context, content.Hash);
|
||||
result = tryCreateHardlinkFunc();
|
||||
}
|
||||
|
||||
return Task.FromResult(result);
|
||||
});
|
||||
return Task.FromResult(result);
|
||||
});
|
||||
|
||||
if (putInternalSucceeded)
|
||||
{
|
||||
|
|
@ -876,38 +876,55 @@ namespace BuildXL.Cache.ContentStore.Stores
|
|||
content.Hash,
|
||||
content.Size,
|
||||
pinRequest?.PinContext,
|
||||
onContentAlreadyInCache: (hashHandle, primaryPath, info) => Task.FromResult(true),
|
||||
onContentAlreadyInCache: async (hashHandle, primaryPath, info) =>
|
||||
{
|
||||
// Failed hardlink creation attempt will delete the file if content is already in cache on Linux.
|
||||
// Re-create the file use content from cache if the file was deleted.
|
||||
if (!FileSystem.FileExists(path))
|
||||
{
|
||||
try
|
||||
{
|
||||
await SafeCopyFileAsync(context, content.Hash, primaryPath, path, FileReplacementMode.FailIfExists);
|
||||
}
|
||||
catch (IOException e)
|
||||
{
|
||||
_tracer.Warning(context, e, $"Unable to re-create file {path} from existing content. This file was deleted in previous hardlink creation attempt.");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
},
|
||||
onContentNotInCache: async primaryPath =>
|
||||
{
|
||||
using var trace = _tracer.PutFileNewCopy();
|
||||
alreadyInCache = false;
|
||||
{
|
||||
using var trace = _tracer.PutFileNewCopy();
|
||||
alreadyInCache = false;
|
||||
|
||||
await RetryOnUnexpectedReplicaAsync(
|
||||
context,
|
||||
() =>
|
||||
{
|
||||
if (realizationMode == FileRealizationMode.Move)
|
||||
{
|
||||
return Task.Run(() =>
|
||||
{
|
||||
ApplyPermissions(context, path, FileAccessMode.ReadOnly);
|
||||
FileSystem.MoveFile(path, primaryPath, replaceExisting: false);
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
return SafeCopyFileAsync(
|
||||
context,
|
||||
content.Hash,
|
||||
path,
|
||||
primaryPath,
|
||||
FileReplacementMode.FailIfExists);
|
||||
}
|
||||
},
|
||||
content.Hash,
|
||||
expectedReplicaCount: 0);
|
||||
return true;
|
||||
});
|
||||
await RetryOnUnexpectedReplicaAsync(
|
||||
context,
|
||||
() =>
|
||||
{
|
||||
if (realizationMode == FileRealizationMode.Move)
|
||||
{
|
||||
return Task.Run(() =>
|
||||
{
|
||||
ApplyPermissions(context, path, FileAccessMode.ReadOnly);
|
||||
FileSystem.MoveFile(path, primaryPath, replaceExisting: false);
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
return SafeCopyFileAsync(
|
||||
context,
|
||||
content.Hash,
|
||||
path,
|
||||
primaryPath,
|
||||
FileReplacementMode.FailIfExists);
|
||||
}
|
||||
},
|
||||
content.Hash,
|
||||
expectedReplicaCount: 0);
|
||||
return true;
|
||||
});
|
||||
|
||||
return new PutResult(content.Hash, content.Size, contentAlreadyExistsInCache: alreadyInCache)
|
||||
.WithLockAcquisitionDuration(contentHashHandle);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,125 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using BuildXL.Cache.ContentStore.FileSystem;
|
||||
using BuildXL.Cache.ContentStore.Hashing;
|
||||
using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
|
||||
using BuildXL.Cache.ContentStore.Interfaces.Sessions;
|
||||
using BuildXL.Cache.ContentStore.Interfaces.Tracing;
|
||||
using BuildXL.Cache.ContentStore.InterfacesTest.Results;
|
||||
using BuildXL.Cache.ContentStore.InterfacesTest.Time;
|
||||
using BuildXL.Cache.ContentStore.Stores;
|
||||
using BuildXL.Cache.ContentStore.UtilitiesCore;
|
||||
using BuildXL.Interop.Unix;
|
||||
using BuildXL.Native.IO;
|
||||
using ContentStoreTest.Stores;
|
||||
using ContentStoreTest.Test;
|
||||
using FluentAssertions;
|
||||
using Test.BuildXL.TestUtilities.Xunit;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
using SystemProcess = System.Diagnostics.Process;
|
||||
|
||||
namespace BuildXL.Cache.ContentStore.Test.Stores
|
||||
{
|
||||
[Trait("Category", "Integration")]
|
||||
public class FileSystemContentStoreInternalPassThroughFsTests : FileSystemContentStoreInternalTestBase
|
||||
{
|
||||
private static readonly MemoryClock Clock = new MemoryClock();
|
||||
private static readonly ContentStoreConfiguration Config = ContentStoreConfiguration.CreateWithMaxSizeQuotaMB(5);
|
||||
|
||||
public FileSystemContentStoreInternalPassThroughFsTests(ITestOutputHelper output)
|
||||
: base(() => new PassThroughFileSystem(), TestGlobal.Logger, output)
|
||||
{
|
||||
}
|
||||
|
||||
[FactIfSupported(requiresUnixBasedOperatingSystem: true, requiresAdmin: true)]
|
||||
public async Task PutAttemptHardLinkCacheInDifferentMountFallBackToCopy()
|
||||
{
|
||||
using (var testDirectory = new DisposableDirectory(FileSystem))
|
||||
{
|
||||
var context = new Context(Logger);
|
||||
|
||||
var virtualDiskPath = testDirectory.Path / "vh.img";
|
||||
using var virtualDiskMountPoint = new DisposableDirectory(FileSystem);
|
||||
|
||||
// Create a virtual disk and mount it
|
||||
RunBashCommand($"dd if=/dev/zero of={virtualDiskPath} bs=1M count=3");
|
||||
RunBashCommand($"mkfs.ext4 {virtualDiskPath}");
|
||||
RunBashCommand($"sudo mount {virtualDiskPath} {virtualDiskMountPoint.Path}");
|
||||
|
||||
// Use mounted virtual disk as cache root to create test store
|
||||
await TestStore(context, Clock, virtualDiskMountPoint, async store =>
|
||||
{
|
||||
byte[] bytes = ThreadSafeRandom.GetBytes(ValueSize);
|
||||
ContentHash contentHash;
|
||||
|
||||
// Put content into store
|
||||
using (var memoryStream = new MemoryStream(bytes))
|
||||
{
|
||||
var putStreamResult = await store.PutStreamAsync(context, memoryStream, ContentHashType);
|
||||
contentHash = putStreamResult.ContentHash;
|
||||
Assert.Equal(bytes.Length, putStreamResult.ContentSize);
|
||||
}
|
||||
|
||||
// Create HardLink failed at FailedSourceAndDestinationOnDifferentVolumes
|
||||
AbsolutePath testDesFile = testDirectory.CreateRandomFileName();
|
||||
AbsolutePath testSourceFile = virtualDiskMountPoint.CreateRandomFileName();
|
||||
FileSystem.CreateEmptyFile(testSourceFile);
|
||||
|
||||
var hardlinkCreationResult = FileSystem.CreateHardLink(testSourceFile, testDesFile, true);
|
||||
Assert.True(hardlinkCreationResult == CreateHardLinkResult.FailedSourceAndDestinationOnDifferentVolumes, hardlinkCreationResult.ToString());
|
||||
|
||||
// Create a new file with the same content
|
||||
AbsolutePath newPathWithSameContent = testDirectory.CreateRandomFileName();
|
||||
FileSystem.WriteAllBytes(newPathWithSameContent, bytes);
|
||||
|
||||
// Call PutFileAsync with FileRealizationMode.Any
|
||||
// PutFile try HardLink first and fall back to copy
|
||||
var result = await store.PutFileAsync(context, newPathWithSameContent, FileRealizationMode.Any, ContentHashType).ShouldBeSuccess();
|
||||
result.ContentAlreadyExistsInCache.Should().BeTrue();
|
||||
|
||||
// After PutFileAsync, the file should not be deleted due to failed hardlink creation
|
||||
Assert.True(FileSystem.FileExists(newPathWithSameContent));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private void RunBashCommand(string bashScriptCommand)
|
||||
{
|
||||
_ = FileUtilities.SetExecutePermissionIfNeeded(UnixPaths.BinBash).ThrowIfFailure();
|
||||
|
||||
var startInfo = new ProcessStartInfo
|
||||
{
|
||||
FileName = UnixPaths.BinBash,
|
||||
Arguments = $"-c \"{bashScriptCommand}\"",
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false
|
||||
};
|
||||
|
||||
var process = new SystemProcess
|
||||
{
|
||||
StartInfo = startInfo
|
||||
};
|
||||
|
||||
process.OutputDataReceived += (sender, data) => Logger.Info(data.Data);
|
||||
process.ErrorDataReceived += (sender, data) => Logger.Error(data.Data);
|
||||
|
||||
Logger.Info($"Running {process.StartInfo.FileName} {process.StartInfo.Arguments}");
|
||||
|
||||
process.Start();
|
||||
process.BeginErrorReadLine();
|
||||
process.BeginOutputReadLine();
|
||||
|
||||
process.WaitForExit();
|
||||
|
||||
Assert.Equal(process.ExitCode, 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -545,6 +545,8 @@ namespace BuildXL.Native.IO.Unix
|
|||
var statBuffer = new StatBuffer();
|
||||
if (StatFileDescriptor(sourceHandle, ref statBuffer) != 0)
|
||||
{
|
||||
destinationHandle.Close();
|
||||
DeleteFile(destination, false);
|
||||
return new NativeFailure(Marshal.GetLastWin32Error(), I($"Failed to stat source file '{source}' for size query in {nameof(InKernelFileCopy)}"));
|
||||
}
|
||||
|
||||
|
|
@ -557,6 +559,8 @@ namespace BuildXL.Native.IO.Unix
|
|||
bytesCopied = CopyBytes(sourceHandle, destinationHandle);
|
||||
if (bytesCopied == -1)
|
||||
{
|
||||
destinationHandle.Close();
|
||||
DeleteFile(destination, false);
|
||||
return new NativeFailure(Marshal.GetLastWin32Error(), I($"{nameof(InKernelFileCopy)} failed copying '{source}' to '{destination}' with error code: {lastError}"));
|
||||
}
|
||||
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче