Merged PR 684087: Detour copy_file_range with splice

Due to (possibly) kernel bug, copy_file_range no longer works when the file descriptors are not mounted on the same filesystems, despite what is said in the manual https://man7.org/linux/man-pages/man2/copy_file_range.2.html.

This bug breaks AnyBuild virtual filesystem (VFS) because the source file will be in the read-only (lower) layer of overlayfs, and this layer is mounted on AnyBuild FUSE, and the target file will be in the writable (upper) layer of overlayfs.

Without this PR AnyBuild cannot update its Linux image.

Related work items: #2000792, #2000797

.azdo/linux/step-install-mono.yml

@@ -4,6 +4,6 @@ steps:
 
     # install mono
     sudo apt-get update
-    sudo apt-get install -y mono-complete mono-devel 
+    sudo apt-get install -y mono-complete mono-devel
     mono --version
   displayName: Install Mono

Public/Src/Sandbox/Linux/UnitTests/BuildXL.Sandbox.Linux.UnitTests.dsc

@@ -0,0 +1,78 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import {Cmd, Artifact, Transformer} from "Sdk.Transformers";
+import {Sandbox} from "BuildXL.Sandbox.Linux";
+import * as boost from "boost";
+
+namespace UnitTests
+{
+    export declare const qualifier : {
+        configuration: "debug" | "release",
+        targetRuntime: "linux-x64"
+    };
+
+    const testFiles = glob(d`.`, "*.cpp");
+    const isLinux = Context.getCurrentHost().os === "unix";
+    const gxx = getTool("g++");
+    const env = getTool("env");
+    const libDetours = Sandbox.libDetours;
+    const boostLibDir = boost.Contents.all.ensureContents({subFolder: r`lib/native/include`});
+
+    @@public
+    export const tests = testFiles.map(s => runBoostTest(compileForBoost(s)));
+
+    function compileForBoost(srcFile: File) : DerivedFile
+    {
+        if (!isLinux) return undefined;
+
+        const compiler = gxx;
+        const outDir = Context.getNewOutputDirectory(compiler.exe.name);
+        const exeFile = p`${outDir}/${srcFile.name.changeExtension("")}`;
+        const result = Transformer.execute({
+            tool: compiler,
+            workingDirectory: outDir,
+            dependencies: [boostLibDir],
+            arguments: [
+                Cmd.option("-I ", Artifact.none(boostLibDir)),
+                Cmd.argument(Artifact.input(srcFile)),
+                Cmd.option("-o ", Artifact.output(exeFile)),
+            ]
+        });
+        return result.getOutputFile(exeFile);
+    }
+
+    function runBoostTest(exeFile: File) : TransformerExecuteResult
+    {
+        if (!isLinux) return undefined;
+
+        const workDir = Context.getNewOutputDirectory(exeFile.name);
+        const outDir = Context.getNewOutputDirectory(exeFile.name);
+        return Transformer.execute({
+            tool: env,
+            workingDirectory: workDir,
+            arguments: [
+                Cmd.option("LD_PRELOAD=", Artifact.input(libDetours)),
+                Cmd.argument(Artifact.input(exeFile)),
+            ],
+            outputs: [{existence: "optional", artifact: p`${outDir}/dummy`}],
+            unsafe: {
+                untrackedScopes: [ workDir ],
+                // Using nested interpose does not work.
+                disableSandboxing: true
+            }
+        });
+    }
+
+    function getTool(toolName: string) : Transformer.ToolDefinition {
+        if (!isLinux) return undefined;
+
+        return {
+            exe: f`/usr/bin/${toolName}`,
+            dependsOnCurrentHostOSDirectories: true,
+            prepareTempDirectory: true,
+            untrackedDirectoryScopes: [ d`/lib` ],
+            untrackedFiles: []
+        };
+    }
+}

Public/Src/Sandbox/Linux/UnitTests/interpose.cpp

@@ -0,0 +1,36 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+#define BOOST_TEST_MODULE LinuxSandboxTest
+#define _DO_NOT_EXPORT
+
+#include <boost/test/included/unit_test.hpp>
+#include <iostream>
+#include <fstream>
+#include <fcntl.h>
+
+using namespace std;
+
+BOOST_AUTO_TEST_SUITE(InterposeTests)
+
+BOOST_AUTO_TEST_CASE(TestCopyFileRange)
+{
+    int data_len = 100;
+    int copy_len = 50;
+    string input = "input.txt";
+    string output = "output.txt";
+
+    string data(data_len, 'd');
+    ofstream in_stream("input.txt");
+    in_stream << data;
+    in_stream.close();
+
+    int fdIn = open(input.c_str(), O_RDONLY);
+    int fdOut = open(output.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0644);
+    int copied = copy_file_range(fdIn, 0, fdOut, 0, copy_len, 0);
+    BOOST_CHECK_EQUAL(copied, copy_len);
+    close(fdIn);
+    close(fdOut);
+}
+
+BOOST_AUTO_TEST_SUITE_END()

Public/Src/Sandbox/Linux/UnitTests/module.config.dsc

@@ -0,0 +1,6 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+module({
+    name: "BuildXL.Sandbox.Linux.UnitTests"
+});

Public/Src/Sandbox/Linux/detours.cpp

@@ -19,6 +19,7 @@
 #include <sys/param.h>
 #include <sys/mount.h>
 #include <sys/stat.h>
+#include <sys/sysmacros.h>
 #include <sys/fcntl.h>
 #include <sys/xattr.h>
 
@@ -612,7 +613,77 @@ INTERPOSE(ssize_t, sendfile64, int out_fd, int in_fd, off_t *offset, size_t coun
 
 INTERPOSE(ssize_t, copy_file_range, int fd_in, loff_t *off_in, int fd_out, loff_t *off_out, size_t len, unsigned int flags)({
     auto check = bxl->report_access_fd(__func__, ES_EVENT_TYPE_NOTIFY_WRITE, fd_out);
-    return bxl->check_and_fwd_copy_file_range(check, (ssize_t)ERROR_RETURN_VALUE, fd_in, off_in, fd_out, off_out, len, flags);
+    if (bxl->should_deny(check)) {
+        errno = EPERM;
+        return (ssize_t)ERROR_RETURN_VALUE;
+    }
+
+    // TODO: Remove the following workaround when the kernel bug is fixed.
+    //
+    // Due to (possibly) kernel bug, copy_file_range does no longer work when the file descriptors are not mounted on the same
+    // filesystems, despite what is said in the manual https://man7.org/linux/man-pages/man2/copy_file_range.2.html.
+    // This bug breaks AnyBuild virtual filesystem (VFS) because the source file will be in the read-only (lower) layer of overlayfs, and this
+    // layer is mounted on AnyBuild FUSE, and the target file will be in the writable (upper) layer of overlayfs.
+    //
+    // In the commented code below, we try to check if the file descriptors are mounted on the same filesystem, and if so, we simply call
+    // copy_file_range. On the user space, the descriptors are mounted on the same filesystem. However, when copy_file_range is called,
+    // and the call goes into the kernel space, those descriptors are identified from different filesystems, and so the call will fail with EXDEV.
+    //
+    // ------------------------------------------------------------------------------------------
+    // struct stat st_in;
+    // if (fstat(fd_in, &st_in) == -1)
+    //     return (ssize_t)ERROR_RETURN_VALUE;
+    // struct stat st_out;
+    // if (fstat(fd_out, &st_out) == -1)
+    //     return (ssize_t)ERROR_RETURN_VALUE;
+    // errno = 0;
+    // if ((uintmax_t)major(st_in.st_dev) == (uintmax_t)major(st_out.st_dev)
+    //     && (uintmax_t)minor(st_in.st_dev) == (uintmax_t)minor(st_out.st_dev))
+    //     return bxl->fwd_copy_file_range(fd_in, off_in, fd_out, off_out, len, flags).restore();
+    // ------------------------------------------------------------------------------------------
+
+    // The code below implements copy_file_range using splice(2). The idea is the content is first
+    // copied to the pipe and then transferred to the target.
+
+    // Check for flags.
+    if (flags != 0) {
+        errno = EINVAL;
+        return (ssize_t)ERROR_RETURN_VALUE;
+    }
+
+    // Check for overlapped range.
+    if (fd_in == fd_out) {
+        off64_t start_off_in = off_in == NULL ? lseek(fd_in, 0, SEEK_CUR) : *off_in;
+        off64_t end_off_in = start_off_in + len;
+        off64_t start_off_out = off_out == NULL ? lseek(fd_out, 0, SEEK_CUR) : *off_out;
+        off64_t end_off_out = start_off_out + len;
+        if ((start_off_in <= end_off_out && end_off_in >= start_off_out)
+            || (start_off_out <= end_off_in && end_off_out >= start_off_in)) {
+                errno = EINVAL;
+                return (ssize_t)ERROR_RETURN_VALUE;
+            }
+    }
+
+    errno = 0;
+
+    // Creates a pipe.
+    int pipefd[2];
+    ssize_t result = pipe(pipefd);
+    if (result < 0)
+        return result;
+
+    // Copy from input to pipe.
+    result = splice(fd_in, off_in, pipefd[1], NULL, len, 0);
+    if (result < 0)
+        goto exit;
+
+    // Copy from pipe to output.
+    result = splice(pipefd[0], NULL, fd_out, off_out, result, 0);
+
+exit:
+    close(pipefd[0]);
+    close(pipefd[1]);
+    return result;
 })
 
 INTERPOSE(int, name_to_handle_at, int dirfd, const char *pathname, struct file_handle *handle, int *mount_id, int flags)({

Public/Src/Sandbox/Linux/mknuget.sh

@@ -21,7 +21,7 @@ touch "${pkgName}/ref/netstandard/_._"
 mkdir -p "${pkgName}/runtimes/linux-x64/native"
 cp -r debug release "${pkgName}/runtimes/linux-x64/native"
 
-cat > "${pkgName}/runtime.linux-x64.buildxl.nuspec" <<EOF
+cat > "${pkgName}/runtime.linux-x64.BuildXL.nuspec" <<EOF
 <?xml version="1.0" encoding="utf-8"?>
 <package xmlns="http://schemas.microsoft.com/packaging/2013/01/nuspec.xsd">
   <metadata minClientVersion="2.12">

cg/nuget/cgmanifest.json

@@ -4614,7 +4614,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "runtime.linux-x64.BuildXL",
-          "Version": "0.1.0-20220726.16"
+          "Version": "0.1.0-20221024.27.1"
         }
       }
     },

config.dsc

@@ -419,7 +419,7 @@ config({
                 // Runtime dependencies for Linux
                 {
                     id: "runtime.linux-x64.BuildXL",
-                    version: "0.1.0-20220726.16",
+                    version: "0.1.0-20221024.27.1",
                     osSkip: importFile(f`config.microsoftInternal.dsc`).isMicrosoftInternal
                         ? []
                         : [ "win", "macOS", "unix" ]