Search for post-build-check absolute paths in parallel (#1213)

include/vcpkg/base/parallel-algorithms.h

@@ -0,0 +1,85 @@
+#pragma once
+
+#include <vcpkg/base/system.h>
+
+#include <atomic>
+#include <future>
+#include <vector>
+
+namespace vcpkg
+{
+    template<class F>
+    inline void execute_in_parallel(size_t work_count, F&& work) noexcept
+    {
+        const size_t thread_count = static_cast<size_t>(get_concurrency());
+        const size_t num_threads = std::max(static_cast<size_t>(1), std::min(thread_count, work_count));
+
+        std::vector<std::future<void>> workers;
+        workers.reserve(num_threads - 1);
+
+        for (size_t i = 0; i < num_threads - 1; ++i)
+        {
+            workers.emplace_back(std::async(std::launch::async | std::launch::deferred, [&work]() { work(); }));
+        }
+        work();
+
+        for (auto&& w : workers)
+        {
+            w.get();
+        }
+    }
+
+    template<class RanIt, class F>
+    void parallel_for_each_n(RanIt begin, size_t work_count, F cb) noexcept
+    {
+        if (work_count == 0)
+        {
+            return;
+        }
+        if (work_count == 1)
+        {
+            cb(*begin);
+            return;
+        }
+
+        std::atomic_size_t next{0};
+
+        execute_in_parallel(work_count, [&]() {
+            size_t i = 0;
+            while (i < work_count)
+            {
+                if (next.compare_exchange_weak(i, i + 1, std::memory_order_relaxed))
+                {
+                    cb(*(begin + i));
+                }
+            }
+        });
+    }
+
+    template<class RanItSource, class RanItTarget, class F>
+    void parallel_transform(RanItSource begin, size_t work_count, RanItTarget out_begin, F&& cb) noexcept
+    {
+        if (work_count == 0)
+        {
+            return;
+        }
+        if (work_count == 1)
+        {
+            *out_begin = cb(*begin);
+            return;
+        }
+
+        std::atomic_size_t next{0};
+
+        execute_in_parallel(work_count, [&]() {
+            size_t i = 0;
+            while (i < work_count)
+            {
+                if (next.compare_exchange_weak(i, i + 1, std::memory_order_relaxed))
+                {
+                    *(out_begin + i) = cb(*(begin + i));
+                }
+            }
+        });
+    }
+}

src/vcpkg/base/system.process.cpp

@@ -3,6 +3,7 @@
 #include <vcpkg/base/checks.h>
 #include <vcpkg/base/chrono.h>
 #include <vcpkg/base/files.h>
+#include <vcpkg/base/parallel-algorithms.h>
 #include <vcpkg/base/parse.h>
 #include <vcpkg/base/strings.h>
 #include <vcpkg/base/system.debug.h>
@@ -674,44 +675,11 @@ namespace vcpkg
                                                                                       const Environment& env)
     {
         std::vector<ExpectedL<ExitCodeAndOutput>> res(cmd_lines.size(), LocalizedString{});
-        if (cmd_lines.empty())
-        {
-            return res;
-        }
 
-        if (cmd_lines.size() == 1)
+        parallel_transform(cmd_lines.begin(), cmd_lines.size(), res.begin(), [&](const Command& cmd_line) {
-        {
+            return cmd_execute_and_capture_output(cmd_line, wd, env);
-            res[0] = cmd_execute_and_capture_output(cmd_lines[0], wd, env);
+        });
-            return res;
-        }
 
-        std::atomic<size_t> work_item{0};
-        const auto num_threads =
-            std::max(static_cast<size_t>(1), std::min(static_cast<size_t>(get_concurrency()), cmd_lines.size()));
-
-        auto work = [&]() {
-            std::size_t item;
-            while (item = work_item.fetch_add(1), item < cmd_lines.size())
-            {
-                res[item] = cmd_execute_and_capture_output(cmd_lines[item], wd, env);
-            }
-        };
-
-        std::vector<std::future<void>> workers;
-        workers.reserve(num_threads - 1);
-        for (size_t x = 0; x < num_threads - 1; ++x)
-        {
-            workers.emplace_back(std::async(std::launch::async | std::launch::deferred, work));
-            if (work_item >= cmd_lines.size())
-            {
-                break;
-            }
-        }
-        work();
-        for (auto&& w : workers)
-        {
-            w.get();
-        }
         return res;
     }
 

src/vcpkg/postbuildlint.cpp

@@ -2,6 +2,7 @@
 #include <vcpkg/base/files.h>
 #include <vcpkg/base/message_sinks.h>
 #include <vcpkg/base/messages.h>
+#include <vcpkg/base/parallel-algorithms.h>
 #include <vcpkg/base/system.debug.h>
 #include <vcpkg/base/system.process.h>
 #include <vcpkg/base/util.h>
@@ -1281,13 +1282,16 @@ namespace vcpkg
         const auto stringview_paths = Util::fmap(string_paths, [](std::string& s) { return StringView(s); });
 
         std::vector<Path> failing_files;
-        for (auto&& file : fs.get_regular_files_recursive(dir, IgnoreErrors{}))
+        std::mutex mtx;
-        {
+        auto files = fs.get_regular_files_recursive(dir, IgnoreErrors{});
+
+        parallel_for_each_n(files.begin(), files.size(), [&](const Path& file) {
             if (file_contains_absolute_paths(fs, file, stringview_paths))
             {
+                std::lock_guard lock{mtx};
                 failing_files.push_back(file);
             }
-        }
+        });
 
         if (failing_files.empty())
         {