From 53afbeb6345d5e23a40c8c3b6b93eecc2d0ad46c Mon Sep 17 00:00:00 2001 From: Evgeny Budilovsky Date: Mon, 6 Jan 2025 15:30:21 +0200 Subject: [PATCH] [Filestore] issue-2806: support multiple aio engines in local filestore Additionally dispatch async read/write into thread pool and increase default number of worker threads to 8 --- .../filestore/libs/daemon/vhost/bootstrap.cpp | 2 + cloud/filestore/libs/service_local/config.cpp | 2 +- .../filestore/libs/service_local/service.cpp | 4 +- cloud/filestore/libs/vfs_fuse/loop.cpp | 3 +- cloud/storage/core/libs/aio/service.cpp | 64 ++++++++++++++++++- cloud/storage/core/libs/aio/service.h | 2 + 6 files changed, 73 insertions(+), 4 deletions(-) diff --git a/cloud/filestore/libs/daemon/vhost/bootstrap.cpp b/cloud/filestore/libs/daemon/vhost/bootstrap.cpp index 26143541208..8936e7aa892 100644 --- a/cloud/filestore/libs/daemon/vhost/bootstrap.cpp +++ b/cloud/filestore/libs/daemon/vhost/bootstrap.cpp @@ -32,6 +32,7 @@ #include #include +#include #include #include #include @@ -285,6 +286,7 @@ void TBootstrapVhost::InitEndpoints() auto serviceConfig = std::make_shared( *localServiceConfig); ThreadPool = CreateThreadPool("svc", serviceConfig->GetNumThreads()); + FileIOService = CreateThreadedAIOService(serviceConfig->GetNumThreads()); LocalService = CreateLocalFileStore( std::move(serviceConfig), Timer, diff --git a/cloud/filestore/libs/service_local/config.cpp b/cloud/filestore/libs/service_local/config.cpp index b9ca4ac02c7..0db574f37dd 100644 --- a/cloud/filestore/libs/service_local/config.cpp +++ b/cloud/filestore/libs/service_local/config.cpp @@ -19,7 +19,7 @@ constexpr TDuration AsyncHandleOpsPeriod = TDuration::MilliSeconds(50); xxx(PathPrefix, TString, "nfs_" )\ xxx(DefaultPermissions, ui32, 0775 )\ xxx(IdleSessionTimeout, TDuration, TDuration::Seconds(30) )\ - xxx(NumThreads, ui32, 4 )\ + xxx(NumThreads, ui32, 8 )\ xxx(StatePath, TString, "./" )\ xxx(MaxNodeCount, ui32, 1000000 )\ xxx(MaxHandlePerSessionCount, ui32, 10000 )\ diff --git a/cloud/filestore/libs/service_local/service.cpp b/cloud/filestore/libs/service_local/service.cpp index d4c891a8931..9eb5bfdc08a 100644 --- a/cloud/filestore/libs/service_local/service.cpp +++ b/cloud/filestore/libs/service_local/service.cpp @@ -207,7 +207,9 @@ class TLocalFileStore final std::shared_ptr request) override \ { \ Y_UNUSED(callContext); \ - return ExecuteWithProfileLogAsync(*request); \ + return TaskQueue->Execute([this, request = std::move(request)] { \ + return ExecuteWithProfileLogAsync(*request); \ + }); \ } \ // FILESTORE_IMPLEMENT_METHOD_ASYNC diff --git a/cloud/filestore/libs/vfs_fuse/loop.cpp b/cloud/filestore/libs/vfs_fuse/loop.cpp index be7a70994db..2e6f51ddf56 100644 --- a/cloud/filestore/libs/vfs_fuse/loop.cpp +++ b/cloud/filestore/libs/vfs_fuse/loop.cpp @@ -474,7 +474,8 @@ class TSessionThread final { STORAGE_INFO("starting FUSE loop"); - ::NCloud::SetCurrentThreadName("FUSE"); + static std::atomic index = 0; + ::NCloud::SetCurrentThreadName("FUSE" + ToString(index++)); AtomicSet(ThreadId, pthread_self()); fuse_session_loop(Session); diff --git a/cloud/storage/core/libs/aio/service.cpp b/cloud/storage/core/libs/aio/service.cpp index 5bb1b50c2ec..360746492f2 100644 --- a/cloud/storage/core/libs/aio/service.cpp +++ b/cloud/storage/core/libs/aio/service.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include @@ -223,7 +224,9 @@ class TAIOService final void Run() { SetHighestThreadPriority(); - NCloud::SetCurrentThreadName("AIO"); + + static std::atomic index = 0; + NCloud::SetCurrentThreadName("AIO" + ToString(index++)); timespec timeout = WAIT_TIMEOUT; @@ -248,6 +251,60 @@ class TAIOService final } }; +//////////////////////////////////////////////////////////////////////////////// + +class TThreadedAIOService final + : public IFileIOService +{ +private: + TVector IoServices; + std::atomic NextService = 0; + +public: + TThreadedAIOService(ui32 threadCount, size_t maxEvents) + { + for (ui32 i = 0; i < threadCount; i++) { + IoServices.push_back(CreateAIOService(maxEvents)); + } + } + + void AsyncRead( + TFileHandle& file, + i64 offset, + TArrayRef buffer, + TFileIOCompletion* completion) override + { + auto index = NextService++; + IoServices[index % IoServices.size()] + ->AsyncRead(file, offset, buffer, completion); + } + + void AsyncWrite( + TFileHandle& file, + i64 offset, + TArrayRef buffer, + TFileIOCompletion* completion) override + { + auto index = NextService++; + IoServices[index % IoServices.size()] + ->AsyncWrite(file, offset, buffer, completion); + } + + void Start() override + { + for (auto& ioService : IoServices) { + ioService->Start(); + } + } + + void Stop() override + { + for (auto& ioService : IoServices) { + ioService->Stop(); + } + } +}; + } // namespace //////////////////////////////////////////////////////////////////////////////// @@ -257,4 +314,9 @@ IFileIOServicePtr CreateAIOService(size_t maxEvents) return std::make_shared(maxEvents); } +IFileIOServicePtr CreateThreadedAIOService(ui32 threadCount, size_t maxEvents) +{ + return std::make_shared(threadCount, maxEvents); +} + } // namespace NCloud diff --git a/cloud/storage/core/libs/aio/service.h b/cloud/storage/core/libs/aio/service.h index 53e67a769a9..5af7a6f916c 100644 --- a/cloud/storage/core/libs/aio/service.h +++ b/cloud/storage/core/libs/aio/service.h @@ -8,4 +8,6 @@ namespace NCloud { IFileIOServicePtr CreateAIOService(size_t maxEvents = 1024); +IFileIOServicePtr CreateThreadedAIOService(ui32 threadCount, size_t maxEvents = 1024); + } // namespace NCloud