From ecd160e602a8877c8320c1858856bb2ebb4dc09d Mon Sep 17 00:00:00 2001 From: Stepanyuk Vladislav Date: Thu, 16 Jan 2025 09:20:35 +0000 Subject: [PATCH 01/15] issue-2542: add partition stop request --- cloud/blockstore/libs/storage/api/volume.h | 44 ++++++++------- .../libs/storage/protos/volume.proto | 22 ++++++++ .../storage/service/service_actor_destroy.cpp | 28 +++++++++- .../libs/storage/volume/testlib/test_env.cpp | 6 +++ .../libs/storage/volume/testlib/test_env.h | 3 ++ .../libs/storage/volume/volume_actor.cpp | 4 +- .../storage/volume/volume_actor_startstop.cpp | 47 ++++++++++++++++ .../libs/storage/volume/volume_ut.cpp | 54 +++++++++++++++++++ 8 files changed, 185 insertions(+), 23 deletions(-) diff --git a/cloud/blockstore/libs/storage/api/volume.h b/cloud/blockstore/libs/storage/api/volume.h index 1e2e18e6fd4..81749c1db4b 100644 --- a/cloud/blockstore/libs/storage/api/volume.h +++ b/cloud/blockstore/libs/storage/api/volume.h @@ -15,26 +15,27 @@ namespace NCloud::NBlockStore::NStorage { //////////////////////////////////////////////////////////////////////////////// #define BLOCKSTORE_VOLUME_REQUESTS(xxx, ...) \ - xxx(AddClient, __VA_ARGS__) \ - xxx(RemoveClient, __VA_ARGS__) \ - xxx(WaitReady, __VA_ARGS__) \ - xxx(DescribeBlocks, __VA_ARGS__) \ - xxx(GetPartitionInfo, __VA_ARGS__) \ - xxx(CompactRange, __VA_ARGS__) \ - xxx(GetCompactionStatus, __VA_ARGS__) \ - xxx(ReallocateDisk, __VA_ARGS__) \ - xxx(GetVolumeLoadInfo, __VA_ARGS__) \ - xxx(GetUsedBlocks, __VA_ARGS__) \ - xxx(DeleteCheckpointData, __VA_ARGS__) \ - xxx(UpdateUsedBlocks, __VA_ARGS__) \ - xxx(RebuildMetadata, __VA_ARGS__) \ - xxx(GetRebuildMetadataStatus, __VA_ARGS__) \ - xxx(ScanDisk, __VA_ARGS__) \ - xxx(GetScanDiskStatus, __VA_ARGS__) \ - xxx(GetVolumeInfo, __VA_ARGS__) \ - xxx(UpdateVolumeParams, __VA_ARGS__) \ - xxx(ChangeStorageConfig, __VA_ARGS__) \ - xxx(GetStorageConfig, __VA_ARGS__) \ + xxx(AddClient, __VA_ARGS__) \ + xxx(RemoveClient, __VA_ARGS__) \ + xxx(WaitReady, __VA_ARGS__) \ + xxx(DescribeBlocks, __VA_ARGS__) \ + xxx(GetPartitionInfo, __VA_ARGS__) \ + xxx(CompactRange, __VA_ARGS__) \ + xxx(GetCompactionStatus, __VA_ARGS__) \ + xxx(ReallocateDisk, __VA_ARGS__) \ + xxx(GetVolumeLoadInfo, __VA_ARGS__) \ + xxx(GetUsedBlocks, __VA_ARGS__) \ + xxx(DeleteCheckpointData, __VA_ARGS__) \ + xxx(UpdateUsedBlocks, __VA_ARGS__) \ + xxx(RebuildMetadata, __VA_ARGS__) \ + xxx(GetRebuildMetadataStatus, __VA_ARGS__) \ + xxx(ScanDisk, __VA_ARGS__) \ + xxx(GetScanDiskStatus, __VA_ARGS__) \ + xxx(GetVolumeInfo, __VA_ARGS__) \ + xxx(UpdateVolumeParams, __VA_ARGS__) \ + xxx(ChangeStorageConfig, __VA_ARGS__) \ + xxx(GetStorageConfig, __VA_ARGS__) \ + xxx(StopPartionBeforeVolumeDestruction, __VA_ARGS__) \ // BLOCKSTORE_VOLUME_REQUESTS @@ -331,6 +332,9 @@ struct TEvVolume EvGetStorageConfigRequest = EvBegin + 58, EvGetStorageConfigResponse = EvBegin + 59, + EvStopPartionBeforeVolumeDestructionRequest = EvBegin + 60, + EvStopPartionBeforeVolumeDestructionResponse = EvBegin + 61, + EvEnd }; diff --git a/cloud/blockstore/libs/storage/protos/volume.proto b/cloud/blockstore/libs/storage/protos/volume.proto index dfae6356a6a..3d77f8e1249 100644 --- a/cloud/blockstore/libs/storage/protos/volume.proto +++ b/cloud/blockstore/libs/storage/protos/volume.proto @@ -652,3 +652,25 @@ message TGetStorageConfigResponse // Result Storage config. NProto.TStorageServiceConfig StorageConfig = 3; } + +//////////////////////////////////////////////////////////////////////////////// +// StopPartionBeforeVolumeDestruction request/response. + +message TStopPartionBeforeVolumeDestructionRequest +{ + // Optional request headers. + THeaders Headers = 1; + + // Label of volume to query its storage config. Or empty to + // query server storage config. + string DiskId = 2; +} + +message TStopPartionBeforeVolumeDestructionResponse +{ + // Optional error, set only if error happened. + NCloud.NProto.TError Error = 1; + + // Request traces. + NCloud.NProto.TTraceInfo Trace = 2; +} \ No newline at end of file diff --git a/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp b/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp index b78674d6e2f..689843a8633 100644 --- a/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp +++ b/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp @@ -57,6 +57,7 @@ class TDestroyVolumeActor final void NotifyDiskRegistry(const TActorContext& ctx); void StatVolume(const TActorContext& ctx); void DeallocateDisk(const TActorContext& ctx); + void StopPartitions(const TActorContext& ctx); NProto::TError CheckIfDestructionIsAllowed() const; void HandleModifyResponse( @@ -79,6 +80,11 @@ class TDestroyVolumeActor final const TEvDiskRegistry::TEvDeallocateDiskResponse::TPtr& ev, const TActorContext& ctx); + void HandleStopPartitions( + const TEvVolume::TEvStopPartionBeforeVolumeDestructionResponse::TPtr& + ev, + const TActorContext& ctx); + void ReplyAndDie(const TActorContext& ctx, NProto::TError error); private: @@ -180,6 +186,14 @@ void TDestroyVolumeActor::DeallocateDisk(const TActorContext& ctx) NCloud::Send(ctx, MakeDiskRegistryProxyServiceId(), std::move(request)); } +void TDestroyVolumeActor::StopPartitions(const TActorContext& ctx) +{ + auto stopPartReq = std::make_unique< + TEvVolume::TEvStopPartionBeforeVolumeDestructionRequest>(); + stopPartReq->Record.SetDiskId(DiskId); + NCloud::Send(ctx, MakeVolumeProxyServiceId(), std::move(stopPartReq)); +} + NProto::TError TDestroyVolumeActor::CheckIfDestructionIsAllowed() const { const auto& prefixes = DestructionAllowedOnlyForDisksWithIdPrefixes; @@ -282,7 +296,7 @@ void TDestroyVolumeActor::HandleMarkDiskForCleanupResponse( return; } - DestroyVolume(ctx); + StopPartitions(ctx); } void TDestroyVolumeActor::HandleDeallocateDiskResponse( @@ -383,6 +397,14 @@ void TDestroyVolumeActor::HandleStatVolumeResponse( } } +void TDestroyVolumeActor::HandleStopPartitions( + const TEvVolume::TEvStopPartionBeforeVolumeDestructionResponse::TPtr& ev, + const TActorContext& ctx) +{ + Y_UNUSED(ev); + DestroyVolume(ctx); +} + void TDestroyVolumeActor::ReplyAndDie( const TActorContext& ctx, NProto::TError error) @@ -412,6 +434,10 @@ STFUNC(TDestroyVolumeActor::StateWork) TEvService::TEvStatVolumeResponse, HandleStatVolumeResponse); + HFunc( + TEvVolume::TEvStopPartionBeforeVolumeDestructionResponse, + HandleStopPartitions); + default: HandleUnexpectedEvent(ev, TBlockStoreComponents::SERVICE); break; diff --git a/cloud/blockstore/libs/storage/volume/testlib/test_env.cpp b/cloud/blockstore/libs/storage/volume/testlib/test_env.cpp index 87f58a1facc..b8e54e4fd60 100644 --- a/cloud/blockstore/libs/storage/volume/testlib/test_env.cpp +++ b/cloud/blockstore/libs/storage/volume/testlib/test_env.cpp @@ -551,6 +551,12 @@ TVolumeClient::CreateReadMetaHistoryRequest() return std::make_unique(); } +std::unique_ptr +TVolumeClient::CreateStopPartionBeforeVolumeDestructionRequest() { + return std::make_unique< + TEvVolume::TEvStopPartionBeforeVolumeDestructionRequest>(); +} + void TVolumeClient::SendRemoteHttpInfo( const TString& params, HTTP_METHOD method) diff --git a/cloud/blockstore/libs/storage/volume/testlib/test_env.h b/cloud/blockstore/libs/storage/volume/testlib/test_env.h index 6aaadd6f87e..21167d8affe 100644 --- a/cloud/blockstore/libs/storage/volume/testlib/test_env.h +++ b/cloud/blockstore/libs/storage/volume/testlib/test_env.h @@ -460,6 +460,9 @@ class TVolumeClient std::unique_ptr CreateReadMetaHistoryRequest(); + std::unique_ptr + CreateStopPartionBeforeVolumeDestructionRequest(); + void SendRemoteHttpInfo( const TString& params, HTTP_METHOD method); diff --git a/cloud/blockstore/libs/storage/volume/volume_actor.cpp b/cloud/blockstore/libs/storage/volume/volume_actor.cpp index c533063cbfa..4fab8d6248c 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor.cpp @@ -1125,8 +1125,8 @@ STFUNC(TVolumeActor::StateZombie) IgnoreFunc(TEvPartition::TEvWaitReadyResponse); - IgnoreFunc(TEvents::TEvPoisonPill); - IgnoreFunc(TEvents::TEvPoisonTaken); + HFunc(TEvents::TEvPoisonPill, HandlePoisonPill); + HFunc(TEvents::TEvPoisonTaken, HandlePoisonTaken); IgnoreFunc(TEvLocal::TEvTabletMetrics); diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp index 002fe1e9033..bc255860c8d 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp @@ -346,6 +346,53 @@ void TVolumeActor::StartPartitionsForGc(const TActorContext& ctx) PartitionsStartedReason = EPartitionsStartedReason::STARTED_FOR_GC; } +void TVolumeActor::HandleStopPartionBeforeVolumeDestruction( + const TEvVolume::TEvStopPartionBeforeVolumeDestructionRequest::TPtr& ev, + const TActorContext& ctx) +{ + if (!State->GetDiskRegistryBasedPartitionActor()) { + LOG_ERROR( + ctx, + TBlockStoreComponents::VOLUME, + "[%lu] StopPartionBeforeVolumeDestruction req was send to not DR " + "based " + "volume", + TabletID()); + + NCloud::Reply( + ctx, + *ev, + std::make_unique< + TEvVolume::TEvStopPartionBeforeVolumeDestructionResponse>( + MakeError(E_REJECTED, "request not supported"))); + return; + } + + LOG_DEBUG( + ctx, + TBlockStoreComponents::VOLUME, + "[%lu] Stop Partition before volume destruction", + TabletID()); + + CancelRequests(ctx); + + auto reqInfo = + CreateRequestInfo(ev->Sender, ev->Cookie, ev->Get()->CallContext); + StopPartitions( + ctx, + [reqInfo = std::move(reqInfo)](const auto& ctx) + { + NCloud::Reply( + ctx, + *reqInfo, + std::make_unique< + TEvVolume:: + TEvStopPartionBeforeVolumeDestructionResponse>()); + }); + BecomeAux(ctx, STATE_ZOMBIE); + +} + void TVolumeActor::StopPartitions( const TActorContext& ctx, TDiskRegistryBasedPartitionStoppedCallback onPartitionStopped) diff --git a/cloud/blockstore/libs/storage/volume/volume_ut.cpp b/cloud/blockstore/libs/storage/volume/volume_ut.cpp index f1c9f2287c5..0ea9d8afa48 100644 --- a/cloud/blockstore/libs/storage/volume/volume_ut.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_ut.cpp @@ -8148,6 +8148,60 @@ Y_UNIT_TEST_SUITE(TVolumeTest) UNIT_ASSERT(partitionsStopped); } + Y_UNIT_TEST(ShouldStopPartitionsOnMessage) + { + auto runtime = PrepareTestActorRuntime(); + TVolumeClient volume(*runtime); + + bool partitionsStopped = false; + runtime->SetEventFilter( + [&](TTestActorRuntimeBase&, TAutoPtr& event) + { + switch (event->GetTypeRewrite()) { + // Poison pill send to DR based partition actor. + case TEvents::TEvPoisonPill::EventType: { + partitionsStopped = true; + break; + } + } + return false; + }); + + volume.UpdateVolumeConfig( + // default arguments + 0, + 0, + 0, + 0, + false, + 1, + NCloud::NProto::EStorageMediaKind::STORAGE_MEDIA_SSD_NONREPLICATED, + 1024, + "vol0", + "cloud", + "folder", + 1 // partitions count + ); + volume.RebootTablet(); + + auto clientInfo = CreateVolumeClientInfo( + NProto::VOLUME_ACCESS_READ_WRITE, + NProto::VOLUME_MOUNT_LOCAL, + false); + + + volume.StopPartionBeforeVolumeDestruction(); + UNIT_ASSERT(partitionsStopped); + + // Check that volume after TEvStopPartionBeforeVolumeDestructionRequest + // in zombie state and rejects requsts. + volume.SendGetVolumeInfoRequest(); + auto response = volume.RecvGetVolumeInfoResponse(); + UNIT_ASSERT_VALUES_EQUAL( + response->GetStatus(), + E_REJECTED); + } + Y_UNIT_TEST(ShouldReturnClientsAndHostnameInStatVolumeResponse) { auto runtime = PrepareTestActorRuntime(); From 213270a5b3995012dc4c239add9e62b965dd320f Mon Sep 17 00:00:00 2001 From: Stepanyuk Vladislav Date: Thu, 16 Jan 2025 09:22:55 +0000 Subject: [PATCH 02/15] issue-2542: add new line at EOF --- cloud/blockstore/libs/storage/protos/volume.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/blockstore/libs/storage/protos/volume.proto b/cloud/blockstore/libs/storage/protos/volume.proto index 3d77f8e1249..ede25bb25b2 100644 --- a/cloud/blockstore/libs/storage/protos/volume.proto +++ b/cloud/blockstore/libs/storage/protos/volume.proto @@ -673,4 +673,4 @@ message TStopPartionBeforeVolumeDestructionResponse // Request traces. NCloud.NProto.TTraceInfo Trace = 2; -} \ No newline at end of file +} From d31391f12f468c2d1d08fa4287594cfbc92f665a Mon Sep 17 00:00:00 2001 From: Stepanyuk Vladislav Date: Thu, 16 Jan 2025 09:31:08 +0000 Subject: [PATCH 03/15] issue-2542: correct log msg --- .../blockstore/libs/storage/volume/volume_actor_startstop.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp index bc255860c8d..4838d2249ed 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp @@ -355,8 +355,7 @@ void TVolumeActor::HandleStopPartionBeforeVolumeDestruction( ctx, TBlockStoreComponents::VOLUME, "[%lu] StopPartionBeforeVolumeDestruction req was send to not DR " - "based " - "volume", + "based disk", TabletID()); NCloud::Reply( From 31f1f584f62a2b752829c0070435f0b7c0ede0ae Mon Sep 17 00:00:00 2001 From: Stepanyuk Vladislav Date: Fri, 17 Jan 2025 10:52:38 +0000 Subject: [PATCH 04/15] issue-2542: correct issues --- .../libs/storage/volume/volume_actor_startstop.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp index 4838d2249ed..1cd881b41cc 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp @@ -354,8 +354,8 @@ void TVolumeActor::HandleStopPartionBeforeVolumeDestruction( LOG_ERROR( ctx, TBlockStoreComponents::VOLUME, - "[%lu] StopPartionBeforeVolumeDestruction req was send to not DR " - "based disk", + "[%lu] StopPartionBeforeVolumeDestruction request was send to not " + "DR based disk", TabletID()); NCloud::Reply( @@ -363,11 +363,11 @@ void TVolumeActor::HandleStopPartionBeforeVolumeDestruction( *ev, std::make_unique< TEvVolume::TEvStopPartionBeforeVolumeDestructionResponse>( - MakeError(E_REJECTED, "request not supported"))); + MakeError(E_NOT_IMPLEMENTED, "request not supported"))); return; } - LOG_DEBUG( + LOG_INFO( ctx, TBlockStoreComponents::VOLUME, "[%lu] Stop Partition before volume destruction", From bc1252232e380bc2dba9e2f802bf51384d9d2cbb Mon Sep 17 00:00:00 2001 From: Stepanyuk Vladislav Date: Fri, 17 Jan 2025 11:18:01 +0000 Subject: [PATCH 05/15] issue-2542: fix build issue --- cloud/blockstore/libs/storage/volume/volume_actor.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cloud/blockstore/libs/storage/volume/volume_actor.cpp b/cloud/blockstore/libs/storage/volume/volume_actor.cpp index 4fab8d6248c..803e1de0361 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor.cpp @@ -1127,6 +1127,7 @@ STFUNC(TVolumeActor::StateZombie) HFunc(TEvents::TEvPoisonPill, HandlePoisonPill); HFunc(TEvents::TEvPoisonTaken, HandlePoisonTaken); + HFunc(TEvTablet::TEvTabletStop, HandleTabletStop); IgnoreFunc(TEvLocal::TEvTabletMetrics); From 2267137c9aec6f8fe6e04f475b2d6ae95689c9c4 Mon Sep 17 00:00:00 2001 From: Stepanyuk Vladislav Date: Fri, 17 Jan 2025 11:19:56 +0000 Subject: [PATCH 06/15] issue-2542: fmt --- cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp index 1cd881b41cc..82e89dd6523 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp @@ -389,7 +389,6 @@ void TVolumeActor::HandleStopPartionBeforeVolumeDestruction( TEvStopPartionBeforeVolumeDestructionResponse>()); }); BecomeAux(ctx, STATE_ZOMBIE); - } void TVolumeActor::StopPartitions( From 437a71f779981a136207483e397049a983aafb0c Mon Sep 17 00:00:00 2001 From: Stepanyuk Vladislav Date: Mon, 20 Jan 2025 16:26:12 +0700 Subject: [PATCH 07/15] issue-2542:correct message name and correct stop req handler --- cloud/blockstore/libs/storage/api/volume.h | 6 +++--- .../libs/storage/protos/volume.proto | 6 +++--- .../storage/service/service_actor_destroy.cpp | 8 ++++---- .../libs/storage/volume/testlib/test_env.cpp | 6 +++--- .../libs/storage/volume/testlib/test_env.h | 4 ++-- .../storage/volume/volume_actor_startstop.cpp | 18 +++++++++++------- .../libs/storage/volume/volume_ut.cpp | 4 ++-- 7 files changed, 28 insertions(+), 24 deletions(-) diff --git a/cloud/blockstore/libs/storage/api/volume.h b/cloud/blockstore/libs/storage/api/volume.h index 81749c1db4b..123ce988ad1 100644 --- a/cloud/blockstore/libs/storage/api/volume.h +++ b/cloud/blockstore/libs/storage/api/volume.h @@ -35,7 +35,7 @@ namespace NCloud::NBlockStore::NStorage { xxx(UpdateVolumeParams, __VA_ARGS__) \ xxx(ChangeStorageConfig, __VA_ARGS__) \ xxx(GetStorageConfig, __VA_ARGS__) \ - xxx(StopPartionBeforeVolumeDestruction, __VA_ARGS__) \ + xxx(StopPartitionBeforeVolumeDestruction, __VA_ARGS__) \ // BLOCKSTORE_VOLUME_REQUESTS @@ -332,8 +332,8 @@ struct TEvVolume EvGetStorageConfigRequest = EvBegin + 58, EvGetStorageConfigResponse = EvBegin + 59, - EvStopPartionBeforeVolumeDestructionRequest = EvBegin + 60, - EvStopPartionBeforeVolumeDestructionResponse = EvBegin + 61, + EvStopPartitionBeforeVolumeDestructionRequest = EvBegin + 60, + EvStopPartitionBeforeVolumeDestructionResponse = EvBegin + 61, EvEnd }; diff --git a/cloud/blockstore/libs/storage/protos/volume.proto b/cloud/blockstore/libs/storage/protos/volume.proto index ede25bb25b2..03870cf8243 100644 --- a/cloud/blockstore/libs/storage/protos/volume.proto +++ b/cloud/blockstore/libs/storage/protos/volume.proto @@ -654,9 +654,9 @@ message TGetStorageConfigResponse } //////////////////////////////////////////////////////////////////////////////// -// StopPartionBeforeVolumeDestruction request/response. +// StopPartitionBeforeVolumeDestruction request/response. -message TStopPartionBeforeVolumeDestructionRequest +message TStopPartitionBeforeVolumeDestructionRequest { // Optional request headers. THeaders Headers = 1; @@ -666,7 +666,7 @@ message TStopPartionBeforeVolumeDestructionRequest string DiskId = 2; } -message TStopPartionBeforeVolumeDestructionResponse +message TStopPartitionBeforeVolumeDestructionResponse { // Optional error, set only if error happened. NCloud.NProto.TError Error = 1; diff --git a/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp b/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp index 689843a8633..dbb51e47b77 100644 --- a/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp +++ b/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp @@ -81,7 +81,7 @@ class TDestroyVolumeActor final const TActorContext& ctx); void HandleStopPartitions( - const TEvVolume::TEvStopPartionBeforeVolumeDestructionResponse::TPtr& + const TEvVolume::TEvStopPartitionBeforeVolumeDestructionResponse::TPtr& ev, const TActorContext& ctx); @@ -189,7 +189,7 @@ void TDestroyVolumeActor::DeallocateDisk(const TActorContext& ctx) void TDestroyVolumeActor::StopPartitions(const TActorContext& ctx) { auto stopPartReq = std::make_unique< - TEvVolume::TEvStopPartionBeforeVolumeDestructionRequest>(); + TEvVolume::TEvStopPartitionBeforeVolumeDestructionRequest>(); stopPartReq->Record.SetDiskId(DiskId); NCloud::Send(ctx, MakeVolumeProxyServiceId(), std::move(stopPartReq)); } @@ -398,7 +398,7 @@ void TDestroyVolumeActor::HandleStatVolumeResponse( } void TDestroyVolumeActor::HandleStopPartitions( - const TEvVolume::TEvStopPartionBeforeVolumeDestructionResponse::TPtr& ev, + const TEvVolume::TEvStopPartitionBeforeVolumeDestructionResponse::TPtr& ev, const TActorContext& ctx) { Y_UNUSED(ev); @@ -435,7 +435,7 @@ STFUNC(TDestroyVolumeActor::StateWork) HandleStatVolumeResponse); HFunc( - TEvVolume::TEvStopPartionBeforeVolumeDestructionResponse, + TEvVolume::TEvStopPartitionBeforeVolumeDestructionResponse, HandleStopPartitions); default: diff --git a/cloud/blockstore/libs/storage/volume/testlib/test_env.cpp b/cloud/blockstore/libs/storage/volume/testlib/test_env.cpp index b8e54e4fd60..1bbb1cac70f 100644 --- a/cloud/blockstore/libs/storage/volume/testlib/test_env.cpp +++ b/cloud/blockstore/libs/storage/volume/testlib/test_env.cpp @@ -551,10 +551,10 @@ TVolumeClient::CreateReadMetaHistoryRequest() return std::make_unique(); } -std::unique_ptr -TVolumeClient::CreateStopPartionBeforeVolumeDestructionRequest() { +std::unique_ptr +TVolumeClient::CreateStopPartitionBeforeVolumeDestructionRequest() { return std::make_unique< - TEvVolume::TEvStopPartionBeforeVolumeDestructionRequest>(); + TEvVolume::TEvStopPartitionBeforeVolumeDestructionRequest>(); } void TVolumeClient::SendRemoteHttpInfo( diff --git a/cloud/blockstore/libs/storage/volume/testlib/test_env.h b/cloud/blockstore/libs/storage/volume/testlib/test_env.h index 21167d8affe..91406a0a52b 100644 --- a/cloud/blockstore/libs/storage/volume/testlib/test_env.h +++ b/cloud/blockstore/libs/storage/volume/testlib/test_env.h @@ -460,8 +460,8 @@ class TVolumeClient std::unique_ptr CreateReadMetaHistoryRequest(); - std::unique_ptr - CreateStopPartionBeforeVolumeDestructionRequest(); + std::unique_ptr + CreateStopPartitionBeforeVolumeDestructionRequest(); void SendRemoteHttpInfo( const TString& params, diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp index 82e89dd6523..73ce240d4c8 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp @@ -346,15 +346,16 @@ void TVolumeActor::StartPartitionsForGc(const TActorContext& ctx) PartitionsStartedReason = EPartitionsStartedReason::STARTED_FOR_GC; } -void TVolumeActor::HandleStopPartionBeforeVolumeDestruction( - const TEvVolume::TEvStopPartionBeforeVolumeDestructionRequest::TPtr& ev, +void TVolumeActor::HandleStopPartitionBeforeVolumeDestruction( + const TEvVolume::TEvStopPartitionBeforeVolumeDestructionRequest::TPtr& ev, const TActorContext& ctx) { if (!State->GetDiskRegistryBasedPartitionActor()) { LOG_ERROR( ctx, TBlockStoreComponents::VOLUME, - "[%lu] StopPartionBeforeVolumeDestruction request was send to not " + "[%lu] StopPartitionBeforeVolumeDestruction request was send to " + "not " "DR based disk", TabletID()); @@ -362,7 +363,7 @@ void TVolumeActor::HandleStopPartionBeforeVolumeDestruction( ctx, *ev, std::make_unique< - TEvVolume::TEvStopPartionBeforeVolumeDestructionResponse>( + TEvVolume::TEvStopPartitionBeforeVolumeDestructionResponse>( MakeError(E_NOT_IMPLEMENTED, "request not supported"))); return; } @@ -373,8 +374,6 @@ void TVolumeActor::HandleStopPartionBeforeVolumeDestruction( "[%lu] Stop Partition before volume destruction", TabletID()); - CancelRequests(ctx); - auto reqInfo = CreateRequestInfo(ev->Sender, ev->Cookie, ev->Get()->CallContext); StopPartitions( @@ -386,8 +385,13 @@ void TVolumeActor::HandleStopPartionBeforeVolumeDestruction( *reqInfo, std::make_unique< TEvVolume:: - TEvStopPartionBeforeVolumeDestructionResponse>()); + TEvStopPartitionBeforeVolumeDestructionResponse>()); }); + + TerminateTransactions(ctx); + KillActors(ctx); + CancelRequests(ctx); + BecomeAux(ctx, STATE_ZOMBIE); } diff --git a/cloud/blockstore/libs/storage/volume/volume_ut.cpp b/cloud/blockstore/libs/storage/volume/volume_ut.cpp index 0ea9d8afa48..b295dd5f611 100644 --- a/cloud/blockstore/libs/storage/volume/volume_ut.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_ut.cpp @@ -8190,10 +8190,10 @@ Y_UNIT_TEST_SUITE(TVolumeTest) false); - volume.StopPartionBeforeVolumeDestruction(); + volume.StopPartitionBeforeVolumeDestruction(); UNIT_ASSERT(partitionsStopped); - // Check that volume after TEvStopPartionBeforeVolumeDestructionRequest + // Check that volume after TEvStopPartitionBeforeVolumeDestructionRequest // in zombie state and rejects requsts. volume.SendGetVolumeInfoRequest(); auto response = volume.RecvGetVolumeInfoResponse(); From 55a01fa93d2426eeaabf28522699f83dc63b0c57 Mon Sep 17 00:00:00 2001 From: Stepanyuk Vladislav Date: Wed, 22 Jan 2025 07:30:48 +0000 Subject: [PATCH 08/15] issue-2542: add timeout and rename request --- cloud/blockstore/config/storage.proto | 3 + cloud/blockstore/libs/storage/api/volume.h | 46 +++++++-------- cloud/blockstore/libs/storage/core/config.h | 1 + .../libs/storage/protos/volume.proto | 6 +- .../storage/service/service_actor_destroy.cpp | 58 ++++++++++++++----- .../libs/storage/volume/testlib/test_env.cpp | 8 +-- .../libs/storage/volume/testlib/test_env.h | 4 +- .../storage/volume/volume_actor_startstop.cpp | 16 ++--- .../libs/storage/volume/volume_ut.cpp | 6 +- 9 files changed, 87 insertions(+), 61 deletions(-) diff --git a/cloud/blockstore/config/storage.proto b/cloud/blockstore/config/storage.proto index 2730433783e..60be1fe6137 100644 --- a/cloud/blockstore/config/storage.proto +++ b/cloud/blockstore/config/storage.proto @@ -1110,4 +1110,7 @@ message TStorageServiceConfig // Enabling direct sending AcquireDevices to disk agent. optional bool NonReplicatedVolumeDirectAcquireEnabled = 404; + + // Timeout for TDestroyVolumeActor (in milliseconds) + optional uint32 DestroyVolumeTimeout = 405; } diff --git a/cloud/blockstore/libs/storage/api/volume.h b/cloud/blockstore/libs/storage/api/volume.h index 123ce988ad1..515bf915ae1 100644 --- a/cloud/blockstore/libs/storage/api/volume.h +++ b/cloud/blockstore/libs/storage/api/volume.h @@ -15,27 +15,27 @@ namespace NCloud::NBlockStore::NStorage { //////////////////////////////////////////////////////////////////////////////// #define BLOCKSTORE_VOLUME_REQUESTS(xxx, ...) \ - xxx(AddClient, __VA_ARGS__) \ - xxx(RemoveClient, __VA_ARGS__) \ - xxx(WaitReady, __VA_ARGS__) \ - xxx(DescribeBlocks, __VA_ARGS__) \ - xxx(GetPartitionInfo, __VA_ARGS__) \ - xxx(CompactRange, __VA_ARGS__) \ - xxx(GetCompactionStatus, __VA_ARGS__) \ - xxx(ReallocateDisk, __VA_ARGS__) \ - xxx(GetVolumeLoadInfo, __VA_ARGS__) \ - xxx(GetUsedBlocks, __VA_ARGS__) \ - xxx(DeleteCheckpointData, __VA_ARGS__) \ - xxx(UpdateUsedBlocks, __VA_ARGS__) \ - xxx(RebuildMetadata, __VA_ARGS__) \ - xxx(GetRebuildMetadataStatus, __VA_ARGS__) \ - xxx(ScanDisk, __VA_ARGS__) \ - xxx(GetScanDiskStatus, __VA_ARGS__) \ - xxx(GetVolumeInfo, __VA_ARGS__) \ - xxx(UpdateVolumeParams, __VA_ARGS__) \ - xxx(ChangeStorageConfig, __VA_ARGS__) \ - xxx(GetStorageConfig, __VA_ARGS__) \ - xxx(StopPartitionBeforeVolumeDestruction, __VA_ARGS__) \ + xxx(AddClient, __VA_ARGS__) \ + xxx(RemoveClient, __VA_ARGS__) \ + xxx(WaitReady, __VA_ARGS__) \ + xxx(DescribeBlocks, __VA_ARGS__) \ + xxx(GetPartitionInfo, __VA_ARGS__) \ + xxx(CompactRange, __VA_ARGS__) \ + xxx(GetCompactionStatus, __VA_ARGS__) \ + xxx(ReallocateDisk, __VA_ARGS__) \ + xxx(GetVolumeLoadInfo, __VA_ARGS__) \ + xxx(GetUsedBlocks, __VA_ARGS__) \ + xxx(DeleteCheckpointData, __VA_ARGS__) \ + xxx(UpdateUsedBlocks, __VA_ARGS__) \ + xxx(RebuildMetadata, __VA_ARGS__) \ + xxx(GetRebuildMetadataStatus, __VA_ARGS__) \ + xxx(ScanDisk, __VA_ARGS__) \ + xxx(GetScanDiskStatus, __VA_ARGS__) \ + xxx(GetVolumeInfo, __VA_ARGS__) \ + xxx(UpdateVolumeParams, __VA_ARGS__) \ + xxx(ChangeStorageConfig, __VA_ARGS__) \ + xxx(GetStorageConfig, __VA_ARGS__) \ + xxx(GracefulShutdown, __VA_ARGS__) \ // BLOCKSTORE_VOLUME_REQUESTS @@ -332,8 +332,8 @@ struct TEvVolume EvGetStorageConfigRequest = EvBegin + 58, EvGetStorageConfigResponse = EvBegin + 59, - EvStopPartitionBeforeVolumeDestructionRequest = EvBegin + 60, - EvStopPartitionBeforeVolumeDestructionResponse = EvBegin + 61, + EvGracefulShutdownRequest = EvBegin + 60, + EvGracefulShutdownResponse = EvBegin + 61, EvEnd }; diff --git a/cloud/blockstore/libs/storage/core/config.h b/cloud/blockstore/libs/storage/core/config.h index bef384c063c..bf48eab3566 100644 --- a/cloud/blockstore/libs/storage/core/config.h +++ b/cloud/blockstore/libs/storage/core/config.h @@ -634,6 +634,7 @@ class TStorageConfig bool GetAutomaticallyEnableBufferCopyingAfterChecksumMismatch() const; [[nodiscard]] bool GetNonReplicatedVolumeDirectAcquireEnabled() const; + [[nodiscard]] TDuration GetDestroyVolumeTimeout() const; }; ui64 GetAllocationUnit( diff --git a/cloud/blockstore/libs/storage/protos/volume.proto b/cloud/blockstore/libs/storage/protos/volume.proto index 03870cf8243..c46ab4efae4 100644 --- a/cloud/blockstore/libs/storage/protos/volume.proto +++ b/cloud/blockstore/libs/storage/protos/volume.proto @@ -654,9 +654,9 @@ message TGetStorageConfigResponse } //////////////////////////////////////////////////////////////////////////////// -// StopPartitionBeforeVolumeDestruction request/response. +// GracefulShutdown request/response. -message TStopPartitionBeforeVolumeDestructionRequest +message TGracefulShutdownRequest { // Optional request headers. THeaders Headers = 1; @@ -666,7 +666,7 @@ message TStopPartitionBeforeVolumeDestructionRequest string DiskId = 2; } -message TStopPartitionBeforeVolumeDestructionResponse +message TGracefulShutdownResponse { // Optional error, set only if error happened. NCloud.NProto.TError Error = 1; diff --git a/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp b/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp index dbb51e47b77..62ef4b7bf28 100644 --- a/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp +++ b/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp @@ -34,6 +34,7 @@ class TDestroyVolumeActor final const bool DestroyIfBroken; const bool Sync; const ui64 FillGeneration; + const TDuration Timeout; bool IsDiskRegistryBased = false; bool VolumeNotFoundInSS = false; @@ -47,7 +48,8 @@ class TDestroyVolumeActor final TString diskId, bool destroyIfBroken, bool sync, - ui64 fillGeneration); + ui64 fillGeneration, + TDuration timeout); void Bootstrap(const TActorContext& ctx); @@ -57,7 +59,7 @@ class TDestroyVolumeActor final void NotifyDiskRegistry(const TActorContext& ctx); void StatVolume(const TActorContext& ctx); void DeallocateDisk(const TActorContext& ctx); - void StopPartitions(const TActorContext& ctx); + void GracefulShutdown(const TActorContext& ctx); NProto::TError CheckIfDestructionIsAllowed() const; void HandleModifyResponse( @@ -80,11 +82,15 @@ class TDestroyVolumeActor final const TEvDiskRegistry::TEvDeallocateDiskResponse::TPtr& ev, const TActorContext& ctx); - void HandleStopPartitions( - const TEvVolume::TEvStopPartitionBeforeVolumeDestructionResponse::TPtr& + void HandleGracefulShutdownResponse( + const TEvVolume::TEvGracefulShutdownResponse::TPtr& ev, const TActorContext& ctx); + void HandleTimeout( + const TEvents::TEvWakeup::TPtr& ev, + const TActorContext& ctx); + void ReplyAndDie(const TActorContext& ctx, NProto::TError error); private: @@ -101,7 +107,8 @@ TDestroyVolumeActor::TDestroyVolumeActor( TString diskId, bool destroyIfBroken, bool sync, - ui64 fillGeneration) + ui64 fillGeneration, + TDuration timeout) : Sender(sender) , Cookie(cookie) , AttachedDiskDestructionTimeout(attachedDiskDestructionTimeout) @@ -111,10 +118,12 @@ TDestroyVolumeActor::TDestroyVolumeActor( , DestroyIfBroken(destroyIfBroken) , Sync(sync) , FillGeneration(fillGeneration) + , Timeout(timeout) {} void TDestroyVolumeActor::Bootstrap(const TActorContext& ctx) { + ctx.Schedule(Timeout, new TEvents::TEvWakeup()); if (DestroyIfBroken) { WaitReady(ctx); } else { @@ -186,12 +195,12 @@ void TDestroyVolumeActor::DeallocateDisk(const TActorContext& ctx) NCloud::Send(ctx, MakeDiskRegistryProxyServiceId(), std::move(request)); } -void TDestroyVolumeActor::StopPartitions(const TActorContext& ctx) +void TDestroyVolumeActor::GracefulShutdown(const TActorContext& ctx) { - auto stopPartReq = std::make_unique< - TEvVolume::TEvStopPartitionBeforeVolumeDestructionRequest>(); - stopPartReq->Record.SetDiskId(DiskId); - NCloud::Send(ctx, MakeVolumeProxyServiceId(), std::move(stopPartReq)); + auto request = std::make_unique< + TEvVolume::TEvGracefulShutdownRequest>(); + request->Record.SetDiskId(DiskId); + NCloud::Send(ctx, MakeVolumeProxyServiceId(), std::move(request)); } NProto::TError TDestroyVolumeActor::CheckIfDestructionIsAllowed() const @@ -296,7 +305,7 @@ void TDestroyVolumeActor::HandleMarkDiskForCleanupResponse( return; } - StopPartitions(ctx); + GracefulShutdown(ctx); } void TDestroyVolumeActor::HandleDeallocateDiskResponse( @@ -397,14 +406,28 @@ void TDestroyVolumeActor::HandleStatVolumeResponse( } } -void TDestroyVolumeActor::HandleStopPartitions( - const TEvVolume::TEvStopPartitionBeforeVolumeDestructionResponse::TPtr& ev, +void TDestroyVolumeActor::HandleGracefulShutdownResponse( + const TEvVolume::TEvGracefulShutdownResponse::TPtr& ev, const TActorContext& ctx) { Y_UNUSED(ev); DestroyVolume(ctx); } +void TDestroyVolumeActor::HandleTimeout( + const TEvents::TEvWakeup::TPtr& ev, + const TActorContext& ctx) +{ + Y_UNUSED(ev); + + LOG_WARN( + ctx, + TBlockStoreComponents::SERVICE, + "Timeout destroy volume request"); + + ReplyAndDie(ctx, MakeError(E_TIMEOUT, "Timeout")); +} + void TDestroyVolumeActor::ReplyAndDie( const TActorContext& ctx, NProto::TError error) @@ -435,8 +458,10 @@ STFUNC(TDestroyVolumeActor::StateWork) HandleStatVolumeResponse); HFunc( - TEvVolume::TEvStopPartitionBeforeVolumeDestructionResponse, - HandleStopPartitions); + TEvVolume::TEvGracefulShutdownResponse, + HandleGracefulShutdownResponse); + + HFunc(TEvents::TEvWakeup, HandleTimeout); default: HandleUnexpectedEvent(ev, TBlockStoreComponents::SERVICE); @@ -475,7 +500,8 @@ void TServiceActor::HandleDestroyVolume( diskId, destroyIfBroken, sync, - fillGeneration); + fillGeneration, + Config->GetDestroyVolumeTimeout()); } } // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/volume/testlib/test_env.cpp b/cloud/blockstore/libs/storage/volume/testlib/test_env.cpp index 1bbb1cac70f..98be0f8bc88 100644 --- a/cloud/blockstore/libs/storage/volume/testlib/test_env.cpp +++ b/cloud/blockstore/libs/storage/volume/testlib/test_env.cpp @@ -551,10 +551,10 @@ TVolumeClient::CreateReadMetaHistoryRequest() return std::make_unique(); } -std::unique_ptr -TVolumeClient::CreateStopPartitionBeforeVolumeDestructionRequest() { - return std::make_unique< - TEvVolume::TEvStopPartitionBeforeVolumeDestructionRequest>(); +std::unique_ptr +TVolumeClient::CreateGracefulShutdownRequest() +{ + return std::make_unique(); } void TVolumeClient::SendRemoteHttpInfo( diff --git a/cloud/blockstore/libs/storage/volume/testlib/test_env.h b/cloud/blockstore/libs/storage/volume/testlib/test_env.h index 91406a0a52b..f3bef0a49b0 100644 --- a/cloud/blockstore/libs/storage/volume/testlib/test_env.h +++ b/cloud/blockstore/libs/storage/volume/testlib/test_env.h @@ -460,8 +460,8 @@ class TVolumeClient std::unique_ptr CreateReadMetaHistoryRequest(); - std::unique_ptr - CreateStopPartitionBeforeVolumeDestructionRequest(); + std::unique_ptr + CreateGracefulShutdownRequest(); void SendRemoteHttpInfo( const TString& params, diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp index 73ce240d4c8..b84d9675ebe 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp @@ -346,24 +346,22 @@ void TVolumeActor::StartPartitionsForGc(const TActorContext& ctx) PartitionsStartedReason = EPartitionsStartedReason::STARTED_FOR_GC; } -void TVolumeActor::HandleStopPartitionBeforeVolumeDestruction( - const TEvVolume::TEvStopPartitionBeforeVolumeDestructionRequest::TPtr& ev, +void TVolumeActor::HandleGracefulShutdown( + const TEvVolume::TEvGracefulShutdownRequest::TPtr& ev, const TActorContext& ctx) { if (!State->GetDiskRegistryBasedPartitionActor()) { LOG_ERROR( ctx, TBlockStoreComponents::VOLUME, - "[%lu] StopPartitionBeforeVolumeDestruction request was send to " - "not " - "DR based disk", + "[%lu] GracefulShutdown request was send to " + "not DR based disk", TabletID()); NCloud::Reply( ctx, *ev, - std::make_unique< - TEvVolume::TEvStopPartitionBeforeVolumeDestructionResponse>( + std::make_unique( MakeError(E_NOT_IMPLEMENTED, "request not supported"))); return; } @@ -383,9 +381,7 @@ void TVolumeActor::HandleStopPartitionBeforeVolumeDestruction( NCloud::Reply( ctx, *reqInfo, - std::make_unique< - TEvVolume:: - TEvStopPartitionBeforeVolumeDestructionResponse>()); + std::make_unique()); }); TerminateTransactions(ctx); diff --git a/cloud/blockstore/libs/storage/volume/volume_ut.cpp b/cloud/blockstore/libs/storage/volume/volume_ut.cpp index b295dd5f611..8026751f72e 100644 --- a/cloud/blockstore/libs/storage/volume/volume_ut.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_ut.cpp @@ -8175,7 +8175,7 @@ Y_UNIT_TEST_SUITE(TVolumeTest) 0, false, 1, - NCloud::NProto::EStorageMediaKind::STORAGE_MEDIA_SSD_NONREPLICATED, + NCloud::NProto::STORAGE_MEDIA_SSD_NONREPLICATED, 1024, "vol0", "cloud", @@ -8190,10 +8190,10 @@ Y_UNIT_TEST_SUITE(TVolumeTest) false); - volume.StopPartitionBeforeVolumeDestruction(); + volume.GracefulShutdown(); UNIT_ASSERT(partitionsStopped); - // Check that volume after TEvStopPartitionBeforeVolumeDestructionRequest + // Check that volume after TEvGracefulShutdownRequest // in zombie state and rejects requsts. volume.SendGetVolumeInfoRequest(); auto response = volume.RecvGetVolumeInfoResponse(); From 77de4101f7126436d044d8c102335c8199599b1f Mon Sep 17 00:00:00 2001 From: Stepanyuk Vladislav Date: Wed, 22 Jan 2025 07:42:49 +0000 Subject: [PATCH 09/15] issue-2542: add error handling --- .../storage/service/service_actor_destroy.cpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp b/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp index 62ef4b7bf28..eaee607d367 100644 --- a/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp +++ b/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp @@ -410,7 +410,20 @@ void TDestroyVolumeActor::HandleGracefulShutdownResponse( const TEvVolume::TEvGracefulShutdownResponse::TPtr& ev, const TActorContext& ctx) { - Y_UNUSED(ev); + const auto* msg = ev->Get(); + + if (auto error = msg->GetError(); HasError(error)) { + LOG_ERROR( + ctx, + TBlockStoreComponents::SERVICE, + "Volume %s: unable to gracefully stop volume: %s", + DiskId.Quote().data(), + FormatError(error).data()); + + ReplyAndDie(ctx, std::move(error)); + return; + } + DestroyVolume(ctx); } @@ -420,7 +433,7 @@ void TDestroyVolumeActor::HandleTimeout( { Y_UNUSED(ev); - LOG_WARN( + LOG_ERROR( ctx, TBlockStoreComponents::SERVICE, "Timeout destroy volume request"); From 0dd10e2eb3b31a7f37a60ec65c3362e3e8af7043 Mon Sep 17 00:00:00 2001 From: Stepanyuk Vladislav Date: Wed, 22 Jan 2025 07:47:02 +0000 Subject: [PATCH 10/15] issue-2542: fmt --- .../libs/storage/service/service_actor_destroy.cpp | 3 +-- cloud/blockstore/libs/storage/volume/volume_ut.cpp | 7 ++----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp b/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp index eaee607d367..e96759ec4c5 100644 --- a/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp +++ b/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp @@ -197,8 +197,7 @@ void TDestroyVolumeActor::DeallocateDisk(const TActorContext& ctx) void TDestroyVolumeActor::GracefulShutdown(const TActorContext& ctx) { - auto request = std::make_unique< - TEvVolume::TEvGracefulShutdownRequest>(); + auto request = std::make_unique(); request->Record.SetDiskId(DiskId); NCloud::Send(ctx, MakeVolumeProxyServiceId(), std::move(request)); } diff --git a/cloud/blockstore/libs/storage/volume/volume_ut.cpp b/cloud/blockstore/libs/storage/volume/volume_ut.cpp index 8026751f72e..74db76b2d20 100644 --- a/cloud/blockstore/libs/storage/volume/volume_ut.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_ut.cpp @@ -8148,7 +8148,7 @@ Y_UNIT_TEST_SUITE(TVolumeTest) UNIT_ASSERT(partitionsStopped); } - Y_UNIT_TEST(ShouldStopPartitionsOnMessage) + Y_UNIT_TEST(ShouldGracefulyShutdownVolume) { auto runtime = PrepareTestActorRuntime(); TVolumeClient volume(*runtime); @@ -8189,7 +8189,6 @@ Y_UNIT_TEST_SUITE(TVolumeTest) NProto::VOLUME_MOUNT_LOCAL, false); - volume.GracefulShutdown(); UNIT_ASSERT(partitionsStopped); @@ -8197,9 +8196,7 @@ Y_UNIT_TEST_SUITE(TVolumeTest) // in zombie state and rejects requsts. volume.SendGetVolumeInfoRequest(); auto response = volume.RecvGetVolumeInfoResponse(); - UNIT_ASSERT_VALUES_EQUAL( - response->GetStatus(), - E_REJECTED); + UNIT_ASSERT_VALUES_EQUAL(response->GetStatus(), E_REJECTED); } Y_UNIT_TEST(ShouldReturnClientsAndHostnameInStatVolumeResponse) From 18be0f0b5e83c804831110a1c0ec7f9e8ab70b05 Mon Sep 17 00:00:00 2001 From: Stepanyuk Vladislav Date: Wed, 22 Jan 2025 09:56:26 +0000 Subject: [PATCH 11/15] issue-2542: fix build issue --- cloud/blockstore/libs/storage/core/config.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cloud/blockstore/libs/storage/core/config.cpp b/cloud/blockstore/libs/storage/core/config.cpp index 8a56a2947e2..c1111e1a1bb 100644 --- a/cloud/blockstore/libs/storage/core/config.cpp +++ b/cloud/blockstore/libs/storage/core/config.cpp @@ -529,6 +529,8 @@ TDuration MSeconds(ui32 value) xxx(EnableToChangeStatesFromDiskRegistryMonpage, bool, false )\ xxx(EnableToChangeErrorStatesFromDiskRegistryMonpage, bool, false )\ xxx(CalculateSplittedUsedQuotaMetric, bool, false )\ + \ + xxx(DestroyVolumeTimeout, TDuration, Seconds(30) )\ // BLOCKSTORE_STORAGE_CONFIG_RW #define BLOCKSTORE_STORAGE_CONFIG(xxx) \ From de1cd5c3267daa4da53f95177230dd11a26a940f Mon Sep 17 00:00:00 2001 From: Stepanyuk Vladislav Date: Thu, 23 Jan 2025 05:26:23 +0000 Subject: [PATCH 12/15] issue-2542: fix test issue --- .../libs/storage/service/service_actor_destroy.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp b/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp index e96759ec4c5..171f9aeae05 100644 --- a/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp +++ b/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp @@ -292,9 +292,16 @@ void TDestroyVolumeActor::HandleMarkDiskForCleanupResponse( // disk is broken and will be removed by DR at some point if (error.GetCode() == E_NOT_FOUND) { - LOG_INFO(ctx, TBlockStoreComponents::SERVICE, - "volume %s not found in registry", DiskId.Quote().data()); - } else if (HasError(error)) { + LOG_INFO( + ctx, + TBlockStoreComponents::SERVICE, + "volume %s not found in registry", + DiskId.Quote().data()); + DestroyVolume(ctx); + return; + } + + if (HasError(error)) { LOG_ERROR(ctx, TBlockStoreComponents::SERVICE, "Volume %s: unable to notify DR about disk destruction: %s", DiskId.Quote().data(), From f8ea413858995e6d069e3f5b27892e4b850cec63 Mon Sep 17 00:00:00 2001 From: Stepanyuk Vladislav Date: Tue, 28 Jan 2025 15:10:04 +0700 Subject: [PATCH 13/15] issue-2542: correct issues --- .../libs/storage/service/service_actor_destroy.cpp | 6 +++++- .../libs/storage/volume/volume_actor_startstop.cpp | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp b/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp index 171f9aeae05..f35d0ab8310 100644 --- a/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp +++ b/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp @@ -442,7 +442,11 @@ void TDestroyVolumeActor::HandleTimeout( LOG_ERROR( ctx, TBlockStoreComponents::SERVICE, - "Timeout destroy volume request"); + "Timeout destroy volume request, diskId = %s, destroyIfBroken = %d, " + "sync = %d", + DiskId, + DestroyIfBroken, + Sync); ReplyAndDie(ctx, MakeError(E_TIMEOUT, "Timeout")); } diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp index b84d9675ebe..59b70fb3f60 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp @@ -354,15 +354,15 @@ void TVolumeActor::HandleGracefulShutdown( LOG_ERROR( ctx, TBlockStoreComponents::VOLUME, - "[%lu] GracefulShutdown request was send to " - "not DR based disk", + "[%lu] GracefulShutdown request was sent to " + "non-DR based disk", TabletID()); NCloud::Reply( ctx, *ev, std::make_unique( - MakeError(E_NOT_IMPLEMENTED, "request not supported"))); + MakeError(E_NOT_IMPLEMENTED, "request is not supported"))); return; } From 6aac9cb6ab601246d87f07997c27bc59bc16be02 Mon Sep 17 00:00:00 2001 From: Stepanyuk Vladislav Date: Tue, 28 Jan 2025 15:17:09 +0700 Subject: [PATCH 14/15] issue-2542: correct comment --- cloud/blockstore/libs/storage/protos/volume.proto | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cloud/blockstore/libs/storage/protos/volume.proto b/cloud/blockstore/libs/storage/protos/volume.proto index c46ab4efae4..5b98af3f8c1 100644 --- a/cloud/blockstore/libs/storage/protos/volume.proto +++ b/cloud/blockstore/libs/storage/protos/volume.proto @@ -661,8 +661,7 @@ message TGracefulShutdownRequest // Optional request headers. THeaders Headers = 1; - // Label of volume to query its storage config. Or empty to - // query server storage config. + // Label of volume to shutdown. string DiskId = 2; } From 6ddb848a95ac1f186cd63c3adf802c5e1de300f9 Mon Sep 17 00:00:00 2001 From: Stepanyuk Vladislav Date: Wed, 29 Jan 2025 18:33:59 +0700 Subject: [PATCH 15/15] issue-2542: correct build issue --- cloud/blockstore/libs/storage/service/service_actor_destroy.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp b/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp index f35d0ab8310..1244a527c03 100644 --- a/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp +++ b/cloud/blockstore/libs/storage/service/service_actor_destroy.cpp @@ -444,7 +444,7 @@ void TDestroyVolumeActor::HandleTimeout( TBlockStoreComponents::SERVICE, "Timeout destroy volume request, diskId = %s, destroyIfBroken = %d, " "sync = %d", - DiskId, + DiskId.c_str(), DestroyIfBroken, Sync);