From 36a7eb123eda952322d13e055645a5af6d9187d3 Mon Sep 17 00:00:00 2001 From: Abhishek Pal Date: Thu, 16 Jan 2025 19:44:07 +0530 Subject: [PATCH] Address review comments --- .../dist/src/main/smoketest/recon/recon-taskstatus.robot | 6 ++---- .../recon/spi/impl/OzoneManagerServiceProviderImpl.java | 9 +++------ 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/recon/recon-taskstatus.robot b/hadoop-ozone/dist/src/main/smoketest/recon/recon-taskstatus.robot index 0b173204349..24f0b7f6ca5 100644 --- a/hadoop-ozone/dist/src/main/smoketest/recon/recon-taskstatus.robot +++ b/hadoop-ozone/dist/src/main/smoketest/recon/recon-taskstatus.robot @@ -56,7 +56,7 @@ Prepopulate Data and Trigger OM DB Sync [Documentation] Use Freon to prepopulate the OM DB with data and trigger OM DB sync. Kinit as ozone admin - Freon DFSG n=1000 path=${KEYPATH} + Freon DFSG n=100 path=${KEYPATH} Sync OM Data @@ -103,9 +103,7 @@ Validate All Tasks Updated After Sync END Validate Sequence number is updated after sync - Log To Console Generating Freon Data - Kinit as ozone admin - Freon DFSG n=100 path=${VOLUME}/${BUCKET}/seqValidationKey + Log To Console Syncing OM DB to ensure all updates are applied Sync OM Data ${tasks} = Fetch Task Status diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java index d209ab5193c..b0a5e3f7c47 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java @@ -478,10 +478,7 @@ void getAndApplyDeltaUpdatesFromOM( inLoopStartSequenceNumber = inLoopLatestSequenceNumber; loopCount++; } - // We might reach loop count limit and so not all updates from RocksDB might be applied. - // Say we have: 1000 updates, each loop we get only 100 updates, loop limit is 5. - // So this will effectively only apply 500 events, even if we are having total 1000 updates/events; - // All subsequent tasks will effectively process only 500 events, till next sync takes place. + omdbUpdatesHandler.setLatestSequenceNumber(getCurrentOMDBSequenceNumber()); LOG.info("Delta updates received from OM : {} loops, {} records", loopCount, getCurrentOMDBSequenceNumber() - fromSequenceNumber @@ -544,8 +541,8 @@ boolean innerGetAndApplyDeltaUpdatesFromOM(long fromSequenceNumber, *
  • Initially it will fetch a snapshot of OM DB.
    • *
  • If we already have data synced it will try to fetch delta updates.
    • *
  • If the sync is completed successfully it will trigger other OM tasks to process events
    • - *
  • If there is any exception while trying to fetch delta updates, it will fall back to snapshot update
    • - *
  • If there is any exception in snapshot sync it will do nothing.
    • + *
  • If there is any exception while trying to fetch delta updates, it will fall back to full snapshot update
    • + *
  • If there is any exception in full snapshot update it will do nothing, and return true.
    • *
  • In case of an interrupt signal (irrespective of delta or snapshot sync), * it will catch and mark the task as interrupted, and return false i.e. sync failed status.
    • *