Skip to content

Commit

Permalink
[log](mow) reduce log for data load on mow table (apache#47647)
Browse files Browse the repository at this point in the history
reduce some useless logs according to the frequency stats from online
user logs.
  • Loading branch information
zhannngchen authored Feb 10, 2025
1 parent 68b6a54 commit 1dcf4d1
Show file tree
Hide file tree
Showing 4 changed files with 19 additions and 30 deletions.
8 changes: 0 additions & 8 deletions be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -262,7 +262,6 @@ Status CloudTabletCalcDeleteBitmapTask::_handle_rowset(
return status;
}

int64_t t3 = MonotonicMicros();
rowset->set_version(Version(version, version));
TabletTxnInfo txn_info;
txn_info.rowset = rowset;
Expand All @@ -274,7 +273,6 @@ Status CloudTabletCalcDeleteBitmapTask::_handle_rowset(
.base_compaction_cnt = _ms_base_compaction_cnt,
.cumulative_compaction_cnt = _ms_cumulative_compaction_cnt,
.cumulative_point = _ms_cumulative_point};
int64_t update_delete_bitmap_time_us = 0;
if (txn_info.publish_status && (*(txn_info.publish_status) == PublishStatus::SUCCEED) &&
version == previous_publish_info.publish_version &&
_ms_base_compaction_cnt == previous_publish_info.base_compaction_cnt &&
Expand Down Expand Up @@ -304,7 +302,6 @@ Status CloudTabletCalcDeleteBitmapTask::_handle_rowset(
status = CloudTablet::update_delete_bitmap(tablet, &txn_info, transaction_id,
txn_expiration, tablet_delete_bitmap);
}
update_delete_bitmap_time_us = MonotonicMicros() - t3;
}
if (status != Status::OK()) {
LOG(WARNING) << "failed to calculate delete bitmap. rowset_id=" << rowset->rowset_id()
Expand All @@ -314,11 +311,6 @@ Status CloudTabletCalcDeleteBitmapTask::_handle_rowset(
}

_engine_calc_delete_bitmap_task->add_succ_tablet_id(_tablet_id);
LOG(INFO) << "calculate delete bitmap successfully on tablet"
<< ", table_id=" << tablet->table_id() << ", " << txn_str
<< ", tablet_id=" << tablet->tablet_id() << ", num_rows=" << rowset->num_rows()
<< ", update_delete_bitmap_time_us=" << update_delete_bitmap_time_us
<< ", res=" << status;
if (invisible_rowsets != nullptr) {
invisible_rowsets->push_back(rowset);
// see CloudTablet::save_delete_bitmap
Expand Down
5 changes: 0 additions & 5 deletions be/src/cloud/cloud_txn_delete_bitmap_cache.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -202,11 +202,6 @@ Status CloudTxnDeleteBitmapCache::update_tablet_txn_info(TTransactionId transact
// must call release handle to reduce the reference count,
// otherwise there will be memory leak
release(handle);
LOG_INFO("update txn related delete bitmap")
.tag("txn_id", transaction_id)
.tag("tablt_id", tablet_id)
.tag("delete_bitmap_size", charge)
.tag("publish_status", static_cast<int>(publish_status));
return Status::OK();
}

Expand Down
34 changes: 19 additions & 15 deletions be/src/olap/base_tablet.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -541,10 +541,7 @@ Status BaseTablet::calc_delete_bitmap(const BaseTabletSPtr& tablet, RowsetShared
DeleteBitmapPtr delete_bitmap, int64_t end_version,
CalcDeleteBitmapToken* token, RowsetWriter* rowset_writer,
DeleteBitmapPtr tablet_delete_bitmap) {
auto rowset_id = rowset->rowset_id();
if (specified_rowsets.empty() || segments.empty()) {
LOG(INFO) << "skip to construct delete bitmap tablet: " << tablet->tablet_id()
<< " rowset: " << rowset_id;
return Status::OK();
}

Expand Down Expand Up @@ -808,22 +805,29 @@ Status BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
partial_update_info->partial_update_mode_str(), new_generated_rows,
rowset_writer->num_rows(), rids_be_overwritten.size(), tablet_id());
}
LOG(INFO) << "calc segment delete bitmap for "
<< partial_update_info->partial_update_mode_str() << ", tablet: " << tablet_id()
auto cost_us = watch.get_elapse_time_us();
if (cost_us > 10 * 1000) {
LOG(INFO) << "calc segment delete bitmap for "
<< partial_update_info->partial_update_mode_str()
<< ", tablet: " << tablet_id() << " rowset: " << rowset_id
<< " seg_id: " << seg->id() << " dummy_version: " << end_version + 1
<< " rows: " << seg->num_rows() << " conflict rows: " << conflict_rows
<< " filtered rows: " << rids_be_overwritten.size()
<< " new generated rows: " << new_generated_rows
<< " bimap num: " << delete_bitmap->delete_bitmap.size()
<< " cost: " << cost_us << "(us)";
}
return Status::OK();
}
auto cost_us = watch.get_elapse_time_us();
if (cost_us > 10 * 1000) {
LOG(INFO) << "calc segment delete bitmap, tablet: " << tablet_id()
<< " rowset: " << rowset_id << " seg_id: " << seg->id()
<< " dummy_version: " << end_version + 1 << " rows: " << seg->num_rows()
<< " conflict rows: " << conflict_rows
<< " filtered rows: " << rids_be_overwritten.size()
<< " new generated rows: " << new_generated_rows
<< " bimap num: " << delete_bitmap->delete_bitmap.size()
<< " cost: " << watch.get_elapse_time_us() << "(us)";
return Status::OK();
<< " bitmap num: " << delete_bitmap->delete_bitmap.size() << " cost: " << cost_us
<< "(us)";
}
LOG(INFO) << "calc segment delete bitmap, tablet: " << tablet_id() << " rowset: " << rowset_id
<< " seg_id: " << seg->id() << " dummy_version: " << end_version + 1
<< " rows: " << seg->num_rows() << " conflict rows: " << conflict_rows
<< " bitmap num: " << delete_bitmap->delete_bitmap.size()
<< " cost: " << watch.get_elapse_time_us() << "(us)";
return Status::OK();
}

Expand Down
2 changes: 0 additions & 2 deletions be/src/olap/rowset_builder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -322,8 +322,6 @@ Status BaseRowsetBuilder::wait_calc_delete_bitmap() {
std::lock_guard<std::mutex> l(_lock);
SCOPED_TIMER(_wait_delete_bitmap_timer);
RETURN_IF_ERROR(_calc_delete_bitmap_token->wait());
LOG(INFO) << "Got result of calc delete bitmap task from executor, tablet_id: "
<< _tablet->tablet_id() << ", txn_id: " << _req.txn_id;
return Status::OK();
}

Expand Down

0 comments on commit 1dcf4d1

Please sign in to comment.