Skip to content

Commit

Permalink
ticket VS-1409, VS-1410, VS-1411
Browse files Browse the repository at this point in the history 
checking results of detected cpu load
adding dtls information to log
 - L - number of connections
 - P - number of packets
adding parameters for abort or exit when heap is full (if full is detected 10 times in a row)
 - abort_if_heap_full (yes/NO)
 - exit_if_heap_full (yes/NO)
and for abort/exit when heap is full and t2 thread load is low
 - abort_if_heap_full_and_t2cpu_is_low (YES/no)
 - exit_if_heap_full_and_t2cpu_is_low (yes/NO)
  • Loading branch information
rbucek committed Jan 23, 2023
1 parent 6fa44d1 commit 13f875f
Show file tree
Hide file tree
Showing 6 changed files with 88 additions and 8 deletions.
1 change: 1 addition & 0 deletions common.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ struct sVerbose {
int timezones;
int tcpreplay;
int abort_if_heap_full;
int exit_if_heap_full;
int heap_use_time;
int dtmf;
int dtls;
Expand Down
52 changes: 47 additions & 5 deletions pcap_queue.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,7 @@ static double heap_pb_used_perc = 0;
static double heap_pb_trash_perc = 0;
static double heap_pb_pool_perc = 0;
static unsigned heapFullCounter = 0;
static unsigned heapFullIfT2cpuIsLowCounter = 0;

extern MySqlStore *sqlStore;
extern MySqlStore *loadFromQFiles;
Expand Down Expand Up @@ -2288,6 +2289,7 @@ void PcapQueue::pcapStat(int statPeriod, bool statCalls) {
lapTimeDescr.push_back("t1");
}
double t2cpu = this->getCpuUsagePerc(writeThread, true);
double sum_t2cpu = 0;
if(t2cpu >= 0) {
if(isMirrorSender()) {
outStrStat << "t2CPU[" << t2cpu;
Expand Down Expand Up @@ -2353,7 +2355,7 @@ void PcapQueue::pcapStat(int statPeriod, bool statCalls) {
double call_t2cpu_preprocess_packet_out_thread = -2;
double last_t2cpu_preprocess_packet_out_thread_rtp = -2;
int count_t2cpu = 1;
double sum_t2cpu = t2cpu;
sum_t2cpu = t2cpu;
last_t2cpu_preprocess_packet_out_thread_check_next_level = t2cpu;
last_t2cpu_preprocess_packet_out_thread_rtp = t2cpu;
for(int i = 0; i < PreProcessPacket::ppt_end_base; i++) {
Expand All @@ -2363,7 +2365,7 @@ void PcapQueue::pcapStat(int statPeriod, bool statCalls) {
double t2cpu_preprocess_packet_out_thread = preProcessPacket[i]->getCpuUsagePerc(true, j, j == 0 ? &percFullQring : NULL);
if(t2cpu_preprocess_packet_out_thread >= 0) {
outStrStat << "/"
<< preProcessPacket[i]->getShortcatTypeThread()
<< preProcessPacket[i]->getShortcatTypeThread() << ":"
<< setprecision(1) << t2cpu_preprocess_packet_out_thread;
if(sverb.qring_stat) {
double qringFillingPerc = preProcessPacket[i]->getQringFillingPerc();
Expand Down Expand Up @@ -2619,6 +2621,18 @@ void PcapQueue::pcapStat(int statPeriod, bool statCalls) {
lapTimeDescr.push_back("tssl");
}
}
extern link_packets_queue dtls_queue;
u_int32_t dtls_queue_links = dtls_queue.countLinks();
u_int32_t dtls_queue_packets = dtls_queue.countPackets();
if(dtls_queue_links > 0 || dtls_queue_packets > 0) {
outStrStat << "dtls[l:" << dtls_queue_links
<< "/p:" << dtls_queue_packets
<< "] ";
if(sverb.log_profiler) {
lapTime.push_back(getTimeMS_rdtsc());
lapTimeDescr.push_back("dtls");
}
}
if(tcpReassemblySipExt) {
string cpuUsagePerc = tcpReassemblySipExt->getCpuUsagePerc();
if(!cpuUsagePerc.empty()) {
Expand Down Expand Up @@ -3003,17 +3017,45 @@ void PcapQueue::pcapStat(int statPeriod, bool statCalls) {
rrd_update();
}

if(sverb.abort_if_heap_full) {
extern bool opt_abort_if_heap_full;
extern bool opt_exit_if_heap_full;
if(opt_abort_if_heap_full || opt_exit_if_heap_full ||
sverb.abort_if_heap_full || sverb.exit_if_heap_full) {
if(packetbuffer_memory_is_full || heap_pb_perc > 98) {
if(++heapFullCounter > 10) {
syslog(LOG_ERR, "HEAP FULL - ABORT!");
exit(2);
syslog(LOG_ERR, "HEAP FULL - %s!", opt_exit_if_heap_full || sverb.exit_if_heap_full ? "EXIT" : "ABORT");
if(opt_exit_if_heap_full || sverb.exit_if_heap_full) {
extern WDT *wdt;
wdt = NULL;
exit(2);
} else {
abort();
}
}
} else {
heapFullCounter = 0;
}
}

extern bool opt_abort_if_heap_full_and_t2cpu_is_low;
extern bool opt_exit_if_heap_full_and_t2cpu_is_low;
if(opt_abort_if_heap_full_and_t2cpu_is_low || opt_exit_if_heap_full_and_t2cpu_is_low) {
if((packetbuffer_memory_is_full || heap_pb_perc > 98) && sum_t2cpu < 50) {
if(++heapFullIfT2cpuIsLowCounter > 10) {
syslog(LOG_ERR, "HEAP FULL (and t2cpu is low) - %s!", opt_exit_if_heap_full_and_t2cpu_is_low ? "EXIT" : "ABORT");
if(opt_exit_if_heap_full_and_t2cpu_is_low) {
extern WDT *wdt;
wdt = NULL;
exit(2);
} else {
abort();
}
}
} else {
heapFullIfT2cpuIsLowCounter = 0;
}
}

extern int opt_abort_if_rss_gt_gb;
if(opt_abort_if_rss_gt_gb > 0 && (int)(rss/1024/1024/1024) > opt_abort_if_rss_gt_gb) {
syslog(LOG_ERR, "RSS %i > %i - ABORT!",
Expand Down
6 changes: 4 additions & 2 deletions pstat.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,10 @@ void pstat_calc_cpu_usage_pct(const pstat_data* cur_usage,
jiffy = sysconf(_SC_CLK_TCK);
}
double jiffytime = 1.0 / jiffy * 100;
*ucpu_usage = 100 * ((cur_usage->utime_ticks - last_usage->utime_ticks) / (double) total_time_diff) * jiffytime * cpuCore;
*scpu_usage = 100 * ((cur_usage->stime_ticks - last_usage->stime_ticks) / (double) total_time_diff) * jiffytime * cpuCore;
double _ucpu_usage = 100 * ((cur_usage->utime_ticks - last_usage->utime_ticks) / (double) total_time_diff) * jiffytime * cpuCore;
double _scpu_usage = 100 * ((cur_usage->stime_ticks - last_usage->stime_ticks) / (double) total_time_diff) * jiffytime * cpuCore;
*ucpu_usage = _ucpu_usage < 200 ? _ucpu_usage : 0;
*scpu_usage = _scpu_usage < 200 ? _scpu_usage : 0;
}

void pstat_calc_cpu_usage(const pstat_data* cur_usage,
Expand Down
3 changes: 3 additions & 0 deletions sniff.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8805,6 +8805,7 @@ void link_packets_queue::_cleanup(u_int64_t time_ms) {
packet_s_process_0 *packetS = (packet_s_process_0*)*iter;
packetS->blockstore_addflag(125 /*pb lock flag*/);
PACKET_S_PROCESS_DESTROY(&packetS);
__SYNC_DEC(packets_counter);
}
delete link;
links.erase(iter_link++);
Expand All @@ -8823,6 +8824,7 @@ void link_packets_queue::_cleanup(u_int64_t time_ms) {
packetS->blockstore_addflag(126 /*pb lock flag*/);
PACKET_S_PROCESS_DESTROY(&packetS);
link->queue.pop_front();
__SYNC_DEC(packets_counter);
destroy_packet = true;
}
if(destroy_packet) {
Expand All @@ -8846,6 +8848,7 @@ void link_packets_queue::destroyAll() {
for(list<packet_s*>::iterator iter = link->queue.begin(); iter != link->queue.end(); iter++) {
packet_s_process_0 *packetS = (packet_s_process_0*)*iter;
PACKET_S_PROCESS_DESTROY(&packetS);
__SYNC_DEC(packets_counter);
}
delete link;
}
Expand Down
13 changes: 12 additions & 1 deletion sniff.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -935,6 +935,7 @@ class link_packets_queue {
cleanup_interval_ms = 5000;
expiration_link_ms = 10000;
expiration_link_count = 20;
packets_counter = 0;
}
~link_packets_queue() {
destroyAll();
Expand All @@ -961,6 +962,7 @@ class link_packets_queue {
link->first_time_ms = time_ms;
}
link->queue.push_back(packetS);
__SYNC_INC(packets_counter);
link->last_time_ms = time_ms;
if(!link->first_time_ms) {
link->first_time_ms = time_ms;
Expand All @@ -985,6 +987,8 @@ class link_packets_queue {
packetS->insert_packet((packet_s_process_0*)(*iter));
if(keep) {
((packet_s_process_0*)(*iter))->set_reuse_counter();
} else {
__SYNC_DEC(packets_counter);
}
((packet_s_process_0*)(*iter))->blockstore_addflag(122 /*pb lock flag*/);
#if DEBUG_DTLS_QUEUE
Expand All @@ -998,13 +1002,19 @@ class link_packets_queue {
}
}
inline bool existsContent() {
return(links.size());
return(links.size() > 0);
}
inline bool existsLink(packet_s *packetS) {
s_link_id id;
createId(&id, packetS);
return(links.find(id) != links.end());
}
inline u_int32_t countLinks() {
return(links.size());
}
inline u_int64_t countPackets() {
return(packets_counter);
}
void cleanup();
void _cleanup(u_int64_t time_ms);
void destroyAll();
Expand Down Expand Up @@ -1033,6 +1043,7 @@ class link_packets_queue {
unsigned cleanup_interval_ms;
unsigned expiration_link_ms;
unsigned expiration_link_count;
volatile u_int64_t packets_counter;
};


Expand Down
21 changes: 21 additions & 0 deletions voipmonitor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1317,6 +1317,10 @@ int opt_separate_processing = 0;

int opt_abort_if_rss_gt_gb = 0;
int opt_abort_if_alloc_gt_gb = 0;
bool opt_abort_if_heap_full = false;
bool opt_exit_if_heap_full = false;
bool opt_abort_if_heap_full_and_t2cpu_is_low = true;
bool opt_exit_if_heap_full_and_t2cpu_is_low = false;
int opt_next_server_connections = 0;

string opt_coredump_filter = "0x7F";
Expand Down Expand Up @@ -8079,6 +8083,10 @@ void cConfig::addConfigItems() {
"disable:" + intToString(numa_balancing_set_disable)).c_str()));
addConfigItem(new FILE_LINE(0) cConfigItem_integer("abort_if_rss_gt_gb", &opt_abort_if_rss_gt_gb));
addConfigItem(new FILE_LINE(0) cConfigItem_integer("abort_if_alloc_gt_gb", &opt_abort_if_alloc_gt_gb));
addConfigItem(new FILE_LINE(0) cConfigItem_yesno("abort_if_heap_full", &opt_abort_if_heap_full));
addConfigItem(new FILE_LINE(0) cConfigItem_yesno("exit_if_heap_full", &opt_exit_if_heap_full));
addConfigItem(new FILE_LINE(0) cConfigItem_yesno("abort_if_heap_full_and_t2cpu_is_low", &opt_abort_if_heap_full_and_t2cpu_is_low));
addConfigItem(new FILE_LINE(0) cConfigItem_yesno("exit_if_heap_full_and_t2cpu_is_low", &opt_exit_if_heap_full_and_t2cpu_is_low));
addConfigItem(new FILE_LINE(0) cConfigItem_integer("next_server_connections", &opt_next_server_connections));
addConfigItem(new FILE_LINE(0) cConfigItem_string("coredump_filter", &opt_coredump_filter));
addConfigItem(new FILE_LINE(0) cConfigItem_yesno("all_configuration_options_in_gui", &opt_all_configuration_options_in_gui));
Expand Down Expand Up @@ -8619,6 +8627,7 @@ void parse_verb_param(string verbParam) {
else if(verbParam == "timezones") sverb.timezones = 1;
else if(verbParam == "tcpreplay") sverb.tcpreplay = 1;
else if(verbParam == "abort_if_heap_full") sverb.abort_if_heap_full = 1;
else if(verbParam == "exit_if_heap_full") sverb.exit_if_heap_full = 1;
else if(verbParam == "heap_use_time") sverb.heap_use_time = 1;
else if(verbParam == "dtmf") sverb.dtmf = 1;
else if(verbParam == "dtls") sverb.dtls = 1;
Expand Down Expand Up @@ -13115,6 +13124,18 @@ int eval_config(string inistr) {
if((value = ini.GetValue("general", "abort_if_alloc_gt_gb", NULL))) {
opt_abort_if_alloc_gt_gb = atoi(value);
}
if((value = ini.GetValue("general", "abort_if_heap_full", NULL))) {
opt_abort_if_heap_full = yesno(value);
}
if((value = ini.GetValue("general", "exit_if_heap_full", NULL))) {
opt_exit_if_heap_full = yesno(value);
}
if((value = ini.GetValue("general", "abort_if_heap_full_and_t2cpu_is_low", NULL))) {
opt_abort_if_heap_full_and_t2cpu_is_low = yesno(value);
}
if((value = ini.GetValue("general", "exit_if_heap_full_and_t2cpu_is_low", NULL))) {
opt_exit_if_heap_full_and_t2cpu_is_low = yesno(value);
}
if((value = ini.GetValue("general", "coredump_filter", NULL))) {
opt_coredump_filter = value;
}
Expand Down

0 comments on commit 13f875f

Please sign in to comment.