forked from stakezone/nodemonitorgaiad
-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathnodemonitor.sh
executable file
·210 lines (188 loc) · 8.97 KB
/
nodemonitor.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
#!/bin/bash
#set -x # for debugging
### packages required: jq, bc
### if suppressing error messages is preferred, run as './nodemonitor.sh 2> /dev/null'
### CONFIG ##################################################################################################
CONFIG="$HOME/.NODEDIR/config/config.toml" # config.toml file for node, eg. $HOME/.gaia/config/config.toml
### optional: #
NPRECOMMITS="20" # check last n precommits, can be 0 for no checking
VALIDATORADDRESS="" # if left empty default is from status call (validator)
CHECKPERSISTENTPEERS="1" # if 1 the number of disconnected persistent peers is checked (when persistent peers are configured in config.toml)
VALIDATORMETRICS="on" # metrics for validator node
LOGNAME="nodemonitor-root.log" # a custom log file name can be chosen, if left empty default is nodecheck-<username>.log
LOGPATH="/home" # the directory where the log file is stored, for customization insert path like: '/my/path' or '$(pwd)'
LOGSIZE=200 # the max number of lines after that the log will be trimmed to reduce its size
LOGROTATION="1" # options for log rotation: (1) rotate to $LOGNAME.1 every $LOGSIZE lines; (2) append to $LOGNAME.1 every $LOGSIZE lines; (3) truncate $logfile to $LOGSIZE every iteration
SLEEP1="30s" # polls every SLEEP1 sec
### internal: #
colorI='\033[0;32m' # black 30, red 31, green 32, yellow 33, blue 34, magenta 35, cyan 36, white 37
colorD='\033[0;90m' # for light color 9 instead of 3
colorE='\033[0;31m' #
colorW='\033[0;33m' #
noColor='\033[0m' # no color
### END CONFIG ##################################################################################################
if [ -z $CONFIG ]; then
echo "please configure config.toml in script"
exit 1
fi
url=$(sed '/^\[rpc\]/,/^\[/!d;//d' $CONFIG | grep "^laddr\b" | awk -v FS='("tcp://|")' '{print $2}')
chainid=$(jq -r '.result.node_info.network' <<<$(curl -s "$url"/status))
if [ -z $url ]; then
echo "please configure config.toml in script correctly"
exit 1
fi
url="http://${url}"
if [ -z $LOGNAME ]; then LOGNAME="nodemonitor-${USER}.log"; fi
logfile="${LOGPATH}/${LOGNAME}"
touch $logfile
echo "log file: ${logfile}"
echo "rpc url: ${url}"
echo "chain id: ${chainid}"
if [ -z $VALIDATORADDRESS ]; then VALIDATORADDRESS=$(jq -r '.result.validator_info.address' <<<$(curl -s "$url"/status)); fi
if [ -z $VALIDATORADDRESS ]; then
echo "rpc appears to be down, start script again when data can be obtained"
exit 1
fi
echo "validator address: $VALIDATORADDRESS"
if [ "$CHECKPERSISTENTPEERS" -eq 1 ]; then
persistentpeers=$(sed '/^\[p2p\]/,/^\[/!d;//d' $CONFIG | grep "^persistent_peers\b" | awk -v FS='("|")' '{print $2}')
persistentpeerids=$(sed 's/,//g' <<<$(sed 's/@[^ ^,]\+/ /g' <<<$persistentpeers))
totpersistentpeerids=$(wc -w <<<$persistentpeerids)
npersistentpeersmatchcount=0
netinfo=$(curl -s "$url"/net_info)
if [ -z "$netinfo" ]; then
echo "lcd appears to be down, start script again when data can be obtained"
exit 1
fi
for id in $persistentpeerids; do
npersistentpeersmatch=$(grep -c "$id" <<<$netinfo)
if [ $npersistentpeersmatch -eq 0 ]; then
persistentpeersmatch="$id $persistentpeersmatch"
npersistentpeersmatchcount=$(expr $npersistentpeersmatchcount + 1)
fi
done
npersistentpeersoff=$(expr $totpersistentpeerids - $npersistentpeersmatchcount)
echo "$totpersistentpeerids persistent peer(s): $persistentpeerids"
echo "$npersistentpeersmatchcount persistent peer(s) off: $persistentpeersmatch"
fi
if [ $NPRECOMMITS -eq 0 ]; then echo "precommit checks: off"; else echo "precommit checks: on"; fi
if [ $CHECKPERSISTENTPEERS -eq 0 ]; then echo "persistent peer checks: off"; else echo "persistent peer checks: on"; fi
echo ""
status=$(curl -s "$url"/status)
blockheight=$(jq -r '.result.sync_info.latest_block_height' <<<$status)
blockinfo=$(curl -s "$url"/block?height="$blockheight")
if [ $blockheight -gt $NPRECOMMITS ]; then
if [ "$(grep -c 'precommits' <<<$blockinfo)" != "0" ]; then versionstring="precommits"; elif [ "$(grep -c 'signatures' <<<$blockinfo)" != "0" ]; then versionstring="signatures"; else
echo "json parameters of this version not recognised"
exit 1
fi
else
echo "wait for $NPRECOMMITS blocks and start again..."
exit 1
fi
nloglines=$(wc -l <$logfile)
if [ $nloglines -gt $LOGSIZE ]; then sed -i "1,$(expr $nloglines - $LOGSIZE)d" $logfile; fi # the log file is trimmed for logsize
date=$(date --rfc-3339=seconds)
echo "$date status=scriptstarted chainid=$chainid" >>$logfile
while true; do
status=$(curl -s "$url"/status)
result=$(grep -c "result" <<<$status)
if [ "$result" != "0" ]; then
npeers=$(curl -s "$url"/net_info | jq -r '.result.n_peers')
if [ -z $npeers ]; then npeers="na"; fi
blockheight=$(jq -r '.result.sync_info.latest_block_height' <<<$status)
blocktime=$(jq -r '.result.sync_info.latest_block_time' <<<$status)
catchingup=$(jq -r '.result.sync_info.catching_up' <<<$status)
if [ $catchingup == "false" ]; then catchingup="synced"; elif [ $catchingup == "true" ]; then catchingup="catchingup"; fi
if [ "$CHECKPERSISTENTPEERS" -eq 1 ]; then
npersistentpeersmatch=0
netinfo=$(curl -s "$url"/net_info)
for id in $persistentpeerids; do
npersistentpeersmatch=$(expr $npersistentpeersmatch + $(grep -c "$id" <<<$netinfo))
done
npersistentpeersoff=$(expr $totpersistentpeerids - $npersistentpeersmatch)
else
npersistentpeersoff=0
fi
if [ "$VALIDATORMETRICS" == "on" ]; then
#isvalidator=$(grep -c "$VALIDATORADDRESS" <<<$(curl -s "$url"/block?height="$blockheight"))
consdump=$(curl -s "$url"/dump_consensus_state)
validators=$(jq -r '.result.round_state.validators[]' <<<$consdump)
isvalidator=$(grep -c "$VALIDATORADDRESS" <<<$validators)
pcttotcommits=$(jq -r '.result.round_state.last_commit.votes_bit_array' <<<$consdump)
pcttotcommits=$(grep -Po "=\s+\K[^ ^]+" <<<$pcttotcommits)
if [ "$isvalidator" != "0" ]; then
isvalidator="yes"
precommitcount=0
for ((i = $(expr $blockheight - $NPRECOMMITS + 1); i <= $blockheight; i++)); do
validatoraddresses=$(curl -s "$url"/block?height="$i")
validatoraddresses=$(jq ".result.block.last_commit.${versionstring}[].validator_address" <<<$validatoraddresses)
validatorprecommit=$(grep -c "$VALIDATORADDRESS" <<<$validatoraddresses)
precommitcount=$(expr $precommitcount + $validatorprecommit)
done
if [ $NPRECOMMITS -eq 0 ]; then pctprecommits="1.0"; else pctprecommits=$(echo "scale=2 ; $precommitcount / $NPRECOMMITS" | bc); fi
validatorinfo="isvalidator=$isvalidator pctprecommits=$pctprecommits pcttotcommits=$pcttotcommits"
else
isvalidator="no"
validatorinfo="isvalidator=$isvalidator"
fi
fi
status="$catchingup"
now=$(date --rfc-3339=seconds)
blockheightfromnow=$(expr $(date +%s -d "$now") - $(date +%s -d $blocktime))
variables="status=$status blockheight=$blockheight tfromnow=$blockheightfromnow npeers=$npeers npersistentpeersoff=$npersistentpeersoff $validatorinfo"
else
status="error"
now=$(date --rfc-3339=seconds)
variables="status=$status"
fi
logentry="[$now] $variables"
echo "$logentry" >>$logfile
nloglines=$(wc -l <$logfile)
if [ $nloglines -gt $LOGSIZE ]; then
case $LOGROTATION in
1)
mv $logfile "${logfile}.1"
touch $logfile
;;
2)
echo "$(cat $logfile)" >>${logfile}.1
>$logfile
;;
3)
sed -i '1d' $logfile
if [ -f ${logfile}.1 ]; then rm ${logfile}.1; fi # no log rotation with option (3)
;;
*) ;;
esac
fi
case $status in
synced)
color=$colorI
;;
error)
color=$colorE
;;
catchingup)
color=$colorW
;;
*)
color=$noColor
;;
esac
pctprecommits=$(awk '{printf "%f", $0}' <<<"$pctprecommits")
if [[ "$isvalidator" == "yes" ]] && [[ "$pctprecommits" < "1.0" ]]; then color=$colorW; fi
if [[ "$isvalidator" == "no" ]] && [[ "$VALIDATORMETRICS" == "on" ]]; then color=$colorW; fi
logentry="$(sed 's/[^ ]*[\=]/'\\${color}'&'\\${noColor}'/g' <<<$logentry)"
echo -e $logentry
echo -e "${colorD}sleep ${SLEEP1}${noColor}"
variables_=""
for var in $variables; do
var_=$(grep -Po '^[0-9a-zA-Z_-]*' <<<$var)
var_="$var_=\"\""
variables_="$var_; $variables_"
done
#echo $variables_
eval $variables_
sleep $SLEEP1
done