forked from anclrii/Storj-Exporter
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathstorj-exporter.py
193 lines (167 loc) · 9.36 KB
/
storj-exporter.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
import os
import time
import requests
import json
from prometheus_client import start_http_server
from prometheus_client.core import GaugeMetricFamily, InfoMetricFamily, REGISTRY
class StorjCollector(object):
def __init__(self):
self.node_data = None
self.storj_host_address = os.environ.get('STORJ_HOST_ADDRESS', '127.0.0.1')
self.storj_api_port = os.environ.get('STORJ_API_PORT', '14002')
self.storj_collectors = os.environ.get('STORJ_COLLECTORS', 'payout sat').split()
self.baseurl = 'http://' + self.storj_host_address + ':' + self.storj_api_port + '/api/'
def call_api(self, path):
response = None
try:
response=requests.get(url = self.baseurl + path).json()
except:
pass
return response
def get_node_data(self):
self.node_data = self.call_api('sno/')
def get_node_payout_data(self):
self.node_data['payout'] = self.call_api('sno/estimated-payout')
def get_sat_data(self, sat):
sat['sat_data'] = self.call_api('sno/satellite/' + sat['id'])
if isinstance(sat['sat_data'], dict):
sat['sat_data'].update(self.sum_sat_daily_keys(sat['sat_data'], 'bandwidthDaily', ['repair','audit','usage'], 'egress'))
sat['sat_data'].update(self.sum_sat_daily_keys(sat['sat_data'], 'bandwidthDaily', ['repair','usage'], 'ingress'))
def sum_sat_daily_keys(self, daily_data_dict, daily_data_key, data_types, daily_data_path):
sum_month_dict = daily_data_dict['sum_' + daily_data_key + '_' + daily_data_path ] = {}
if daily_data_key in daily_data_dict:
for day in daily_data_dict[daily_data_key]:
for data_type in data_types:
if data_type not in sum_month_dict:
sum_month_dict[data_type] = 0.0
if daily_data_path in day and data_type in day[daily_data_path]:
day_value = day[daily_data_path][data_type]
if day_value:
sum_month_dict.update({data_type: (sum_month_dict[data_type] + day_value)})
return daily_data_dict
def dict_to_metric(self, dict, metric_name, documentation, metric_family, keys, labels,label_values=[] ):
if dict:
metric = metric_family(metric_name, documentation, labels=labels)
for key in keys:
value = 0.0
if key in dict:
key_label_values = [key] + label_values
if metric_family == InfoMetricFamily:
value = {key : str(dict[key])}
elif isinstance(dict[key],(int, float)):
value = dict[key]
metric.add_metric(key_label_values, value)
yield metric
def add_node_metrics(self):
self.get_node_data()
if self.node_data:
labels = ['type']
metric_family = GaugeMetricFamily
metric_name = 'storj_node'
data = self.node_data
documentation = 'Storj node info'
keys = ['nodeID','wallet','lastPinged','upToDate','version','allowedVersion','startedAt']
yield from self.dict_to_metric(data, metric_name, documentation, InfoMetricFamily, keys, labels)
metric_name = 'storj_total_diskspace'
data = self.node_data.get('diskSpace', None)
documentation = 'Storj total diskspace metrics'
keys = ['used','available','trash']
yield from self.dict_to_metric(data, metric_name, documentation, metric_family, keys, labels)
data = self.node_data.get('bandwidth', None)
metric_name = 'storj_total_bandwidth'
documentation = 'Storj total bandwidth metrics'
keys = ['used','available']
yield from self.dict_to_metric(data, metric_name, documentation, metric_family, keys, labels)
## to be deprecated
for key in ['nodeID','wallet','lastPinged','upToDate','version','allowedVersion','startedAt']:
if key in self.node_data:
value = str(self.node_data[key])
metric = InfoMetricFamily('storj_' + key, 'Storj ' + key, value={key : value})
yield metric
##############
def add_payout_metrics(self):
if 'payout' in self.storj_collectors:
self.get_node_payout_data()
metric_name = 'storj_payout_currentMonth'
data = self.node_data.get('payout', {}).get('currentMonth', None)
documentation = 'Storj estimated payouts for current month'
keys = ['egressBandwidth', 'egressBandwidthPayout', 'egressRepairAudit', 'egressRepairAuditPayout', 'diskSpace', 'diskSpacePayout', 'heldRate', 'payout', 'held']
labels = ['type']
metric_family = GaugeMetricFamily
yield from self.dict_to_metric(data, metric_name, documentation, metric_family, keys, labels)
def add_sat_metrics(self):
if 'satellites' in self.node_data:
for sat in self.node_data['satellites']:
sat.update({'disqualified': 1}) if sat['disqualified'] else sat.update({'disqualified': 0})
sat.update({'suspended': 1}) if sat['suspended'] else sat.update({'suspended': 0})
metric_name = 'storj_sat_summary'
data = sat
documentation = 'Storj satellite summary metrics'
keys = ['disqualified','suspended']
labels = ['type', 'satellite', 'url']
label_values = [sat['id'], sat['url']]
metric_family = GaugeMetricFamily
yield from self.dict_to_metric(data, metric_name, documentation, metric_family, keys, labels, label_values)
yield from self.add_extended_sat_metrics(sat)
def add_extended_sat_metrics(self, sat):
if 'sat' in self.storj_collectors:
self.get_sat_data(sat)
labels = ['type', 'satellite', 'url']
label_values = [sat['id'], sat['url']]
metric_family = GaugeMetricFamily
if 'sat_data' in sat:
metric_name = 'storj_sat_nodeJoinedAt'
data = sat['sat_data']
documentation = 'Storj Node joined satellite at'
keys = ['nodeJoinedAt']
yield from self.dict_to_metric(data, metric_name, documentation, InfoMetricFamily, keys, ['satellite', 'url'], label_values)
metric_name = 'storj_sat_summary'
data = sat['sat_data']
documentation = 'Storj satellite summary metrics'
keys = ['storageSummary','bandwidthSummary','egressSummary','ingressSummary','onlineScore']
yield from self.dict_to_metric(data, metric_name, documentation, metric_family, keys, labels, label_values)
metric_name = 'storj_sat_audit'
data = sat['sat_data'].get('audit', None)
documentation = 'Storj satellite audit metrics'
keys = ['totalCount','successCount','alpha','beta','unknownAlpha','unknownBeta','score','unknownScore']
yield from self.dict_to_metric(data, metric_name, documentation, metric_family, keys, labels, label_values)
metric_name = 'storj_sat_uptime'
data = sat['sat_data'].get('uptime', None)
documentation = 'Storj satellite uptime metrics'
keys = ['totalCount','successCount','alpha','beta','unknownAlpha','unknownBeta','score','unknownScore']
yield from self.dict_to_metric(data, metric_name, documentation, metric_family, keys, labels, label_values)
metric_name = 'storj_sat_day_egress'
data = sat['sat_data'].get('bandwidthDaily', [{}])[-1].get('egress', None)
documentation = 'Storj satellite egress since current day start'
keys = ['repair','audit','usage']
yield from self.dict_to_metric(data, metric_name, documentation, metric_family, keys, labels, label_values)
metric_name = 'storj_sat_day_ingress'
data = sat['sat_data'].get('bandwidthDaily', [{}])[-1].get('ingress', None)
documentation = 'Storj satellite ingress since current day start'
keys = ['repair','usage']
yield from self.dict_to_metric(data, metric_name, documentation, metric_family, keys, labels, label_values)
metric_name = 'storj_sat_day_storage'
data = sat['sat_data'].get('storageDaily', [None])[-1]
documentation = 'Storj satellite data stored on disk since current day start'
keys = ['atRestTotal']
yield from self.dict_to_metric(data, metric_name, documentation, metric_family, keys, labels, label_values)
metric_name = 'storj_sat_month_egress'
data = sat['sat_data'].get('sum_bandwidthDaily_egress', None)
documentation = 'Storj satellite egress since current month start'
keys = ['repair','audit','usage']
yield from self.dict_to_metric(data, metric_name, documentation, metric_family, keys, labels, label_values)
metric_name = 'storj_sat_month_ingress'
data = sat['sat_data'].get('sum_bandwidthDaily_ingress', None)
documentation = 'Storj satellite ingress since current month start'
keys = ['repair','usage']
yield from self.dict_to_metric(data, metric_name, documentation, metric_family, keys, labels, label_values)
def collect(self):
yield from self.add_node_metrics()
if self.node_data:
yield from self.add_payout_metrics()
yield from self.add_sat_metrics()
if __name__ == '__main__':
storj_exporter_port = os.environ.get('STORJ_EXPORTER_PORT', '9651')
REGISTRY.register(StorjCollector())
start_http_server(int(storj_exporter_port))
while True: time.sleep(1)