illustris/pvemon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

parallelize nic info collection

Browse Source
This commit is contained in:
illustris 2023-10-02 19:49:47 +05:30
parent eea59a20b3
commit 6e90f95e1c
Signed by: illustris
GPG Key ID: 56C8FC0B899FEFA3
1 changed files with 28 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/pvemon/__init__.py
View File

@ -11,6 +11,9 @@ import pexpect
import logging import logging
import cProfile import cProfile
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
DEFAULT_PORT = 9116 DEFAULT_PORT = 9116
DEFAULT_INTERVAL = 10 DEFAULT_INTERVAL = 10
DEFAULT_PREFIX = "pve" DEFAULT_PREFIX = "pve"
@ -45,14 +48,17 @@ info_settings = [
flag_to_label_value = lambda args, match: next((args[i+1] for i, x in enumerate(args[:-1]) if x == match), "unknown").split(",")[0] flag_to_label_value = lambda args, match: next((args[i+1] for i, x in enumerate(args[:-1]) if x == match), "unknown").split(",")[0]
dynamic_gauges = {} dynamic_gauges = {}
gauge_lock = Lock() # avoid race condition when checking and creating gauges
def create_or_get_gauge(metric_name, labels): def create_or_get_gauge(metric_name, labels):
with gauge_lock:
if metric_name not in dynamic_gauges: if metric_name not in dynamic_gauges:
dynamic_gauges[metric_name] = Gauge(f"{prefix}_{metric_name}", f'{metric_name} for KVM process', labels) dynamic_gauges[metric_name] = Gauge(f"{prefix}_{metric_name}", f'{metric_name} for KVM process', labels)
return dynamic_gauges[metric_name] return dynamic_gauges[metric_name]
dynamic_infos = {} dynamic_infos = {}
info_lock = Lock() # avoid race condition when checking and creating infos
def create_or_get_info(info_name, labels): def create_or_get_info(info_name, labels):
with info_lock:
if (info_name,str(labels)) not in dynamic_infos: if (info_name,str(labels)) not in dynamic_infos:
dynamic_infos[(info_name,str(labels))] = Info(f"{prefix}_{info_name}", f'{info_name} for {str(labels)}', labels) dynamic_infos[(info_name,str(labels))] = Info(f"{prefix}_{info_name}", f'{info_name} for {str(labels)}', labels)
return dynamic_infos[(info_name,str(labels))] return dynamic_infos[(info_name,str(labels))]
@ -167,6 +173,9 @@ def collect_kvm_metrics():
for key, value in memory_metrics.items(): for key, value in memory_metrics.items():
gauge_dict["kvm_memory_extended"].labels(id=id, type=key).set(value) gauge_dict["kvm_memory_extended"].labels(id=id, type=key).set(value)
# upper limit on max_workers for safety
with ThreadPoolExecutor(max_workers=16) as executor:
def map_netstat_proc(id):
for nic_info in extract_nic_info_from_monitor(id): for nic_info in extract_nic_info_from_monitor(id):
queues = nic_info["queues"] queues = nic_info["queues"]
del nic_info["queues"] del nic_info["queues"]
@ -182,6 +191,8 @@ def collect_kvm_metrics():
gauge = create_or_get_gauge(metric_name, nic_labels.keys()) gauge = create_or_get_gauge(metric_name, nic_labels.keys())
gauge.labels(**nic_labels).set(value) gauge.labels(**nic_labels).set(value)
list(executor.map(map_netstat_proc, [ proc[2] for proc in procs ]))
def main(): def main():
parser = argparse.ArgumentParser(description='PVE metrics exporter for Prometheus') parser = argparse.ArgumentParser(description='PVE metrics exporter for Prometheus')
parser.add_argument('--port', type=int, default=DEFAULT_PORT, help='Port for the exporter to listen on') parser.add_argument('--port', type=int, default=DEFAULT_PORT, help='Port for the exporter to listen on')