Compare commits

...

2 Commits

Author SHA1 Message Date
illustris
0ba7c190f8 fix disk info collection 2026-03-12 16:03:36 +05:30
illustris
3808e3b672 add _info suffix to info metrics 2026-03-12 15:41:47 +05:30
4 changed files with 148 additions and 39 deletions

View File

@@ -49,6 +49,7 @@ type PVECollector struct {
descCtxSwitches *prometheus.Desc descCtxSwitches *prometheus.Desc
descNicInfo *prometheus.Desc descNicInfo *prometheus.Desc
descNicQueues *prometheus.Desc descNicQueues *prometheus.Desc
descDiskInfo *prometheus.Desc
descDiskSize *prometheus.Desc descDiskSize *prometheus.Desc
descStorageSize *prometheus.Desc descStorageSize *prometheus.Desc
descStorageFree *prometheus.Desc descStorageFree *prometheus.Desc
@@ -139,9 +140,14 @@ func NewWithDeps(cfg config.Config, proc procfs.ProcReader, sys sysfs.SysReader,
descMemExt: prometheus.NewDesc(p+"_kvm_memory_extended", "Extended memory info", []string{"id", "type"}, nil), descMemExt: prometheus.NewDesc(p+"_kvm_memory_extended", "Extended memory info", []string{"id", "type"}, nil),
descThreads: prometheus.NewDesc(p+"_kvm_threads", "Threads used", []string{"id"}, nil), descThreads: prometheus.NewDesc(p+"_kvm_threads", "Threads used", []string{"id"}, nil),
descCtxSwitches: prometheus.NewDesc(p+"_kvm_ctx_switches_total", "Context switches", []string{"id", "type"}, nil), descCtxSwitches: prometheus.NewDesc(p+"_kvm_ctx_switches_total", "Context switches", []string{"id", "type"}, nil),
descNicInfo: prometheus.NewDesc(p+"_kvm_nic", "NIC info", []string{"id", "ifname", "netdev", "queues", "type", "model", "macaddr"}, nil), descNicInfo: prometheus.NewDesc(p+"_kvm_nic_info", "NIC info", []string{"id", "ifname", "netdev", "queues", "type", "model", "macaddr"}, nil),
descNicQueues: prometheus.NewDesc(p+"_kvm_nic_queues", "NIC queue count", []string{"id", "ifname"}, nil), descNicQueues: prometheus.NewDesc(p+"_kvm_nic_queues", "NIC queue count", []string{"id", "ifname"}, nil),
descDiskSize: prometheus.NewDesc(p+"_kvm_disk_size_bytes", "Disk size bytes", []string{"id", "disk_name"}, nil), descDiskInfo: prometheus.NewDesc(p+"_kvm_disk_info", "Disk info", []string{
"id", "disk_name", "block_id", "disk_path", "disk_type",
"vol_name", "pool", "pool_name", "cluster_id", "vg_name",
"device", "attached_to", "cache_mode", "detect_zeroes", "read_only",
}, nil),
descDiskSize: prometheus.NewDesc(p+"_kvm_disk_size_bytes", "Disk size bytes", []string{"id", "disk_name"}, nil),
descStorageSize: prometheus.NewDesc(p+"_node_storage_size_bytes", "Storage total size", []string{"name", "type"}, nil), descStorageSize: prometheus.NewDesc(p+"_node_storage_size_bytes", "Storage total size", []string{"name", "type"}, nil),
descStorageFree: prometheus.NewDesc(p+"_node_storage_free_bytes", "Storage free space", []string{"name", "type"}, nil), descStorageFree: prometheus.NewDesc(p+"_node_storage_free_bytes", "Storage free space", []string{"name", "type"}, nil),
@@ -363,19 +369,23 @@ func (c *PVECollector) collectDiskMetrics(ch chan<- prometheus.Metric, proc proc
ch <- prometheus.MustNewConstMetric(c.descDiskSize, prometheus.GaugeValue, float64(diskSize), id, diskName) ch <- prometheus.MustNewConstMetric(c.descDiskSize, prometheus.GaugeValue, float64(diskSize), id, diskName)
} }
// Disk info metric - collect all labels // Disk info metric with fixed label set
labelNames := []string{"id", "disk_name", "block_id", "disk_path", "disk_type"} ch <- prometheus.MustNewConstMetric(c.descDiskInfo, prometheus.GaugeValue, 1,
labelValues := []string{id, diskName, disk.BlockID, disk.DiskPath, disk.DiskType} id,
diskName,
// Add variable labels in sorted-ish order disk.BlockID,
for _, key := range sortedKeys(disk.Labels) { disk.DiskPath,
labelNames = append(labelNames, key) disk.DiskType,
labelValues = append(labelValues, disk.Labels[key]) disk.Labels["vol_name"],
} disk.Labels["pool"],
disk.Labels["pool_name"],
ch <- prometheus.MustNewConstMetric( disk.Labels["cluster_id"],
prometheus.NewDesc(c.prefix+"_kvm_disk", "Disk info", labelNames, nil), disk.Labels["vg_name"],
prometheus.GaugeValue, 1, labelValues..., disk.Labels["device"],
disk.Labels["attached_to"],
disk.Labels["cache_mode"],
disk.Labels["detect_zeroes"],
disk.Labels["read_only"],
) )
} }
} }
@@ -383,21 +393,30 @@ func (c *PVECollector) collectDiskMetrics(ch chan<- prometheus.Metric, proc proc
func (c *PVECollector) collectStorage(ch chan<- prometheus.Metric) { func (c *PVECollector) collectStorage(ch chan<- prometheus.Metric) {
entries := c.getStorageEntries() entries := c.getStorageEntries()
// Compute superset of property keys across all entries
keySet := make(map[string]struct{})
for _, entry := range entries {
for k := range entry.Properties {
keySet[k] = struct{}{}
}
}
allKeys := sortedKeySet(keySet)
// Create descriptor once with fixed labels for this scrape
storageInfoDesc := prometheus.NewDesc(
c.prefix+"_node_storage_info", "Storage info", allKeys, nil,
)
for _, entry := range entries { for _, entry := range entries {
storageType := entry.Properties["type"] storageType := entry.Properties["type"]
storageName := entry.Properties["name"] storageName := entry.Properties["name"]
// Info metric // Info metric with consistent labels
labelNames := make([]string, 0, len(entry.Properties)) vals := make([]string, len(allKeys))
labelValues := make([]string, 0, len(entry.Properties)) for i, k := range allKeys {
for _, key := range sortedKeys(entry.Properties) { vals[i] = entry.Properties[k] // "" if missing
labelNames = append(labelNames, key)
labelValues = append(labelValues, entry.Properties[key])
} }
ch <- prometheus.MustNewConstMetric( ch <- prometheus.MustNewConstMetric(storageInfoDesc, prometheus.GaugeValue, 1, vals...)
prometheus.NewDesc(c.prefix+"_node_storage", "Storage info", labelNames, nil),
prometheus.GaugeValue, 1, labelValues...,
)
// Size metrics // Size metrics
var size storage.StorageSize var size storage.StorageSize
@@ -478,3 +497,12 @@ func sortedKeys(m map[string]string) []string {
return keys return keys
} }
func sortedKeySet(m map[string]struct{}) []string {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
slices.Sort(keys)
return keys
}

View File

@@ -307,7 +307,7 @@ func TestCollector_StorageMetrics(t *testing.T) {
} }
// Check storage info // Check storage info
infoMetrics := metrics["pve_node_storage"] infoMetrics := metrics["pve_node_storage_info"]
if len(infoMetrics) != 1 { if len(infoMetrics) != 1 {
t.Fatalf("expected 1 storage info metric, got %d", len(infoMetrics)) t.Fatalf("expected 1 storage info metric, got %d", len(infoMetrics))
} }
@@ -348,7 +348,7 @@ func TestCollector_NICMetrics(t *testing.T) {
metrics := collectMetrics(c) metrics := collectMetrics(c)
// NIC info // NIC info
nicInfo := metrics["pve_kvm_nic"] nicInfo := metrics["pve_kvm_nic_info"]
if len(nicInfo) != 1 { if len(nicInfo) != 1 {
t.Fatalf("expected 1 nic info, got %d", len(nicInfo)) t.Fatalf("expected 1 nic info, got %d", len(nicInfo))
} }
@@ -476,3 +476,89 @@ func TestCollector_BuildInfo(t *testing.T) {
t.Error("build_info missing version label") t.Error("build_info missing version label")
} }
} }
func TestCollector_DiskInfoMetrics(t *testing.T) {
cfg := config.Config{
CollectRunningVMs: true,
CollectStorage: false,
MetricsPrefix: "pve",
}
proc := &mockProcReader{
procs: []procfs.QEMUProcess{
{PID: 1, VMID: "100", Name: "vm", Vcores: 1, MaxMem: 1024},
},
cpuTimes: map[int]procfs.CPUTimes{1: {}},
ioCount: map[int]procfs.IOCounters{1: {}},
status: map[int]procfs.StatusInfo{
1: {Threads: 1, MemoryExtended: procfs.MemoryExtended{}},
},
memPct: map[int]float64{1: 0},
}
blockOutput := `drive-scsi0 (#block100): /dev/zvol/rpool/data/vm-100-disk-0 (raw, read-write)
Attached to: /machine/peripheral/virtioscsi0/virtio-backend
Cache mode: writeback, direct
Detect zeroes: on
drive-scsi1 (#block101): /mnt/storage/images/100/vm-100-disk-1.qcow2 (qcow2, read-only)
Attached to: /machine/peripheral/virtioscsi0/virtio-backend
`
sys := &mockSysReader{
blockSize: map[string]int64{
"/dev/zvol/rpool/data/vm-100-disk-0": 10737418240,
},
}
qm := &mockQMMonitor{responses: map[string]string{
"100:info network": "",
"100:info block": blockOutput,
}}
fr := &mockFileReader{files: map[string]string{"/etc/pve/user.cfg": ""}}
c := NewWithDeps(cfg, proc, sys, qm, &mockStatFS{}, &mockCmdRunner{}, fr)
metrics := collectMetrics(c)
diskInfo := metrics["pve_kvm_disk_info"]
if len(diskInfo) != 2 {
t.Fatalf("expected 2 disk info metrics, got %d", len(diskInfo))
}
// Check zvol disk
m := findMetricWithLabels(diskInfo, map[string]string{
"id": "100",
"disk_name": "scsi0",
"disk_type": "zvol",
"cache_mode": "writeback, direct",
"detect_zeroes": "on",
"read_only": "",
"vol_name": "vm-100-disk-0",
"pool": "rpool/data",
})
if m == nil {
t.Error("zvol disk info metric not found with expected labels")
}
// Check qcow2 disk (read-only, no cache_mode)
m = findMetricWithLabels(diskInfo, map[string]string{
"id": "100",
"disk_name": "scsi1",
"disk_type": "qcow2",
"read_only": "true",
"cache_mode": "",
"vol_name": "vm-100-disk-1",
})
if m == nil {
t.Error("qcow2 disk info metric not found with expected labels")
}
// Verify disk size for zvol
diskSize := metrics["pve_kvm_disk_size_bytes"]
if len(diskSize) < 1 {
t.Fatal("expected at least 1 disk size metric")
}
m = findMetricWithLabels(diskSize, map[string]string{"disk_name": "scsi0"})
if m == nil || metricValue(m) != 10737418240 {
t.Errorf("disk size for scsi0 = %v", m)
}
}

View File

@@ -3,6 +3,7 @@ package qmmonitor
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"log/slog"
"regexp" "regexp"
"strings" "strings"
) )
@@ -95,13 +96,7 @@ func ParseBlockInfo(raw string) map[string]DiskInfo {
info.Labels["attached_to"] = val info.Labels["attached_to"] = val
} else if strings.HasPrefix(line, "Cache mode:") { } else if strings.HasPrefix(line, "Cache mode:") {
val := strings.TrimSpace(strings.TrimPrefix(line, "Cache mode:")) val := strings.TrimSpace(strings.TrimPrefix(line, "Cache mode:"))
for _, mode := range strings.Split(val, ", ") { info.Labels["cache_mode"] = val
mode = strings.TrimSpace(mode)
if mode != "" {
key := "cache_mode_" + strings.ReplaceAll(mode, " ", "_")
info.Labels[key] = "true"
}
}
} else if strings.HasPrefix(line, "Detect zeroes:") { } else if strings.HasPrefix(line, "Detect zeroes:") {
info.Labels["detect_zeroes"] = "on" info.Labels["detect_zeroes"] = "on"
} }
@@ -110,6 +105,9 @@ func ParseBlockInfo(raw string) map[string]DiskInfo {
result[diskName] = info result[diskName] = info
} }
if len(result) == 0 && raw != "" {
slog.Debug("ParseBlockInfo found no disks", "rawLen", len(raw))
}
return result return result
} }

View File

@@ -27,11 +27,8 @@ func TestParseBlockInfo_Qcow2(t *testing.T) {
if d.Labels["detect_zeroes"] != "on" { if d.Labels["detect_zeroes"] != "on" {
t.Errorf("detect_zeroes = %q", d.Labels["detect_zeroes"]) t.Errorf("detect_zeroes = %q", d.Labels["detect_zeroes"])
} }
if d.Labels["cache_mode_writeback"] != "true" { if d.Labels["cache_mode"] != "writeback, direct" {
t.Errorf("cache_mode_writeback missing") t.Errorf("cache_mode = %q, want %q", d.Labels["cache_mode"], "writeback, direct")
}
if d.Labels["cache_mode_direct"] != "true" {
t.Errorf("cache_mode_direct missing")
} }
} }