Feat: add prometheus observability

[sansmoraxz]

Exploring some more examples I saw callbacks is also an option. Wierd thing is they use float for API access even though internally it stores as atomic uint64.

Fixes: #136

[sansmoraxz]

Further research showed we can just have callback hooks for the metrics but for for some wierd reason prometheus exposes api as float even though it uses uint64 internally.

Also although I disabled the default metrics there are some useful info

Details

# HELP go_gc_duration_seconds A summary of the wall-time pause (stop-the-world) duration in garbage collection cycles.
# TYPE go_gc_duration_seconds summary
go_gc_duration_seconds{quantile="0"} 0
go_gc_duration_seconds{quantile="0.25"} 0
go_gc_duration_seconds{quantile="0.5"} 0
go_gc_duration_seconds{quantile="0.75"} 0
go_gc_duration_seconds{quantile="1"} 0
go_gc_duration_seconds_sum 0
go_gc_duration_seconds_count 0
# HELP go_gc_gogc_percent Heap size target percentage configured by the user, otherwise 100. This value is set by the GOGC environment variable, and the runtime/debug.SetGCPercent function. Sourced from /gc/gogc:percent.
# TYPE go_gc_gogc_percent gauge
go_gc_gogc_percent 100
# HELP go_gc_gomemlimit_bytes Go runtime memory limit configured by the user, otherwise math.MaxInt64. This value is set by the GOMEMLIMIT environment variable, and the runtime/debug.SetMemoryLimit function. Sourced from /gc/gomemlimit:bytes.
# TYPE go_gc_gomemlimit_bytes gauge
go_gc_gomemlimit_bytes 9.223372036854776e+18
# HELP go_goroutines Number of goroutines that currently exist.
# TYPE go_goroutines gauge
go_goroutines 12
# HELP go_info Information about the Go environment.
# TYPE go_info gauge
go_info{version="go1.25.4 X:nodwarf5"} 1
# HELP go_memstats_alloc_bytes Number of bytes allocated in heap and currently in use. Equals to /memory/classes/heap/objects:bytes.
# TYPE go_memstats_alloc_bytes gauge
go_memstats_alloc_bytes 2.837512e+06
# HELP go_memstats_alloc_bytes_total Total number of bytes allocated in heap until now, even if released already. Equals to /gc/heap/allocs:bytes.
# TYPE go_memstats_alloc_bytes_total counter
go_memstats_alloc_bytes_total 2.837512e+06
# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. Equals to /memory/classes/profiling/buckets:bytes.
# TYPE go_memstats_buck_hash_sys_bytes gauge
go_memstats_buck_hash_sys_bytes 1.445325e+06
# HELP go_memstats_frees_total Total number of heap objects frees. Equals to /gc/heap/frees:objects + /gc/heap/tiny/allocs:objects.
# TYPE go_memstats_frees_total counter
go_memstats_frees_total 571
# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. Equals to /memory/classes/metadata/other:bytes.
# TYPE go_memstats_gc_sys_bytes gauge
go_memstats_gc_sys_bytes 2.188048e+06
# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and currently in use, same as go_memstats_alloc_bytes. Equals to /memory/classes/heap/objects:bytes.
# TYPE go_memstats_heap_alloc_bytes gauge
go_memstats_heap_alloc_bytes 2.837512e+06
# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. Equals to /memory/classes/heap/released:bytes + /memory/classes/heap/free:bytes.
# TYPE go_memstats_heap_idle_bytes gauge
go_memstats_heap_idle_bytes 3.60448e+06
# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. Equals to /memory/classes/heap/objects:bytes + /memory/classes/heap/unused:bytes
# TYPE go_memstats_heap_inuse_bytes gauge
go_memstats_heap_inuse_bytes 4.096e+06
# HELP go_memstats_heap_objects Number of currently allocated objects. Equals to /gc/heap/objects:objects.
# TYPE go_memstats_heap_objects gauge
go_memstats_heap_objects 22800
# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS. Equals to /memory/classes/heap/released:bytes.
# TYPE go_memstats_heap_released_bytes gauge
go_memstats_heap_released_bytes 3.60448e+06
# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. Equals to /memory/classes/heap/objects:bytes + /memory/classes/heap/unused:bytes + /memory/classes/heap/released:bytes + /memory/classes/heap/free:bytes.
# TYPE go_memstats_heap_sys_bytes gauge
go_memstats_heap_sys_bytes 7.70048e+06
# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection.
# TYPE go_memstats_last_gc_time_seconds gauge
go_memstats_last_gc_time_seconds 0
# HELP go_memstats_mallocs_total Total number of heap objects allocated, both live and gc-ed. Semantically a counter version for go_memstats_heap_objects gauge. Equals to /gc/heap/allocs:objects + /gc/heap/tiny/allocs:objects.
# TYPE go_memstats_mallocs_total counter
go_memstats_mallocs_total 23371
# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. Equals to /memory/classes/metadata/mcache/inuse:bytes.
# TYPE go_memstats_mcache_inuse_bytes gauge
go_memstats_mcache_inuse_bytes 19328
# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system. Equals to /memory/classes/metadata/mcache/inuse:bytes + /memory/classes/metadata/mcache/free:bytes.
# TYPE go_memstats_mcache_sys_bytes gauge
go_memstats_mcache_sys_bytes 31408
# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. Equals to /memory/classes/metadata/mspan/inuse:bytes.
# TYPE go_memstats_mspan_inuse_bytes gauge
go_memstats_mspan_inuse_bytes 96320
# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. Equals to /memory/classes/metadata/mspan/inuse:bytes + /memory/classes/metadata/mspan/free:bytes.
# TYPE go_memstats_mspan_sys_bytes gauge
go_memstats_mspan_sys_bytes 97920
# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. Equals to /gc/heap/goal:bytes.
# TYPE go_memstats_next_gc_bytes gauge
go_memstats_next_gc_bytes 4.194304e+06
# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. Equals to /memory/classes/other:bytes.
# TYPE go_memstats_other_sys_bytes gauge
go_memstats_other_sys_bytes 913147
# HELP go_memstats_stack_inuse_bytes Number of bytes obtained from system for stack allocator in non-CGO environments. Equals to /memory/classes/heap/stacks:bytes.
# TYPE go_memstats_stack_inuse_bytes gauge
go_memstats_stack_inuse_bytes 688128
# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. Equals to /memory/classes/heap/stacks:bytes + /memory/classes/os-stacks:bytes.
# TYPE go_memstats_stack_sys_bytes gauge
go_memstats_stack_sys_bytes 688128
# HELP go_memstats_sys_bytes Number of bytes obtained from system. Equals to /memory/classes/total:byte.
# TYPE go_memstats_sys_bytes gauge
go_memstats_sys_bytes 1.3064456e+07
# HELP go_sched_gomaxprocs_threads The current runtime.GOMAXPROCS setting, or the number of operating system threads that can execute user-level Go code simultaneously. Sourced from /sched/gomaxprocs:threads.
# TYPE go_sched_gomaxprocs_threads gauge
go_sched_gomaxprocs_threads 16
# HELP go_threads Number of OS threads created.
# TYPE go_threads gauge
go_threads 10
# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds.
# TYPE process_cpu_seconds_total counter
process_cpu_seconds_total 0
# HELP process_max_fds Maximum number of open file descriptors.
# TYPE process_max_fds gauge
process_max_fds 524288
# HELP process_network_receive_bytes_total Number of bytes received by the process over the network.
# TYPE process_network_receive_bytes_total counter
process_network_receive_bytes_total 1.39230664e+08
# HELP process_network_transmit_bytes_total Number of bytes sent by the process over the network.
# TYPE process_network_transmit_bytes_total counter
process_network_transmit_bytes_total 2.537586804e+09
# HELP process_open_fds Number of open file descriptors.
# TYPE process_open_fds gauge
process_open_fds 15
# HELP process_resident_memory_bytes Resident memory size in bytes.
# TYPE process_resident_memory_bytes gauge
process_resident_memory_bytes 2.5329664e+07
# HELP process_start_time_seconds Start time of the process since unix epoch in seconds.
# TYPE process_start_time_seconds gauge
process_start_time_seconds 1.76443010589e+09
# HELP process_virtual_memory_bytes Virtual memory size in bytes.
# TYPE process_virtual_memory_bytes gauge
process_virtual_memory_bytes 1.552760832e+09
# HELP process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes.
# TYPE process_virtual_memory_max_bytes gauge
process_virtual_memory_max_bytes 1.8446744073709552e+19
# HELP promhttp_metric_handler_requests_in_flight Current number of scrapes being served.
# TYPE promhttp_metric_handler_requests_in_flight gauge
promhttp_metric_handler_requests_in_flight 1
# HELP promhttp_metric_handler_requests_total Total number of scrapes by HTTP status code.
# TYPE promhttp_metric_handler_requests_total counter
promhttp_metric_handler_requests_total{code="200"} 0
promhttp_metric_handler_requests_total{code="500"} 0
promhttp_metric_handler_requests_total{code="503"} 0

Do we want that too @wind-c ? Or does it feel like too much noise?

[wind-c]

You have added quite a lot of code. I will need some time to review it! Thank you very much for your exploration. Let's work together!

[sansmoraxz]

reminder @wind-c please check once you get time

[wind-c]

Sorry, I've been too busy recently. Could you please attach a screenshot of the visualization indicator effect? @sansmoraxz

[sansmoraxz]

Could you please attach a screenshot of the visualization indicator effect?

Did not get you. Could you explain what exactly you want. It's just publishing metrics here. Do you want something like grafana dashboards? Dashboards aren't part of this PR, only the tools to build it are.

[wind-c]

Just show me the actual results of adding Prometheus observability!

[sansmoraxz]

Ohh check below:

Details

From my local server: http://localhost:8080/metrics

# HELP bytes_received total number of bytes received since the broker started
# TYPE bytes_received counter
bytes_received 777
# HELP bytes_sent total number of bytes sent since the broker started
# TYPE bytes_sent counter
bytes_sent 916
# HELP clients_connected number of currently connected clients
# TYPE clients_connected gauge
clients_connected 1
# HELP clients_disconnected total number of persistent clients (with clean session disabled) that are registered at the broker but are currently disconnected
# TYPE clients_disconnected gauge
clients_disconnected 0
# HELP clients_maximum maximum number of active clients that have been connected
# TYPE clients_maximum gauge
clients_maximum 2
# HELP clients_total total number of connected and disconnected clients with a persistent session currently connected and registered
# TYPE clients_total gauge
clients_total 1
# HELP inflight the number of messages currently in-flight
# TYPE inflight gauge
inflight 0
# HELP inflight_dropped the number of inflight messages which were dropped
# TYPE inflight_dropped gauge
inflight_dropped 0
# HELP memory_alloc memory currently allocated in bytes
# TYPE memory_alloc gauge
memory_alloc 8.8064e+06
# HELP messages_dropped total number of publish messages dropped to slow subscriber
# TYPE messages_dropped counter
messages_dropped 0
# HELP messages_received total number of publish messages received
# TYPE messages_received counter
messages_received 7
# HELP messages_sent total number of publish messages sent
# TYPE messages_sent counter
messages_sent 9
# HELP packets_received the total number of publish messages received
# TYPE packets_received counter
packets_received 35
# HELP packets_sent total number of messages of any type sent since the broker started
# TYPE packets_sent counter
packets_sent 26
# HELP retained total number of retained messages active on the broker
# TYPE retained gauge
retained 22
# HELP subscriptions total number of subscriptions active on the broker
# TYPE subscriptions gauge
subscriptions 1
# HELP threads number of active goroutines, named as threads for platform ambiguity
# TYPE threads gauge
threads 43

[sansmoraxz]

I have taken these from here:

comqtt/mqtt/system/system.go

Lines 16 to 38 in 0949432

	type Info struct {
	Version string `json:"version"` // the current version of the server
	Started int64 `json:"started"` // the time the server started in unix seconds
	Time int64 `json:"time"` // current time on the server
	Uptime int64 `json:"uptime"` // the number of seconds the server has been online
	BytesReceived int64 `json:"bytes_received"` // total number of bytes received since the broker started
	BytesSent int64 `json:"bytes_sent"` // total number of bytes sent since the broker started
	ClientsConnected int64 `json:"clients_connected"` // number of currently connected clients
	ClientsDisconnected int64 `json:"clients_disconnected"` // total number of persistent clients (with clean session disabled) that are registered at the broker but are currently disconnected
	ClientsMaximum int64 `json:"clients_maximum"` // maximum number of active clients that have been connected
	ClientsTotal int64 `json:"clients_total"` // total number of connected and disconnected clients with a persistent session currently connected and registered
	MessagesReceived int64 `json:"messages_received"` // total number of publish messages received
	MessagesSent int64 `json:"messages_sent"` // total number of publish messages sent
	MessagesDropped int64 `json:"messages_dropped"` // total number of publish messages dropped to slow subscriber
	Retained int64 `json:"retained"` // total number of retained messages active on the broker
	Inflight int64 `json:"inflight"` // the number of messages currently in-flight
	InflightDropped int64 `json:"inflight_dropped"` // the number of inflight messages which were dropped
	Subscriptions int64 `json:"subscriptions"` // total number of subscriptions active on the broker
	PacketsReceived int64 `json:"packets_received"` // the total number of publish messages received
	PacketsSent int64 `json:"packets_sent"` // total number of messages of any type sent since the broker started
	MemoryAlloc int64 `json:"memory_alloc"` // memory currently allocated (in bytes)
	Threads int64 `json:"threads"` // number of active goroutines, named as threads for platform ambiguity
	}

[wind-c]

Thanks！