From 020546f1d9d8e4446d61bef7e074205bfedeca06 Mon Sep 17 00:00:00 2001 From: matanbaruch Date: Sun, 26 Jul 2026 00:23:03 +0300 Subject: [PATCH] feat(api): add an admin only Prometheus metrics endpoint Add a GET /api/v1/metrics operation returning the device, user and group counters of the platform using the Prometheus text exposition format, so device usage can be graphed in Prometheus and Grafana. The operation is tagged admin and the controller checks the privilege of the caller as well, since the device channel branch of accessTokenAuth() does not set req.user. The counters cover all the objects of the platform whatever the group they belong to, which a simple user is not allowed to see. Counters are computed on scrape rather than on a timer, so no extra unit or background job is needed and the returned values are always those of the very moment the Prometheus server asked for them. Device states are computed with the same state machine the device table uses in get-device-state.util.ts, including the Apple PREPARING and the UNHEALTHY cases, and every label value is known in advance so that a counter dropping to zero is exported as zero instead of vanishing. Signed-off-by: matanbaruch --- README.md | 5 + doc/METRICS.md | 138 ++++++++++++++++ lib/db/models/device/model.js | 16 ++ lib/units/api/controllers/metrics.js | 41 +++++ lib/units/api/paths/metrics.js | 10 ++ lib/units/api/swagger/api_v1.yaml | 31 ++++ lib/util/metrics.js | 227 +++++++++++++++++++++++++++ package-lock.json | 47 ++++-- package.json | 1 + 9 files changed, 500 insertions(+), 16 deletions(-) create mode 100644 doc/METRICS.md create mode 100644 lib/units/api/controllers/metrics.js create mode 100644 lib/units/api/paths/metrics.js create mode 100644 lib/util/metrics.js diff --git a/README.md b/README.md index dc22ab0d95..ae40f7000a 100644 --- a/README.md +++ b/README.md @@ -217,6 +217,11 @@ If you want use scrcpy instead minicap run app with next command(Scrcpy function stf local --need-scrcpy true ``` +## Monitoring + +DeviceHub exposes the platform counters in the Prometheus format. Refer to +[METRICS.md](doc/METRICS.md) for the exposed metrics and the scrape setup. + ## Testing Refer to [TESTING.md](doc/TESTING.md) for testing instructions. diff --git a/doc/METRICS.md b/doc/METRICS.md new file mode 100644 index 0000000000..1208ccc09a --- /dev/null +++ b/doc/METRICS.md @@ -0,0 +1,138 @@ +# Metrics + +DeviceHub exposes the counters of the whole platform through the `GET /api/v1/metrics` operation of +the [API](API.md), using the [Prometheus text exposition format][exposition-format]. + +The operation is a privileged one: it is tagged `admin` in the API specification and the controller +checks the privilege of the caller as well, so it is reserved to administrator users. This is +required because the returned counters cover all the devices, users and groups whatever the group +they belong to, which a simple user is not allowed to see. + +The counters are computed when the endpoint is scraped, not on a timer, so the returned values are +always those of the very moment the Prometheus server asked for them, and no extra unit or +background job has to run. + +## Exposed metrics + +| Metric | Type | Labels | Description | +| ------ | ---- | ------ | ----------- | +| `devicehub_devices_total` | gauge | | Number of devices known to DeviceHub, whether they are present or not | +| `devicehub_devices_by_state` | gauge | `state` | Number of devices per aggregate device state | +| `devicehub_devices_available` | gauge | | Number of devices in the `available` state | +| `devicehub_devices_busy` | gauge | | Number of devices in the `busy` state | +| `devicehub_providers_total` | gauge | | Number of distinct providers serving at least one present device | +| `devicehub_users_total` | gauge | | Number of users known to DeviceHub | +| `devicehub_users_by_privilege` | gauge | `privilege` | Number of users per privilege (`root`, `admin`, `user`) | +| `devicehub_groups_total` | gauge | | Number of groups known to DeviceHub | +| `devicehub_groups_active` | gauge | | Number of groups which are currently active | +| `devicehub_groups_by_state` | gauge | `state` | Number of groups per group state (`pending`, `ready`, `waiting`) | +| `devicehub_groups_by_class` | gauge | `class` | Number of groups per group class (`once`, `bookable`, `standard`, `hourly`, ...) | + +The `app="devicehub"` label is added to every metric, and the standard `process_*` and `nodejs_*` +metrics of the API process are exposed as well. + +The `state` label of `devicehub_devices_by_state` holds the aggregate device state, computed with the +same state machine the device table uses in `ui/src/lib/utils/get-device-state.util.ts`, with the +branches that depend on who is looking at the device dropped: + +| State | Meaning | +| ----- | ------- | +| `absent` | The device is not plugged to any provider | +| `offline` | The device is present but reported offline | +| `unauthorized` | The device is present but not authorized | +| `preparing` | The device is online but not ready yet | +| `available` | The device is ready and owned by nobody, or an Apple device in the `PREPARING` status | +| `busy` | The device is ready and owned by a user | +| `unhealthy` | The device reports the `UNHEALTHY` status | +| `present` | The device is present and in none of the states above | + +The `using` and `automation` states of the device table are not exposed since they are relative to +the user looking at the device, which has no meaning for a scraper. For the same reason the +`isDeviceUsable` correction the device table applies to a device held by somebody else is not +applied here: a ready device with an owner is counted as `busy`. + +Every label value is known in advance, so a counter which drops to zero is exported as zero instead +of vanishing, and a document holding an unexpected value can't create new time series. + +## Scraping the endpoint + +The operation uses the same authentication as the rest of the API, so the Prometheus server needs an +access token belonging to an administrator user. Generate one from the UI, in *Settings* > *Keys* > +*Access Tokens*, while logged in as an administrator. + +```yaml +scrape_configs: + - job_name: devicehub + metrics_path: /api/v1/metrics + scheme: http + authorization: + type: Bearer + credentials: + static_configs: + - targets: ['devicehub.example.org:7100'] +``` + +Errors are reported the way the rest of the API reports them, as a JSON body: `401` when the token +is missing or invalid, `403` when the token belongs to a simple user, and `500` when the counters +can't be read from the database. Only the successful response uses the Prometheus text format, since +that is what the format is specified for. + +## Setting up a minimal test environment + +Start DeviceHub as usual, for instance with +[docker-compose-dev.yaml](../docker-compose-dev.yaml), then add a Prometheus server and a Grafana +instance next to it: + +```yaml +services: + prometheus: + image: prom/prometheus:v3.1.0 + ports: + - "9090:9090" + volumes: + - "./prometheus.yml:/etc/prometheus/prometheus.yml" + + grafana: + image: grafana/grafana:11.5.1 + ports: + - "3000:3000" + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin +``` + +With a `prometheus.yml` holding the scrape configuration above and a `15s` scrape interval: + +```yaml +global: + scrape_interval: 15s +``` + +Check that the endpoint answers, then that Prometheus scrapes it: + +```bash +curl -H "Authorization: Bearer $DEVICEHUB_ADMIN_ACCESS_TOKEN" \ + http://localhost:7100/api/v1/metrics +``` + +``` +# HELP devicehub_devices_total Number of devices known to DeviceHub, whether they are present or not +# TYPE devicehub_devices_total gauge +devicehub_devices_total{app="devicehub"} 5 +# HELP devicehub_devices_by_state Number of devices per aggregate device state +# TYPE devicehub_devices_by_state gauge +devicehub_devices_by_state{state="absent",app="devicehub"} 1 +devicehub_devices_by_state{state="available",app="devicehub"} 2 +devicehub_devices_by_state{state="busy",app="devicehub"} 1 +devicehub_devices_by_state{state="unhealthy",app="devicehub"} 1 +... +``` + +The target then shows up as `UP` on http://localhost:9090/targets, and Grafana can be pointed at +`http://prometheus:9090` to graph the series, for example the share of devices in use: + +``` +sum(devicehub_devices_busy) / sum(devicehub_devices_total) +``` + +[exposition-format]: diff --git a/lib/db/models/device/model.js b/lib/db/models/device/model.js index b00c715014..25bea3b44a 100644 --- a/lib/db/models/device/model.js +++ b/lib/db/models/device/model.js @@ -46,6 +46,22 @@ export const getDevicesCount = function() { return db.devices.countDocuments() } +// Returns every device with only the fields needed to compute its state, whatever the group it +// belongs to; reserved to privileged (admin) operations +export const getDevicesForMetrics = function() { + return db.devices.find({}, { + projection: { + _id: 0, + present: 1, + status: 1, + ready: 1, + owner: 1, + manufacturer: 1, + 'provider.name': 1 + } + }).toArray() +} + // dbapi.getOfflineDevicesCount = function() { export const getOfflineDevicesCount = function() { return db.devices.countDocuments( diff --git a/lib/units/api/controllers/metrics.js b/lib/units/api/controllers/metrics.js new file mode 100644 index 0000000000..a16b2caead --- /dev/null +++ b/lib/units/api/controllers/metrics.js @@ -0,0 +1,41 @@ +/* * + * Copyright 2026 Matan Baruch - Licensed under the Apache license 2.0 + * */ +import * as apiutil from '../../../util/apiutil.js' +import * as metrics from '../../../util/metrics.js' +import DeviceModel from '../../../db/models/device/index.js' +import GroupModel from '../../../db/models/group/index.js' +import UserModel from '../../../db/models/user/index.js' + +// Counters are computed on scrape rather than on a timer, so that the returned values are always +// the ones of the very moment the Prometheus server asked for them +function getMetrics(req, res) { + // The admin tag of the operation is not enough on its own: the device channel branch of + // accessTokenAuth() authenticates without setting req.user at all, so check the caller here + if (!req.user || req.user.privilege === apiutil.USER) { + apiutil.respond(res, 403, 'Forbidden: privileged operation (admin)') + return + } + + Promise.all([ + DeviceModel.getDevicesForMetrics(), + UserModel.getUsers(), + GroupModel.getGroups({}) + ]) + .then(([devices, users, groups]) => { + metrics.update(devices, users, groups) + return metrics.register.metrics() + }) + .then((body) => { + res.set('Content-Type', metrics.register.contentType) + res.status(200).send(body) + }) + .catch((err) => { + apiutil.internalError(res, 'Failed to get metrics: ', err.stack) + }) +} + +export {getMetrics} +export default { + getMetrics: getMetrics +} diff --git a/lib/units/api/paths/metrics.js b/lib/units/api/paths/metrics.js new file mode 100644 index 0000000000..6ec35ca7a4 --- /dev/null +++ b/lib/units/api/paths/metrics.js @@ -0,0 +1,10 @@ +// Generated by /lib/units/api/gen_routes.py. DO NOT EDIT MANUALLY +// Generated for controller metrics + +import {getMetrics} from '../controllers/metrics.js' + +export function get(req, res) { + return getMetrics(req, res) +} + + diff --git a/lib/units/api/swagger/api_v1.yaml b/lib/units/api/swagger/api_v1.yaml index 1f6f2ed149..a6d3c56c59 100644 --- a/lib/units/api/swagger/api_v1.yaml +++ b/lib/units/api/swagger/api_v1.yaml @@ -25,6 +25,8 @@ tags: description: Privileged Operations - name: autotests description: Autotests Operations +- name: metrics + description: Metrics Operations paths: /groups: get: @@ -3426,6 +3428,35 @@ paths: $ref: '#/components/schemas/UnexpectedErrorResponse' x-codegen-request-body-name: serial x-swagger-router-controller: autotests + /metrics: + get: + tags: + - metrics + - admin + summary: Gets the platform metrics + description: Returns the device, user and group counters of the whole platform using the + Prometheus text exposition format; this is a privileged operation reserved to the + administrator user because the returned counters cover all the objects of the platform, + whatever the group they belong to + operationId: getMetrics + responses: + "200": + description: Platform metrics using the Prometheus text exposition format + content: + text/plain: + schema: + type: string + default: + description: | + Unexpected Error: + * 401: Unauthorized => bad credentials + * 403: Forbidden => privileged operation (admin) + * 500: Internal Server Error + content: + application/json: + schema: + $ref: '#/components/schemas/UnexpectedErrorResponse' + x-swagger-router-controller: metrics /stats: post: tags: diff --git a/lib/util/metrics.js b/lib/util/metrics.js new file mode 100644 index 0000000000..8bc1c02007 --- /dev/null +++ b/lib/util/metrics.js @@ -0,0 +1,227 @@ +/* * + * Copyright 2026 Matan Baruch - Licensed under the Apache license 2.0 + * */ +import client from 'prom-client' +import * as apiutil from './apiutil.js' +import {DeviceStatus} from '../wire/wire.js' + +// Aggregate device states, computed the same way the device table does in +// ui/src/lib/utils/get-device-state.util.ts; the 'using' and 'automation' states are not computed +// here since they only mean something inside a given user session +export const DEVICE_STATES = [ + 'absent', + 'offline', + 'unauthorized', + 'preparing', + 'busy', + 'available', + 'unhealthy', + 'present' +] + +export const GROUP_STATES = [apiutil.PENDING, apiutil.READY, apiutil.WAITING] +export const GROUP_CLASSES = Object.keys(apiutil.CLASS_DURATION) +export const USER_PRIVILEGES = [apiutil.ROOT, apiutil.ADMIN, apiutil.USER] + +export const register = new client.Registry() + +register.setDefaultLabels({app: 'devicehub'}) +client.collectDefaultMetrics({register: register}) + +export const gauges = { + devicesTotal: new client.Gauge({ + name: 'devicehub_devices_total', + help: 'Number of devices known to DeviceHub, whether they are present or not', + registers: [register] + }), + devicesByState: new client.Gauge({ + name: 'devicehub_devices_by_state', + help: 'Number of devices per aggregate device state', + labelNames: ['state'], + registers: [register] + }), + devicesAvailable: new client.Gauge({ + name: 'devicehub_devices_available', + help: 'Number of devices in the available state', + registers: [register] + }), + devicesBusy: new client.Gauge({ + name: 'devicehub_devices_busy', + help: 'Number of devices in the busy state', + registers: [register] + }), + providersTotal: new client.Gauge({ + name: 'devicehub_providers_total', + help: 'Number of distinct providers serving at least one present device', + registers: [register] + }), + usersTotal: new client.Gauge({ + name: 'devicehub_users_total', + help: 'Number of users known to DeviceHub', + registers: [register] + }), + usersByPrivilege: new client.Gauge({ + name: 'devicehub_users_by_privilege', + help: 'Number of users per privilege', + labelNames: ['privilege'], + registers: [register] + }), + groupsTotal: new client.Gauge({ + name: 'devicehub_groups_total', + help: 'Number of groups known to DeviceHub', + registers: [register] + }), + groupsActive: new client.Gauge({ + name: 'devicehub_groups_active', + help: 'Number of groups which are currently active', + registers: [register] + }), + groupsByState: new client.Gauge({ + name: 'devicehub_groups_by_state', + help: 'Number of groups per group state', + labelNames: ['state'], + registers: [register] + }), + groupsByClass: new client.Gauge({ + name: 'devicehub_groups_by_class', + help: 'Number of groups per group class', + labelNames: ['class'], + registers: [register] + }) +} + +const zeroFill = function(values) { + return values.reduce((counts, value) => { + counts[value] = 0 + return counts + }, Object.create(null)) +} + +// Counts only the already known label values so that an unexpected document can't create an +// unbounded number of time series +const count = function(counts, value) { + const current = counts[value] + + if (typeof current === 'number') { + counts[value] = current + 1 + } +} + +const setLabeled = function(gauge, label, counts) { + gauge.reset() + Object.keys(counts).forEach((value) => { + gauge.set({[label]: value}, counts[value]) + }) +} + +export const deviceState = function(device) { + if (!device.present) { + return 'absent' + } + if (device.status === DeviceStatus.OFFLINE) { + return 'offline' + } + if (device.status === DeviceStatus.UNAUTHORIZED) { + return 'unauthorized' + } + if (device.status === DeviceStatus.ONLINE) { + if (!device.ready) { + return 'preparing' + } + return device.owner ? 'busy' : 'available' + } + if (device.status === DeviceStatus.PREPARING && device.manufacturer === 'Apple') { + return 'available' + } + if (device.status === DeviceStatus.UNHEALTHY) { + return 'unhealthy' + } + return 'present' +} + +export const aggregateDevices = function(devices) { + const stats = { + total: devices.length, + available: 0, + busy: 0, + providers: 0, + byState: zeroFill(DEVICE_STATES) + } + const providers = Object.create(null) + + devices.forEach((device) => { + const state = deviceState(device) + + count(stats.byState, state) + + if (state === 'available') { + stats.available += 1 + } + if (state === 'busy') { + stats.busy += 1 + } + if (device.present && device.provider && device.provider.name) { + providers[device.provider.name] = true + } + }) + stats.providers = Object.keys(providers).length + return stats +} + +export const aggregateUsers = function(users) { + const stats = { + total: users.length, + byPrivilege: zeroFill(USER_PRIVILEGES) + } + + users.forEach((user) => count(stats.byPrivilege, user.privilege)) + return stats +} + +export const aggregateGroups = function(groups) { + const stats = { + total: groups.length, + active: 0, + byState: zeroFill(GROUP_STATES), + byClass: zeroFill(GROUP_CLASSES) + } + + groups.forEach((group) => { + if (group.isActive) { + stats.active += 1 + } + count(stats.byState, group.state) + count(stats.byClass, group.class) + }) + return stats +} + +export const update = function(devices, users, groups) { + const deviceStats = aggregateDevices(devices) + const userStats = aggregateUsers(users) + const groupStats = aggregateGroups(groups) + + gauges.devicesTotal.set(deviceStats.total) + gauges.devicesAvailable.set(deviceStats.available) + gauges.devicesBusy.set(deviceStats.busy) + gauges.providersTotal.set(deviceStats.providers) + setLabeled(gauges.devicesByState, 'state', deviceStats.byState) + + gauges.usersTotal.set(userStats.total) + setLabeled(gauges.usersByPrivilege, 'privilege', userStats.byPrivilege) + + gauges.groupsTotal.set(groupStats.total) + gauges.groupsActive.set(groupStats.active) + setLabeled(gauges.groupsByState, 'state', groupStats.byState) + setLabeled(gauges.groupsByClass, 'class', groupStats.byClass) +} + +export default { + register: register, + gauges: gauges, + deviceState: deviceState, + aggregateDevices: aggregateDevices, + aggregateUsers: aggregateUsers, + aggregateGroups: aggregateGroups, + update: update +} diff --git a/package-lock.json b/package-lock.json index 72c2916a66..0b0abda223 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "vk-devicehub", - "version": "1.5.0", + "version": "1.5.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "vk-devicehub", - "version": "1.5.0", + "version": "1.5.2", "hasInstallScript": true, "dependencies": { "@aws-sdk/client-s3": "^3.772.0", @@ -80,6 +80,7 @@ "passport": "0.6.0", "passport-oauth2": "1.7.0", "postman-request": "^2.88.1-postman.33", + "prom-client": "^15.1.3", "promise-socket": "7.0.0", "proper-lockfile": "^4.1.2", "protobufjs": "5.0.3", @@ -7711,20 +7712,6 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, - "node_modules/are-we-there-yet/node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "license": "MIT", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -7967,6 +7954,12 @@ "node": ">=0.6" } }, + "node_modules/bintrees": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", + "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==", + "license": "MIT" + }, "node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -14090,6 +14083,19 @@ "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "license": "MIT" }, + "node_modules/prom-client": { + "version": "15.1.3", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz", + "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.4.0", + "tdigest": "^0.1.1" + }, + "engines": { + "node": "^16 || ^18 || >=20" + } + }, "node_modules/promise-duplex": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/promise-duplex/-/promise-duplex-6.0.0.tgz", @@ -16109,6 +16115,15 @@ "node": ">=6" } }, + "node_modules/tdigest": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", + "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", + "license": "MIT", + "dependencies": { + "bintrees": "1.0.2" + } + }, "node_modules/teen_process": { "version": "1.16.0", "resolved": "https://registry.npmjs.org/teen_process/-/teen_process-1.16.0.tgz", diff --git a/package.json b/package.json index 6369834519..127fa4e2ce 100644 --- a/package.json +++ b/package.json @@ -108,6 +108,7 @@ "passport": "0.6.0", "passport-oauth2": "1.7.0", "postman-request": "^2.88.1-postman.33", + "prom-client": "^15.1.3", "promise-socket": "7.0.0", "proper-lockfile": "^4.1.2", "protobufjs": "5.0.3",