Design a Live Metrics Dashboard

groups:
  - name: api-rollups
    interval: 30s                 # evaluated on a schedule, independent of the scrape
    rules:
      # Collapses 40 pods into one series per route and status class. The
      # dashboard then reads 100 series instead of 4,000, every refresh.
      - record: job:http_requests:rate5m
        expr: sum by (job, route, code_class) (rate(http_requests_total[5m]))

      # Same idea for latency, but le MUST survive the aggregation or
      # histogram_quantile has no distribution left to interpolate over.
      - record: job:http_request_duration_seconds_bucket:rate5m
        expr: sum by (job, route, le) (rate(http_request_duration_seconds_bucket[5m]))

      # Error ratio precomputed once, so the SLO panel is a single series read
      # rather than two range queries divided in the browser.
      - record: job:http_errors:ratio5m
        expr: |
          sum by (job, route) (rate(http_requests_total{code_class="5xx"}[5m]))
            /
          sum by (job, route) (rate(http_requests_total[5m]))

// Agent-side aggregation: the application counts in memory and flushes one
// point per series per interval, instead of shipping one event per request.
// At 50,000 requests/second this is the difference between 50,000 writes/s and
// a few hundred.
const counters = new Map();
const MAX_SERIES = 5_000;        // the budget, enforced where series are created
let dropped = 0;

function inc(metric, labels, by = 1) {
  const key = metric + '|' + Object.keys(labels).sort()
    .map((k) => `${k}=${labels[k]}`).join(',');

  // Shed rather than grow. An unbounded label would otherwise take down the
  // ingest path, and the breakage would surface as a dead dashboard, not as a
  // bad label.
  if (!counters.has(key) && counters.size >= MAX_SERIES) { dropped++; return; }
  counters.set(key, (counters.get(key) ?? 0) + by);
}

// Route templates, never raw paths: /orders/8123 and /orders/8124 must share a
// series, or every order id becomes one.
inc('http_requests_total', { route: '/orders/:id', code_class: '2xx' });

setInterval(() => {
  // Export the shedding as its own metric and alert on it, otherwise silent
  // data loss looks exactly like a quiet service.
  inc('metrics_series_dropped_total', {}, dropped); dropped = 0;
  flush([...counters]);
  counters.clear();              // delta counters; the backend accumulates
}, 10_000);

// Distinct active users per minute. An exact answer means storing every user id
// per bucket; HyperLogLog answers in a fixed ~12 KB per key with a 0.81%
// standard error, which is well inside what a dashboard can show.
const minuteKey = (ts) => `dau:${Math.floor(ts / 60_000)}`;

await redis.pfAdd(minuteKey(Date.now()), String(userId));
const activeThisMinute = await redis.pfCount(minuteKey(Date.now()));

// Mergeability is the whole point: a day's unique users is the UNION of its
// minutes, not their sum, and summing would double-count every returning user.
await redis.pfMerge('dau:2026-10-01', last1440MinuteKeys);
const activeToday = await redis.pfCount('dau:2026-10-01');

// The same argument applies to latency across shards. A mergeable quantile
// sketch - t-digest, or DDSketch, which guarantees a bounded RELATIVE error on
// each quantile - can be computed per shard and merged centrally. Two shards'
// p99 values cannot be combined at all; their sketches can.
const global = shardSketches.reduce((acc, s) => acc.merge(s), newSketch());
console.log(global.quantile(0.99));