SLIs, SLOs and Error Budgets

# Two recording rules keep the expressions readable and let the ratio be
# computed once. 'code' here is the HTTP status class label.
groups:
  - name: slo-checkout
    rules:
      - record: job:sli_error_ratio:rate1h
        expr: |
          sum(rate(http_requests_total{job="checkout",code=~"5.."}[1h]))
            / sum(rate(http_requests_total{job="checkout"}[1h]))
      - record: job:sli_error_ratio:rate5m
        expr: |
          sum(rate(http_requests_total{job="checkout",code=~"5.."}[5m]))
            / sum(rate(http_requests_total{job="checkout"}[5m]))

      # 14.4 x the budgeted 0.1% error rate = 1.44%. Both windows must breach:
      # the long one for precision, the short one so the alert also RESOLVES
      # quickly once the burn stops.
      - alert: CheckoutErrorBudgetBurningFast
        expr: |
          job:sli_error_ratio:rate1h > (14.4 * 0.001)
            and
          job:sli_error_ratio:rate5m > (14.4 * 0.001)
        labels:
          severity: page
        annotations:
          summary: "Checkout is burning 2% of the 30-day error budget per hour"

      # Latency SLI from a histogram, not from an averaged quantile: count the
      # requests inside the bound and divide. 'le' is the bucket upper bound.
      - record: job:sli_latency_ratio:rate30m
        expr: |
          sum(rate(http_request_duration_seconds_bucket{job="checkout",le="0.3"}[30m]))
            / sum(rate(http_request_duration_seconds_count{job="checkout"}[30m]))

// Most SLIs are wrong in the same way: they count what the server did, not
// what the user experienced. These are the corrections that matter.
function classify(event) {
  // 1. Not every error is yours. A 400 is the client's bug and must not
  //    spend your budget; a 429 you issued deliberately must not either.
  if (event.status === 400 || event.status === 404 || event.status === 429) {
    return 'excluded';
  }

  // 2. A slow success is a failure. Fold the latency objective into the
  //    availability SLI so there is one number, not two that disagree.
  const LATENCY_OBJECTIVE_MS = 300;
  if (event.status < 500 && event.durationMs <= LATENCY_OBJECTIVE_MS) return 'good';
  return 'bad';
}

// 3. Weight by user journey, not by URL volume. A health check polled every
// second will otherwise outvote checkout in your availability number.
const JOURNEYS = {
  'POST /checkout': { weight: 10, objectiveMs: 1000 },
  'GET /product/:id': { weight: 3, objectiveMs: 300 },
  'GET /healthz': { weight: 0, objectiveMs: 50 },  // excluded from the SLI
};

// 4. Measure where the user is. Server-side metrics cannot see DNS failures,
// TLS errors, a saturated load balancer or a bad CDN POP, so pair them
// with real-user timings reported from the browser.
function reportFromBrowser() {
  const nav = performance.getEntriesByType('navigation')[0];
  navigator.sendBeacon('/rum', JSON.stringify({
    ttfbMs: nav.responseStart - nav.requestStart,
    // The number users actually feel, which no server-side timer records.
    loadMs: nav.loadEventEnd - nav.startTime,
  }));
}

// The budget is only useful if spending it changes a decision in advance.
// This is the policy as code, which is what a release gate consults.
const POLICY = {
  // Agreed by the team and whoever owns the roadmap, BEFORE an incident.
  thresholds: [
    { remaining: 0.5, action: 'ship freely' },
    { remaining: 0.25, action: 'ship, and spend one engineer on reliability' },
    { remaining: 0.0, action: 'feature freeze: reliability work and rollbacks only' },
    { remaining: -Infinity, action: 'freeze, plus a written review before unfreezing' },
  ],
  // Exemptions, so the policy survives contact with reality.
  exempt: ['security patches', 'changes that reduce the burn rate'],
};

function gate({ remainingFraction, changeKind }) {
  if (POLICY.exempt.includes(changeKind)) return { allow: true, reason: 'exempt' };
  const rule = POLICY.thresholds.find((t) => remainingFraction >= t.remaining);
  return { allow: rule.remaining > 0, action: rule.action };
}

console.log(gate({ remainingFraction: 0.62, changeKind: 'feature' }));
// { allow: true, action: 'ship freely' }
console.log(gate({ remainingFraction: -0.18, changeKind: 'feature' }));
// { allow: false, action: 'freeze, plus a written review before unfreezing' }

// A budget that is never spent is an SLO set too low: it is buying reliability
// nobody asked for. Treat a consistently full budget as a signal to tighten
// the objective or to ship faster, not as a success.