Timeouts, Retries and Backoff

// SHORTHAND. AbortSignal.timeout(ms) -- Node v17.3.0 (backported 16.14.0),
// and in browsers. Aborts with a DOMException whose name is 'TimeoutError'.
const res = await fetch(url, { signal: AbortSignal.timeout(2000) });

// LONGHAND. Exactly the same thing, and what you still write when you need
// a handle on the controller, or on runtimes older than the above.
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), 2000);
try {
  const res2 = await fetch(url, { signal: controller.signal });
} finally {
  clearTimeout(timer); // the shorthand version does this for you; forgetting
                       // it keeps the event loop alive for the full 2s
}

// Combining a timeout with a caller's own cancellation.
// SHORTHAND: AbortSignal.any -- Node v20.3.0 (backported 18.17.0).
const signal = AbortSignal.any([req.signal, AbortSignal.timeout(2000)]);

// LONGHAND, for older runtimes: forward whichever fires first.
const merged = new AbortController();
const forward = () => merged.abort();
req.signal.addEventListener('abort', forward, { once: true });
const t = setTimeout(forward, 2000);

// The edge sets ONE budget. Every hop passes on what is left of it, so a
// three-service chain cannot spend 3 x 250ms against a 300ms promise.
const DEADLINE_HEADER = 'x-deadline-ms'; // gRPC does this as 'grpc-timeout'

// Middleware at the edge, and at every service in the chain.
function deadlines(defaultBudgetMs) {
  return (req, res, next) => {
    const inherited = Number(req.get(DEADLINE_HEADER));
    // Honour an inbound budget, and never extend it: a downstream service
    // must not outlive the client that is waiting for it.
    const budget = Number.isFinite(inherited)
      ? Math.min(inherited, defaultBudgetMs)
      : defaultBudgetMs;
    req.deadline = Date.now() + budget;
    next();
  };
}

async function callDownstream(req, path, body) {
  const remaining = req.deadline - Date.now();
  // Fail fast rather than starting work that cannot finish in time. In a
  // deep call graph this is where most of the saved capacity comes from.
  if (remaining <= 5) {
    throw Object.assign(new Error('deadline exceeded'), { status: 504 });
  }
  return fetch(`http://internal${path}`, {
    method: 'POST',
    body: JSON.stringify(body),
    headers: {
      'content-type': 'application/json',
      [DEADLINE_HEADER]: String(remaining), // pass the remainder down
    },
    signal: AbortSignal.timeout(remaining),
  });
}

// Timeouts are set from data. These are bucket counts straight out of a
// Prometheus-style histogram for one dependency over a quiet week.
const buckets = [
  { leMs: 10, count: 412_000 },
  { leMs: 25, count: 905_000 },
  { leMs: 50, count: 980_000 },
  { leMs: 100, count: 994_000 },
  { leMs: 250, count: 998_600 },
  { leMs: 500, count: 999_400 },
  { leMs: 1000, count: 999_900 },
  { leMs: Infinity, count: 1_000_000 },
];

// Cumulative buckets, so the quantile is the first bucket past the rank.
function quantile(q) {
  const target = buckets.at(-1).count * q;
  const hit = buckets.find((b) => b.count >= target);
  return hit.leMs;
}

console.log(quantile(0.5));   // 25ms   <- the average lives near here
console.log(quantile(0.99));  // 100ms
console.log(quantile(0.999)); // 500ms  <- rank 999,000 is past the 250ms bucket

// A timeout at the mean would abandon a third of healthy requests. Sit just
// above p99 so the timeout means "abnormal", not "slightly unlucky".
const timeoutMs = Math.ceil(quantile(0.99) * 1.5); // 150ms

// Then sanity-check the upper bound: with 2 retries and full jitter capped
// at 20s, worst-case wall time is the sum, which must fit the deadline.
const worstCase = timeoutMs * 3 + 50 + 100; // attempts + both backoff caps
console.log(worstCase);  // 600ms -- fine under a 1s deadline, not under 500ms