Scraping, Bots and Business Flow Abuse

// Layered, business-shaped controls. No single one is sufficient.
app.post('/api/checkout', auth, validate(checkoutSchema), async (req, res) => {
  const item = await db.items.findById(req.body.itemId);
  if (!item) return res.status(404).json({ error: 'not_found' });

  if (item.highDemand) {
    // 1. Per-customer limit — the control per-IP limiting cannot provide
    const bought = await db.orders.countFor(req.user.id, item.id, { since: '24h' });
    if (bought >= item.maxPerCustomer) {
      return res.status(429).json({ error: 'purchase_limit_reached',
        limit: item.maxPerCustomer });
    }

    // 2. Per payment instrument and per shipping address — 500 accounts
    //    sharing one card is the signal that per-account limits miss.
    const fingerprint = await paymentFingerprint(req.user.id);
    if (await db.orders.countByFingerprint(fingerprint, item.id, '24h')
          >= item.maxPerCustomer) {
      return res.status(429).json({ error: 'purchase_limit_reached' });
    }

    // 3. Account age and verification gates
    if (accountAgeDays(req.user) < 30 || !req.user.emailVerified) {
      return res.status(403).json({ error: 'account_not_eligible',
        detail: 'Accounts must be verified and at least 30 days old.' });
    }

    // 4. Cost asymmetry — seconds for a human, expensive for 10,000 bots
    if (!(await verifyProofOfWork(req.body.pow, req.user.id, item.id))) {
      return res.status(400).json({ error: 'proof_of_work_required',
        challenge: await issueChallenge(req.user.id, item.id) });
    }

    // 5. A randomised queue removes the advantage of being fastest, which is
    //    exactly what scalping software optimises for.
    const position = await queue.enter(item.id, req.user.id, { randomise: true });
    return res.status(202).json({
      status: 'queued', position,
      message: 'You are in the queue. Order is randomised, not first-come.',
    });
  }

  await createOrder(req.user.id, item);
  res.status(201).json({ ok: true });
});

// 6. Detect automation, and choose a response other than an immediate block —
//    blocking teaches the attacker what your detection looks like.
const signals = await automationSignals(req);
if (signals.score > 0.8) {
  await flagForReview(req.user.id, signals);
  await queue.deprioritise(req.user.id);        // quietly, without telling them
}

// Individually weak signals; useful in combination. Score, do not decide.
export async function automationSignals(req) {
  const signals = {};
  const userId = req.user?.id;

  // 1. Timing regularity — humans are irregular, scripts are not.
  const gaps = await recentRequestGaps(userId, 20);
  if (gaps.length > 10) {
    const mean = gaps.reduce((a, b) => a + b, 0) / gaps.length;
    const variance = gaps.reduce((a, b) => a + (b - mean) ** 2, 0) / gaps.length;
    // A coefficient of variation near zero means machine-perfect intervals.
    signals.timingRegularity = Math.sqrt(variance) / mean < 0.1 ? 1 : 0;
  }

  // 2. Navigation plausibility — did they load the page before submitting?
  const journey = await recentPaths(userId, 10);
  signals.noPageLoad = journey.every((p) => p.startsWith('/api/')) ? 1 : 0;

  // 3. Impossibly fast interaction — form loaded and submitted in 200ms
  const formAge = Date.now() - (await formIssuedAt(req.body.formToken) ?? 0);
  signals.tooFast = formAge < 1000 ? 1 : 0;
  signals.tooSlow = formAge > 3600_000 ? 1 : 0;      // a replayed old form

  // 4. Client fingerprint mismatches — a browser UA with no browser headers
  const ua = req.get('user-agent') ?? '';
  const claimsBrowser = /Mozilla|Chrome|Safari|Firefox/.test(ua);
  const hasBrowserHeaders = Boolean(req.get('accept-language') && req.get('sec-ch-ua'));
  signals.headerMismatch = claimsBrowser && !hasBrowserHeaders ? 1 : 0;

  // 5. Shared attributes across accounts
  signals.sharedDevice = await accountsSharingDevice(req.body.deviceId) > 5 ? 1 : 0;
  signals.sharedPayment = await accountsSharingPaymentMethod(userId) > 3 ? 1 : 0;

  // 6. Datacentre origin — legitimate consumers are rarely in AWS
  signals.datacentreIp = await isDatacentreAsn(req.ip) ? 1 : 0;

  const weights = {
    timingRegularity: 0.25, noPageLoad: 0.15, tooFast: 0.2, tooSlow: 0.05,
    headerMismatch: 0.1, sharedDevice: 0.15, sharedPayment: 0.15,
    datacentreIp: 0.1,
  };
  const score = Object.entries(signals)
    .reduce((sum, [k, v]) => sum + v * (weights[k] ?? 0), 0);

  return { score: Math.min(1, score), signals };
}

// Graduated response — and note that the highest tier still does not block.
// Silent deprioritisation is far more effective than a 403, because the
// attacker cannot tell it is happening and does not adapt.
export async function respondToAutomation(req, res, { score, signals }) {
  await audit.record({ event: 'automation.detected', userId: req.user?.id, score, signals });

  if (score > 0.9) return { action: 'queue_deprioritise' };
  if (score > 0.7) return { action: 'challenge' };          // CAPTCHA
  if (score > 0.5) return { action: 'delay', ms: 2000 };    // friction only
  return { action: 'allow' };
}

// A CAPTCHA costs the user attention; proof of work costs their CPU. For a
// high-demand flow, a two-second computation is invisible to one person and
// prohibitive at ten thousand concurrent attempts.
import { createHash, randomBytes } from 'crypto';

export async function issueChallenge(userId, resourceId, difficulty = 20) {
  const challenge = randomBytes(16).toString('hex');
  await redis.set(`pow:${userId}:${resourceId}`,
    JSON.stringify({ challenge, difficulty }), 'EX', 300);
  // difficulty = leading zero BITS. 20 ≈ 1M hashes ≈ 1-2s in a browser worker.
  return { challenge, difficulty, algorithm: 'sha256-leading-zero-bits' };
}

export async function verifyProofOfWork(proof, userId, resourceId) {
  const raw = await redis.get(`pow:${userId}:${resourceId}`);
  if (!raw) return false;
  const { challenge, difficulty } = JSON.parse(raw);

  if (typeof proof?.nonce !== 'string' || proof.nonce.length > 64) return false;

  const digest = createHash('sha256').update(challenge + proof.nonce).digest();

  // Count leading zero bits
  let zeroBits = 0;
  for (const byte of digest) {
    if (byte === 0) { zeroBits += 8; continue; }
    zeroBits += Math.clz32(byte) - 24;
    break;
  }
  if (zeroBits < difficulty) return false;

  await redis.del(`pow:${userId}:${resourceId}`);   // single use
  return true;
}

// Client side — in a worker, so the page does not freeze
// worker.js
//   onmessage = async ({ data: { challenge, difficulty } }) => {
//     for (let nonce = 0; ; nonce++) {
//       const digest = await crypto.subtle.digest('SHA-256',
//         new TextEncoder().encode(challenge + nonce));
//       if (leadingZeroBits(new Uint8Array(digest)) >= difficulty) {
//         postMessage({ nonce: String(nonce) });
//         return;
//       }
//     }
//   };

// ── The economics, which is the point ────────────────────────────────
// One purchase:      ~1.5 seconds of CPU. Unnoticed.
// 10,000 purchases:  ~4 CPU-hours, in parallel, per attempt round.
//                    Rented compute costs real money for an uncertain return.
//
// Tune difficulty by demand: raise it during a high-demand release, drop it to
// zero the rest of the time so ordinary customers never encounter it.