Bandwidth, Connections and Cost

// nginx upstream defaults: worker_processes 1, worker_connections 512.
// The 512 includes connections to proxied servers, not just clients - so as a
// reverse proxy each client costs TWO, one inbound and one to the upstream.
const nginx = { workerProcesses: 8, workerConnections: 16_384 };

const totalConnections = nginx.workerProcesses * nginx.workerConnections;  // 131_072
const maxProxiedClients = totalConnections / 2;                            // 65_536

// Those descriptors must actually exist. Raise the per-worker limit to at
// least twice worker_connections:
//
//   worker_processes      8;
//   worker_rlimit_nofile  32768;      # >= 2 * worker_connections
//   events { worker_connections 16384; }
//
// Miss worker_rlimit_nofile and nginx stops at the inherited soft limit -
// often 1024 - and logs "too many open files" while worker_connections
// claims capacity it cannot use.

// The same ceiling applies to any process holding sockets: a Node service
// with 20k open WebSockets needs its own ulimit -n raised, and each socket
// is one descriptor on top of files, logs and DB pool connections.

// A TCP connection is identified by the four-tuple
//   (source IP, source port, destination IP, destination port).
// From one pod IP to one upstream IP:port, the only variable is source port.
const EPHEMERAL_LOW  = 32_768;   // Linux default: net.ipv4.ip_local_port_range
const EPHEMERAL_HIGH = 60_999;
const maxOutbound = EPHEMERAL_HIGH - EPHEMERAL_LOW + 1;   // 28_232

// Worse in practice: a closed connection sits in TIME_WAIT for 2*MSL (60s on
// Linux) still holding its port. A service opening a fresh connection per
// request to one upstream burns ports at its request rate:
const portsHeld = (requestsPerSecond, timeWaitSeconds = 60) =>
  requestsPerSecond * timeWaitSeconds;

portsHeld(1_000);   // 60,000 ports needed - already over the 28,232 available
portsHeld(400);     // 24,000 - surviving, with no margin

// Fixes, in order of preference:
//  1. keep-alive connection pooling, so ports are reused instead of churned
//  2. more destination addresses (several upstream IPs multiplies the tuple)
//  3. widen ip_local_port_range - the widest safe setting, 1024-65535, is
//     64,512 ports, about 2.3x the default, and still only treats the symptom
// This is why an HTTP client with no connection pool fails suddenly at scale
// rather than degrading: you are not out of CPU, you are out of port numbers.

// Short-video feature: 1 million views a day, 15 MB per view.
const viewsPerDay = 1e6;
const bytesPerView = 15e6;
const gbPerMonth = (viewsPerDay * bytesPerView * 30) / 1e9;   // 450,000 GB = 450 TB

// Option A: serve from S3 in us-east-1.
//   storage, 50 TB of masters at $0.023/GB-month ......... $1,150
//   egress, 450 TB: [email protected] + [email protected] + [email protected] +
//                   [email protected] ........................... ~$26,300
//   total ................................................ ~$27,450/month

// Option B: serve from Cloudflare R2.
//   storage, 50 TB at $0.015/GB-month .................... $750
//   egress ............................................... $0
//   total ................................................ ~$750/month

// Both work. Both are affordable for a funded company. One is 36x the other,
// and no amount of application-level tuning closes a 36x gap - which is the
// whole point: this is a storage-provider decision taken at design time, and
// it is nearly impossible to reverse once terabytes have accumulated.
//
// Do not stop here. Check request pricing too: S3 Standard bills $0.005 per
// 1,000 PUTs and $0.0004 per 1,000 GETs, so a workload of billions of tiny
// objects can be dominated by request count rather than by bytes.