The P2P Engineer's
Field Manual Distributed networking, from Kademlia to production profiling

Goal

Implement XOR distance, a k-bucket routing table, and a simulated iterative lookup over an in-memory network of nodes. You will see the log N convergence with your own eyes by printing each hop.

Setup

# no dependencies — pure Node
mkdir kad-lab && cd kad-lab
node --version   # v18+ recommended

Step 1 — IDs and XOR distance

We use 16-bit IDs (a Buffer of 2 bytes) so the numbers stay small and printable. Real systems use 160 or 256 bits, but the math is identical.

// kad.js
const ID_BITS = 16;
const ID_BYTES = ID_BITS / 8;

function randomId() {
  const b = Buffer.allocUnsafe(ID_BYTES);
  for (let i = 0; i < ID_BYTES; i++) b[i] = (Math.random() * 256) | 0;
  return b;
}

// XOR distance as a BigInt so it is directly comparable
function distance(a, b) {
  let d = 0n;
  for (let i = 0; i < ID_BYTES; i++) d = (d << 8n) | BigInt(a[i] ^ b[i]);
  return d;
}

// index of the most-significant differing bit -> which bucket a peer belongs in
function bucketIndex(self, other) {
  for (let i = 0; i < ID_BYTES; i++) {
    const x = self[i] ^ other[i];
    if (x !== 0) return i * 8 + Math.clz32(x) - 24; // clz32 on a byte
  }
  return ID_BITS; // identical
}

Step 2 — the k-bucket routing table

const K = 4; // small k so buckets fill up visibly

class RoutingTable {
  constructor(selfId) {
    this.self = selfId;
    this.buckets = Array.from({ length: ID_BITS + 1 }, () => []);
  }
  add(node) {
    if (node.id.equals(this.self)) return;
    const b = bucketIndex(this.self, node.id);
    const bucket = this.buckets[b];
    if (bucket.find(n => n.id.equals(node.id))) return;
    if (bucket.length < K) bucket.push(node);
    // (real impl: ping least-recently-seen before evicting)
  }
  // return the n contacts closest to a target id
  closest(target, n = K) {
    return this.buckets.flat()
      .sort((x, y) => (distance(x.id, target) < distance(y.id, target) ? -1 : 1))
      .slice(0, n);
  }
}

Step 3 — a simulated network + iterative lookup

const ALPHA = 3;

class Node {
  constructor(id) { this.id = id; this.table = new RoutingTable(id); }
  // the FIND_NODE RPC: just answer with my closest known contacts
  findNode(target) { return this.table.closest(target); }
}

function buildNetwork(count) {
  const nodes = Array.from({ length: count }, () => new Node(randomId()));
  // seed each node's table with random acquaintances (bootstrap gossip)
  for (const n of nodes)
    for (let i = 0; i < 20; i++)
      n.table.add(nodes[(Math.random() * count) | 0]);
  return nodes;
}

// iterative lookup driven by one starting node
function lookup(start, target) {
  let shortlist = start.table.closest(target, ALPHA);
  const queried = new Set();
  let hops = 0, best = distance(start.id, target);

  while (true) {
    const batch = shortlist.filter(n => !queried.has(n.id.toString('hex'))).slice(0, ALPHA);
    if (batch.length === 0) break;
    hops++;
    let found = [];
    for (const peer of batch) {
      queried.add(peer.id.toString('hex'));
      found = found.concat(peer.findNode(target));
    }
    // merge, dedupe, keep K closest overall
    const seen = new Map();
    for (const n of shortlist.concat(found)) seen.set(n.id.toString('hex'), n);
    shortlist = [...seen.values()]
      .sort((x, y) => (distance(x.id, target) < distance(y.id, target) ? -1 : 1))
      .slice(0, K);
    const closest = distance(shortlist[0].id, target);
    if (closest >= best) break; // no improvement -> converged
    best = closest;
  }
  return { hops, winner: shortlist[0], best };
}

// --- run it ---
const N = 5000;
const net = buildNetwork(N);
const target = randomId();
const start = net[0];
const r = lookup(start, target);
console.log(`network size  : ${N}`);
console.log(`target        : ${target.toString('hex')}`);
console.log(`hops to find  : ${r.hops}`);
console.log(`log2(N)       : ${Math.log2(N).toFixed(1)}`);
console.log(`closest found : ${r.winner.id.toString('hex')}`);

Run & expected output

node kad.js
// network size  : 5000
// target        : 9c41
// hops to find  : 4         <- compare to log2(5000) ≈ 12.3
// closest found : 9c2f      <- shares the 9c.. prefix

Experiments to try

• Sweep N from 100 to 100000 and plot hops vs N. The curve is logarithmic — this is the headline property of every DHT.
• Set ALPHA = 1 and re-run. Lookups still work but take more rounds and are fragile if a queried node is "dead." Then try ALPHA = 8 and watch redundant queries climb.
• Reduce the bootstrap acquaintance count from 20 to 3. Lookups degrade or fail — the routing table is too sparse. This is why real DHTs run periodic bucket-refresh lookups.

Goal

Build a tiny append-only log that produces a Merkle root and lets a "reader" verify a single block with a proof — the conceptual core of Hypercore — using only Node's built-in crypto. Then run real Hypercore replication between two instances.

Part A — Merkle log from scratch (no dependencies)

// merkle-log.js
const crypto = require('crypto');
const H = (buf) => crypto.createHash('sha256').update(buf).digest();

class MerkleLog {
  constructor() { this.blocks = []; this.leaves = []; }

  append(data) {
    const buf = Buffer.isBuffer(data) ? data : Buffer.from(data);
    this.blocks.push(buf);
    this.leaves.push(H(buf));
    return this.blocks.length - 1; // index
  }

  // build the tree bottom-up; return the layered hash arrays
  tree() {
    let layer = this.leaves.slice();
    const layers = [layer];
    while (layer.length > 1) {
      const next = [];
      for (let i = 0; i < layer.length; i += 2) {
        const left = layer[i];
        const right = layer[i + 1] || left; // duplicate last if odd
        next.push(H(Buffer.concat([left, right])));
      }
      layers.push(next);
      layer = next;
    }
    return layers;
  }

  root() { const l = this.tree(); return l[l.length - 1][0]; }

  // proof = sibling hash at each level needed to recompute the root
  proof(index) {
    const layers = this.tree();
    const path = [];
    let idx = index;
    for (let lvl = 0; lvl < layers.length - 1; lvl++) {
      const isRight = idx % 2 === 1;
      const sibling = isRight ? idx - 1 : idx + 1;
      const layer = layers[lvl];
      path.push({ hash: (layer[sibling] || layer[idx]), isRight });
      idx = idx >> 1;
    }
    return path;
  }
}

// a reader who has ONLY the trusted root verifies one block
function verify(block, index, proof, trustedRoot) {
  let h = H(block);
  for (const step of proof) {
    h = step.isRight
      ? H(Buffer.concat([step.hash, h]))   // our node was on the right
      : H(Buffer.concat([h, step.hash]));
  }
  return h.equals(trustedRoot);
}

// --- demo ---
const log = new MerkleLog();
['genesis', 'block-1', 'block-2', 'block-3', 'block-4'].forEach(d => log.append(d));
const root = log.root();

const i = 3;
const ok = verify(log.blocks[i], i, log.proof(i), root);
console.log('root      :', root.toString('hex').slice(0, 16), '…');
console.log('verify #3 :', ok);

// tamper test: flip the block, proof must fail
const bad = Buffer.from('block-3-EVIL');
console.log('tampered  :', verify(bad, i, log.proof(i), root));

Expected output

node merkle-log.js
// root      : 2f6a... …
// verify #3 : true
// tampered  : false   <- the Merkle proof rejects altered data

Part B — the real thing: Hypercore replication

Now see it in production form. Modern Hypercore (v11+) is managed through a Corestore backed by a storage path; we replicate a log from a writer store to a key-only reader store over piped streams (the same API works over a real socket).

npm init -y
npm install hypercore corestore

// hc.js
const Corestore = require('corestore');
const os = require('os'), path = require('path'), fs = require('fs');

async function main() {
  // two independent stores on disk (Hypercore 11+ requires a storage dir)
  const dirA = fs.mkdtempSync(path.join(os.tmpdir(), 'hc-a-'));
  const dirB = fs.mkdtempSync(path.join(os.tmpdir(), 'hc-b-'));

  // writer core
  const store = new Corestore(dirA);
  const writer = store.get({ name: 'log' });
  await writer.ready();
  await writer.append([Buffer.from('hello'), Buffer.from('p2p'), Buffer.from('world')]);
  console.log('writer length:', writer.length,
              'key:', writer.key.toString('hex').slice(0, 12));

  // reader store: opens the core by PUBLIC KEY only — it has no data yet
  const store2 = new Corestore(dirB);
  const reader = store2.get({ key: writer.key });
  await reader.ready();

  // pipe the two replication streams together (stand-in for a socket)
  const s1 = store.replicate(true);
  const s2 = store2.replicate(false);
  s1.pipe(s2).pipe(s1);

  // reader fetches block 2 — verified against the signed root automatically
  const block = await reader.get(2);
  console.log('reader got block 2:', block.toString()); // "world"
  process.exit(0);
}
main();

Experiments

• In Part A, append 1000 blocks and confirm proof length grows like log₂(n), not n. That logarithmic proof size is the whole point.
• In Part B, call reader.get(0) before the writer appends (start the reader first), then append — observe that get resolves once the block arrives. Hypercore get means "wait until available."
• Inspect reader.length vs reader.contiguousLength to see the difference between "known to exist" and "actually downloaded."

Goal

Build a rendezvous server and two peers using Node's dgram (UDP) module. The server introduces the peers by their observed addresses; the peers then exchange packets directly. Running on localhost there's no real NAT, but the protocol logic — observe address, exchange, simultaneous send — is identical to the real thing.

Step 1 — the rendezvous server

// rendezvous.js — learns each peer's public address and introduces them
const dgram = require('dgram');
const server = dgram.createSocket('udp4');
const peers = {};

server.on('message', (msg, rinfo) => {
  const { name } = JSON.parse(msg);
  // rinfo is the address the SERVER sees — i.e. the public mapping (STUN)
  peers[name] = { address: rinfo.address, port: rinfo.port };
  console.log(`registered ${name} @ ${rinfo.address}:${rinfo.port}`);

  // once both peers are known, introduce them to each other
  const names = Object.keys(peers);
  if (names.length === 2) {
    const [a, b] = names;
    introduce(a, b); introduce(b, a);
  }
});

function introduce(to, about) {
  const payload = Buffer.from(JSON.stringify({ peer: about, addr: peers[about] }));
  server.send(payload, peers[to].port, peers[to].address);
}

server.bind(9000, () => console.log('rendezvous on :9000'));

Step 2 — a peer

// peer.js — usage: node peer.js <myName>
const dgram = require('dgram');
const myName = process.argv[2] || 'peerA';
const sock = dgram.createSocket('udp4');

const RDV = { port: 9000, host: '127.0.0.1' };
let partner = null;
let connected = false;

sock.on('message', (msg, rinfo) => {
  let data; try { data = JSON.parse(msg); } catch { data = { raw: msg.toString() }; }

  // (a) introduction from the rendezvous server
  if (data.addr) {
    partner = data.addr;
    console.log(`got partner ${data.peer} @ ${partner.address}:${partner.port}`);
    startPunching();
    return;
  }
  // (b) a direct packet from the partner — the hole is open!
  if (data.hello) {
    if (!connected) {
      connected = true;
      console.log(`✅ DIRECT connection established with ${data.hello}`);
    }
    return;
  }
});

// punch: fire packets straight at the partner's public mapping repeatedly
function startPunching() {
  const timer = setInterval(() => {
    if (connected) return clearInterval(timer);
    const pkt = Buffer.from(JSON.stringify({ hello: myName }));
    sock.send(pkt, partner.port, partner.address); // opens our hole + may pass theirs
  }, 200);
}

// register with the rendezvous server on startup
sock.send(Buffer.from(JSON.stringify({ name: myName })), RDV.port, RDV.host);
console.log(`${myName} registering…`);

Run it (three terminals)

# terminal 1
node rendezvous.js
# terminal 2
node peer.js peerA
# terminal 3
node peer.js peerB

# peerA output:
# peerA registering…
# got partner peerB @ 127.0.0.1:53412
# ✅ DIRECT connection established with peerB

What just happened (and what's different on real NAT)

• The server learned each peer's address as it observed it — that's the STUN principle. On real NAT this is the public mapping, not the private LAN address.
• Both peers fired at each other simultaneously; the repeated sends model the "first packets may be dropped while the hole opens" reality.
• Once connected, the server is never used again — true direct P2P.
• On symmetric NAT, the port the server observed would not match the port used peer-to-peer, so the punch fails and you'd fall back to a TURN-style relay. Try simulating this by having a peer send its punch packets from a different socket than the one it registered with — the introduction address no longer matches and the connection never forms.

Goal

Implement varint encode/decode, then a length-prefixed framer that correctly reassembles messages from an arbitrarily-chunked byte stream. Finish by encoding a structured message by hand. Then compare against real compact-encoding.

Step 1 — varints

// varint.js
function encodeVarint(value) {
  const bytes = [];
  while (value > 0x7f) {            // while more than 7 bits remain
    bytes.push((value & 0x7f) | 0x80); // low 7 bits + continuation flag
    value = Math.floor(value / 128);  // >>> 7 but safe past 32 bits
  }
  bytes.push(value & 0x7f);            // final byte, no continuation
  return Buffer.from(bytes);
}

function decodeVarint(buf, offset = 0) {
  let result = 0, shift = 0, pos = offset;
  while (true) {
    const byte = buf[pos++];
    result += (byte & 0x7f) * Math.pow(2, shift);
    if ((byte & 0x80) === 0) break;  // continuation bit clear -> done
    shift += 7;
  }
  return { value: result, bytesRead: pos - offset };
}

// sanity
for (const v of [0, 1, 127, 128, 300, 1000000]) {
  const enc = encodeVarint(v);
  const dec = decodeVarint(enc).value;
  console.log(v, '->', [...enc], `(${enc.length} bytes) ->`, dec);
}
module.exports = { encodeVarint, decodeVarint };

node varint.js
// 0 -> [0] (1 bytes) -> 0
// 127 -> [127] (1 bytes) -> 127
// 128 -> [128,1] (2 bytes) -> 128
// 300 -> [172,2] (2 bytes) -> 300
// 1000000 -> [192,132,61] (3 bytes) -> 1000000   <- vs 15 bytes as JSON

Step 2 — a correct length-prefixed framer

This is the part interviewers love, because the naive version is subtly broken. The framer buffers raw bytes and only emits a message when a complete frame is present — handling split and coalesced reads.

// framer.js
const { encodeVarint, decodeVarint } = require('./varint');
const EventEmitter = require('events');

function frame(payload) {
  const len = encodeVarint(payload.length);
  return Buffer.concat([len, payload]); // [varint length][payload bytes]
}

class Decoder extends EventEmitter {
  constructor() { super(); this.buf = Buffer.alloc(0); }
  push(chunk) {
    this.buf = Buffer.concat([this.buf, chunk]); // accumulate
    while (this.buf.length > 0) {
      let header;
      try { header = decodeVarint(this.buf, 0); }
      catch { return; }                       // length not fully arrived
      const total = header.bytesRead + header.value;
      if (this.buf.length < total) return;     // full frame not here yet — wait
      const payload = this.buf.subarray(header.bytesRead, total);
      this.emit('message', Buffer.from(payload));
      this.buf = this.buf.subarray(total);   // keep the remainder
    }
  }
}
module.exports = { frame, Decoder };

Step 3 — prove it survives evil chunking

// test-framer.js
const { frame, Decoder } = require('./framer');

const messages = ['hello', 'a', 'a much longer message here', 'last']
  .map(s => Buffer.from(s));
const stream = Buffer.concat(messages.map(frame)); // all frames glued together

const dec = new Decoder();
const out = [];
dec.on('message', m => out.push(m.toString()));

// feed the stream in deliberately awful 3-byte chunks
for (let i = 0; i < stream.length; i += 3)
  dec.push(stream.subarray(i, i + 3));

console.log(out);
// [ 'hello', 'a', 'a much longer message here', 'last' ]  ✅ reassembled perfectly

Step 4 — encode a structured message by hand

A "peer announce" message: a 1-byte type tag, a varint topic length + topic bytes, and a fixed 6-byte address (4-byte IPv4 + 2-byte port).

// announce.js
const { encodeVarint, decodeVarint } = require('./varint');
const TYPE_ANNOUNCE = 0x01;

function encodeAnnounce({ topic, ip, port }) {
  const topicBuf = Buffer.from(topic);
  const addr = Buffer.alloc(6);
  ip.split('.').forEach((o, i) => addr[i] = Number(o));
  addr.writeUInt16BE(port, 4); // big-endian = network byte order
  return Buffer.concat([
    Buffer.from([TYPE_ANNOUNCE]),
    encodeVarint(topicBuf.length), topicBuf,
    addr
  ]);
}

function decodeAnnounce(buf) {
  let off = 0;
  const type = buf[off++];
  const { value: tlen, bytesRead } = decodeVarint(buf, off); off += bytesRead;
  const topic = buf.subarray(off, off + tlen).toString(); off += tlen;
  const ip = [...buf.subarray(off, off + 4)].join('.'); off += 4;
  const port = buf.readUInt16BE(off);
  return { type, topic, ip, port };
}

const wire = encodeAnnounce({ topic: 'chat-room-42', ip: '192.168.1.50', port: 9000 });
console.log('bytes :', wire.length, [...wire]);
console.log('decoded:', decodeAnnounce(wire));

Step 5 — the real thing: compact-encoding

npm install compact-encoding

// ce.js
const c = require('compact-encoding');

// compose a struct codec from primitives
const announce = {
  preencode(state, m) {     // pass 1: measure
    c.string.preencode(state, m.topic);
    c.uint16.preencode(state, m.port);
  },
  encode(state, m) {        // pass 2: write
    c.string.encode(state, m.topic);
    c.uint16.encode(state, m.port);
  },
  decode(state) {
    return { topic: c.string.decode(state), port: c.uint16.decode(state) };
  }
};

const buf = c.encode(announce, { topic: 'chat-room-42', port: 9000 });
console.log('compact bytes:', buf.length, [...buf]);
console.log('decoded      :', c.decode(announce, buf));

Notice the two-pass design: preencode computes the exact size so encode can write into a single pre-allocated buffer with no resizing — an allocation optimization you'll revisit on Day 6.

Experiments

• Change the test framer chunk size to 1 byte, then to 1000. Both must produce identical output — that's the definition of correct framing.
• Compare your hand-rolled announce size against the equivalent JSON string. Measure the ratio.
• Add a malformed frame (a length prefix larger than the data that ever arrives) and confirm the decoder simply waits rather than crashing or emitting garbage.

Goal

Use real, production primitives to (1) sign and verify like a Hypercore writer, (2) perform an X25519 key exchange and confirm both sides derive the same secret, (3) AEAD-encrypt with tamper detection, and (4) sketch the shape of a Noise-style handshake.

Setup

npm install sodium-native

sodium-native is libsodium bindings — the exact crypto layer under Hypercore. (If it fails to build in your environment, Node's built-in crypto also supports Ed25519/X25519; the concepts are identical.)

Step 1 — Ed25519 sign & verify (the "single writer" proof)

// sign.js
const sodium = require('sodium-native');

// generate a signing keypair
const pk = Buffer.alloc(sodium.crypto_sign_PUBLICKEYBYTES); // 32
const sk = Buffer.alloc(sodium.crypto_sign_SECRETKEYBYTES); // 64
sodium.crypto_sign_keypair(pk, sk);

const message = Buffer.from('append: block #42');
const sig = Buffer.alloc(sodium.crypto_sign_BYTES);   // 64
sodium.crypto_sign_detached(sig, message, sk);

const ok = sodium.crypto_sign_verify_detached(sig, message, pk);
console.log('signature valid:', ok);

// tamper -> verification fails (integrity + authenticity)
const forged = Buffer.from('append: block #43');
console.log('forged valid  :', sodium.crypto_sign_verify_detached(sig, forged, pk));

Step 2 — X25519 key exchange (two strangers, one secret)

// dh.js
const sodium = require('sodium-native');

function keypair() {
  const pk = Buffer.alloc(sodium.crypto_kx_PUBLICKEYBYTES);
  const sk = Buffer.alloc(sodium.crypto_kx_SECRETKEYBYTES);
  sodium.crypto_kx_keypair(pk, sk);
  return { pk, sk };
}

const alice = keypair();
const bob = keypair();

// each derives session keys from own secret + peer's public key
const aRx = Buffer.alloc(sodium.crypto_kx_SESSIONKEYBYTES);
const aTx = Buffer.alloc(sodium.crypto_kx_SESSIONKEYBYTES);
const bRx = Buffer.alloc(sodium.crypto_kx_SESSIONKEYBYTES);
const bTx = Buffer.alloc(sodium.crypto_kx_SESSIONKEYBYTES);

sodium.crypto_kx_client_session_keys(aRx, aTx, alice.pk, alice.sk, bob.pk);
sodium.crypto_kx_server_session_keys(bRx, bTx, bob.pk, bob.sk, alice.pk);

// alice's TX key == bob's RX key  -> shared secret agreed over a public channel
console.log('keys match:', aTx.equals(bRx) && bTx.equals(aRx));

Step 3 — AEAD encryption with tamper detection

// aead.js
const sodium = require('sodium-native');

const key = Buffer.alloc(sodium.crypto_aead_chacha20poly1305_ietf_KEYBYTES);
sodium.randombytes_buf(key);
const nonce = Buffer.alloc(sodium.crypto_aead_chacha20poly1305_ietf_NPUBBYTES);
sodium.randombytes_buf(nonce); // MUST be unique per message under a key

const plaintext = Buffer.from('secret peer payload');
const ad = Buffer.from('msg-type:3'); // associated data: authenticated, not encrypted
const ct = Buffer.alloc(plaintext.length + sodium.crypto_aead_chacha20poly1305_ietf_ABYTES);

sodium.crypto_aead_chacha20poly1305_ietf_encrypt(ct, plaintext, ad, null, nonce, key);

const out = Buffer.alloc(ct.length - sodium.crypto_aead_chacha20poly1305_ietf_ABYTES);
sodium.crypto_aead_chacha20poly1305_ietf_decrypt(out, null, ct, ad, nonce, key);
console.log('decrypted:', out.toString());

// flip one ciphertext byte -> decrypt throws (the Poly1305 tag fails)
ct[0] ^= 0xff;
try {
  sodium.crypto_aead_chacha20poly1305_ietf_decrypt(out, null, ct, ad, nonce, key);
  console.log('tamper undetected (BAD)');
} catch {
  console.log('tamper detected: decryption rejected ✅');
}

Step 4 — sketch a Noise-style handshake

This is a conceptual 2-message exchange (not a spec-compliant Noise impl) to show how DH + hashing compose into a session. Each side mixes ephemeral DH output into a running hash to derive a shared key.

// mini-noise.js — illustrative only; use the 'noise-protocol' / hyperswarm libs for real
const sodium = require('sodium-native');
const hash = (b) => { const o = Buffer.alloc(32); sodium.crypto_generichash(o, b); return o; };

function eph() {
  const pk = Buffer.alloc(32), sk = Buffer.alloc(32);
  sodium.crypto_box_keypair(pk, sk); return { pk, sk };
}
function dh(sk, pk) {
  const out = Buffer.alloc(32);
  sodium.crypto_scalarmult(out, sk, pk); return out;
}

const i = eph(), r = eph();        // initiator + responder ephemerals
// -> e   : initiator sends its ephemeral public key
// <- e   : responder sends its ephemeral public key
// both compute the same DH(e_i, e_r) and mix it into a session key
const kI = hash(dh(i.sk, r.pk));
const kR = hash(dh(r.sk, i.pk));
console.log('handshake keys match:', kI.equals(kR));
console.log('(real Noise also mixes static keys + transcript hash for auth)');

Experiments

• In Step 3, reuse the same nonce for two messages. Crypto won't error, but you've just broken the cipher's security — this is why nonce management is the #1 AEAD footgun. Note how real protocols derive per-message nonces from a counter.
• In Step 4, generate fresh ephemerals each run and confirm the derived key changes every time — that's forward secrecy in action.
• Look up the noise-protocol npm package and identify which functions correspond to the XX vs IK patterns.

Goal

Measure three real performance effects: (1) the Buffer.concat O(n²) trap vs a pre-sized buffer, (2) allocUnsafe vs alloc in a hot loop, and (3) backpressure on a TCP socket. You'll produce numbers, not just theory.

Step 1 — the accumulating-concat trap

// concat-trap.js — why naive reassembly is O(n^2)
const { performance } = require('perf_hooks');
const CHUNKS = 20000;
const chunk = Buffer.allocUnsafe(64);

// BAD: concat the whole growing buffer on every chunk
let t = performance.now();
let acc = Buffer.alloc(0);
for (let i = 0; i < CHUNKS; i++) acc = Buffer.concat([acc, chunk]);
console.log('concat-each-time :', (performance.now() - t).toFixed(1), 'ms');

// GOOD: collect references, concat ONCE at the end
t = performance.now();
const parts = [];
for (let i = 0; i < CHUNKS; i++) parts.push(chunk);
const joined = Buffer.concat(parts);
console.log('collect-then-join:', (performance.now() - t).toFixed(1), 'ms');

// BEST when total size known: one allocation, copy into it
t = performance.now();
const dst = Buffer.allocUnsafe(CHUNKS * 64);
for (let i = 0; i < CHUNKS; i++) chunk.copy(dst, i * 64);
console.log('presized-copy    :', (performance.now() - t).toFixed(1), 'ms');

node concat-trap.js
// concat-each-time : 3380.0 ms    <- quadratic; explodes with size
// collect-then-join:    2.3 ms     <- linear
// presized-copy    :    6.9 ms     <- linear, one allocation

The exact numbers vary by machine, but the shape is the lesson: the naive version is hundreds to thousands of times slower and gets worse as the buffer grows. This is the framing decoder from Day 4 done wrong.

Step 2 — allocUnsafe vs alloc in a hot path

// alloc-bench.js
const { performance } = require('perf_hooks');
const N = 2_000_000, SIZE = 256;

let t = performance.now();
for (let i = 0; i < N; i++) { const b = Buffer.alloc(SIZE); b[0] = 1; }
console.log('alloc      :', (performance.now() - t).toFixed(0), 'ms');

t = performance.now();
for (let i = 0; i < N; i++) { const b = Buffer.allocUnsafe(SIZE); b[0] = 1; }
console.log('allocUnsafe:', (performance.now() - t).toFixed(0), 'ms');

// BEST: one reusable buffer, no per-iteration allocation at all
t = performance.now();
const reusable = Buffer.allocUnsafe(SIZE);
for (let i = 0; i < N; i++) { reusable[0] = 1; }
console.log('reused     :', (performance.now() - t).toFixed(0), 'ms');

Zero-filling (alloc) costs measurable time at scale, and not allocating at all wins biggest. In a real reader you keep one read buffer and parse out of it.

Step 3 — backpressure on a real socket

// backpressure.js — a server that floods, a client that reads slowly
const net = require('net');

const server = net.createServer((socket) => {
  const payload = Buffer.allocUnsafe(64 * 1024); // 64 KB
  let sent = 0, blocked = 0;

  function pump() {
    while (sent < 2000) {
      const ok = socket.write(payload); // returns false when buffer is full
      sent++;
      if (!ok) {                       // BACKPRESSURE: stop and wait for drain
        blocked++;
        socket.once('drain', pump);     // resume only when flushed
        return;
      }
    }
    console.log(`done: sent ${sent} chunks, paused ${blocked} times for drain`);
    socket.end();
  }
  pump();
});

server.listen(8124, () => {
  const client = net.connect(8124);
  let received = 0;
  client.on('data', (d) => {
    received += d.length;
    client.pause();                       // simulate a slow consumer
    setTimeout(() => client.resume(), 1);
  });
  client.on('end', () => {
    console.log(`client received ${(received / 1024 / 1024).toFixed(1)} MB`);
    server.close();
  });
});

node backpressure.js
// done: sent 2000 chunks, paused 80+ times for drain
// client received 125.0 MB

The "paused for drain" count is the system protecting itself. Delete the if (!ok) branch and keep writing regardless: memory usage climbs without bound because Node queues everything you hand it. That is the bug behind most "my P2P node leaks memory under load" reports.

Experiments

• In Step 1, raise CHUNKS to 40000 and watch the concat time roughly quadruple (4×), confirming O(n²) — while the other two only double.
• In Step 3, log socket.writableLength each time write returns false to see the high-water mark Node is enforcing.
• Add const h = require('perf_hooks').monitorEventLoopDelay(); h.enable(); around a blocking loop and read h.max to quantify a stall in milliseconds.

Goal

Generate a CPU profile, read it, fix the hot spot, and confirm the win — the full measure-fix-measure loop.

// slow.js — a "message processor" with a hidden hot spot
const { performance } = require('perf_hooks');

function processMessages(n) {
  let log = '';
  for (let i = 0; i < n; i++) {
    log += JSON.stringify({ id: i, ok: true }) + '\n'; // string += in a loop = O(n^2)-ish
  }
  return log.length;
}

const t = performance.now();
console.log('len:', processMessages(200000));
console.log('took:', (performance.now() - t).toFixed(0), 'ms');

# profile it
node --cpu-prof --cpu-prof-name=slow.cpuprofile slow.js
# open slow.cpuprofile in Chrome DevTools (Performance tab) -> see the wide frame

# or the zero-dependency route:
node --prof slow.js
node --prof-process isolate-*.log > profile.txt
# inspect the "Summary" and "Bottom up (heavy) profile" sections

The profile fingers string concatenation / stringify as the wide frame. The fix is the same shape as Day 6: collect parts, join once.

// fast.js — same output, collect-then-join
function processMessages(n) {
  const parts = [];
  for (let i = 0; i < n; i++) parts.push('{"id":' + i + ',"ok":true}');
  return parts.join('\n').length;
}

Re-profile and confirm the plateau shrank. That's the loop — never trust a fix you didn't re-measure.

What it integrates

• Day 1/3: a registry/rendezvous resolves a topic → peer address (DHT discovery, condensed).
• Day 5: X25519 key agreement → a shared AEAD key (the handshake).
• Day 4: length-prefixed varint framing over a UDP/stream boundary.
• Day 5: ChaCha20-Poly1305 AEAD on every message, with tamper rejection.
• Day 6: bounded per-event work, slice-not-copy framing, no concat-in-loop.

The capstone (run with node capstone.js)

// capstone.js — discovery + handshake + framed encrypted messaging, in-process
const sodium = require('sodium-native');
const EventEmitter = require('events');

// ---------- Day 4: varint + length-prefix framing ----------
function encodeVarint(v) {
  const b = [];
  while (v > 0x7f) { b.push((v & 0x7f) | 0x80); v = Math.floor(v / 128); }
  b.push(v & 0x7f); return Buffer.from(b);
}
function decodeVarint(buf, off = 0) {
  let r = 0, s = 0, p = off;
  while (true) {
    const x = buf[p++]; if (x === undefined) throw new Error('incomplete');
    r += (x & 0x7f) * Math.pow(2, s); if ((x & 0x80) === 0) break; s += 7;
  }
  return { value: r, bytesRead: p - off };
}
class Framer extends EventEmitter {
  constructor() { super(); this.buf = Buffer.alloc(0); }
  push(chunk) {
    this.buf = Buffer.concat([this.buf, chunk]);
    while (this.buf.length) {
      let h; try { h = decodeVarint(this.buf); } catch { return; }
      const total = h.bytesRead + h.value;
      if (this.buf.length < total) return;
      this.emit('message', Buffer.from(this.buf.subarray(h.bytesRead, total)));
      this.buf = this.buf.subarray(total);
    }
  }
}
const frame = (p) => Buffer.concat([encodeVarint(p.length), p]);

// ---------- Day 5: handshake + AEAD ----------
function kxKeypair() {
  const pk = Buffer.alloc(sodium.crypto_kx_PUBLICKEYBYTES);
  const sk = Buffer.alloc(sodium.crypto_kx_SECRETKEYBYTES);
  sodium.crypto_kx_keypair(pk, sk); return { pk, sk };
}
const A_BYTES = sodium.crypto_aead_chacha20poly1305_ietf_ABYTES;
const N_BYTES = sodium.crypto_aead_chacha20poly1305_ietf_NPUBBYTES;
function seal(key, counter, msg) {
  const nonce = Buffer.alloc(N_BYTES); nonce.writeUInt32LE(counter, 0); // per-message nonce
  const ct = Buffer.alloc(msg.length + A_BYTES);
  sodium.crypto_aead_chacha20poly1305_ietf_encrypt(ct, msg, null, null, nonce, key);
  return ct;
}
function open(key, counter, ct) {
  const nonce = Buffer.alloc(N_BYTES); nonce.writeUInt32LE(counter, 0);
  const out = Buffer.alloc(ct.length - A_BYTES);
  sodium.crypto_aead_chacha20poly1305_ietf_decrypt(out, null, ct, null, nonce, key);
  return out;
}

// ---------- Day 1/3: a trivial discovery registry ----------
const registry = new Map(); // topic -> { inbox: Framer, key, counter }

function makePeer(name, topic) {
  const kp = kxKeypair();
  const inbox = new Framer();
  const peer = { name, topic, kp, inbox, key: null, rxCounter: 0, txCounter: 0, partner: null };
  registry.set(name, peer);
  return peer;
}

// "send" = hand framed bytes to the partner's inbox (stands in for the wire)
function send(from, payload) {
  const ct = seal(from.key, from.txCounter, payload);
  from.txCounter++;
  from.partner.inbox.push(frame(ct)); // length-prefixed encrypted frame
}

// ---------- wire it together ----------
const alice = makePeer('alice', 'chat-42');
const bob = makePeer('bob', 'chat-42');
alice.partner = bob; bob.partner = alice;

// discovery + handshake: derive a shared AEAD key from kx
function handshake(client, server) {
  const cRx = Buffer.alloc(32), cTx = Buffer.alloc(32);
  const sRx = Buffer.alloc(32), sTx = Buffer.alloc(32);
  sodium.crypto_kx_client_session_keys(cRx, cTx, client.kp.pk, client.kp.sk, server.kp.pk);
  sodium.crypto_kx_server_session_keys(sRx, sTx, server.kp.pk, server.kp.sk, client.kp.pk);
  // use one direction's key for both here (demo); real protocols key each direction
  client.key = cTx; server.key = sRx; // cTx === sRx
  return cTx.equals(sRx);
}
console.log('handshake ok:', handshake(alice, bob));

// bob decrypts and verifies each framed message he receives
bob.inbox.on('message', (ct) => {
  try {
    const msg = open(bob.key, bob.rxCounter, ct);
    bob.rxCounter++;
    console.log(`bob received: "${msg.toString()}"`);
  } catch {
    console.log('bob: message failed AEAD verification — dropped');
  }
});

// alice sends three encrypted, framed messages
send(alice, Buffer.from('hello bob'));
send(alice, Buffer.from('this channel is encrypted'));
send(alice, Buffer.from('and every frame is verified'));

// tamper demo: flip a byte in a sealed frame and confirm it's rejected
const good = seal(alice.key, 99, Buffer.from('evil'));
good[2] ^= 0xff;
bob.rxCounter = 99;
bob.inbox.push(frame(good)); // -> "failed AEAD verification — dropped"

Expected output

node capstone.js
// handshake ok: true
// bob received: "hello bob"
// bob received: "this channel is encrypted"
// bob received: "and every frame is verified"
// bob: message failed AEAD verification — dropped

From this skeleton to the real Holepunch stack

Each hand-rolled piece maps to a production module: the registry → hyperdht; the kx handshake → the Noise/UDX transport inside hyperswarm; the framer → compact-encoding + the stream protocol; the append-and-verify pattern → hypercore. Rebuild the capstone using hyperswarm for real: join a topic, and on 'connection' you get an already-encrypted, already-framed stream — every layer you just built by hand, productionized.

Experiments

• Replace the in-process registry with the real dgram rendezvous from Day 3 so the two peers run as separate processes.
• Swap the hand framer for compact-encoding structs (Day 4) and confirm identical behavior.
• Profile the capstone under 100k messages with --cpu-prof; confirm no accidental concat-in-loop and that AEAD dominates CPU (as it should).