Hermeticity & Reproducible Builds - Sandboxes, Pinned Toolchains, the Nix Store & Bit-for-Bit Output

// A simplified model of the Nix store (NOT Nix's exact hashing scheme).
// Input-addressed path = hash(builder, args, env, input paths)  -> known before building
// Content-addressed path = hash(output bytes)                    -> known only after building
// We rebuild "zlib" with a patched compiler that emits identical bytes and watch
// which downstream paths move. Then we compute a closure: everything a path references.

function hash(s: string): string {
  // FNV-1a, 32-bit, printed as 8 hex chars; real Nix uses SHA-256 rendered in Nix32 (base-32).
  let h = 0x811c9dc5;
  for (let i = 0; i < s.length; i++) { h ^= s.charCodeAt(i); h = Math.imul(h, 0x01000193) >>> 0; }
  return h.toString(16).padStart(8, "0");
}

type Drv = { name: string; builder: string; args: string; inputs: string[]; produce: (ins: string[]) => string };

function realise(drvs: Drv[], mode: "input" | "content") {
  const path: Record<string, string> = {};
  const bytes: Record<string, string> = {};
  for (const d of drvs) {                          // drvs are listed in dependency order
    const ins = d.inputs.map((i) => path[i]);
    const out = d.produce(d.inputs.map((i) => bytes[i]));
    const digest = mode === "input"
      ? hash([d.builder, d.args, ...ins].join("|"))   // describes HOW it is built
      : hash(out);                                    // describes WHAT came out
    path[d.name] = `/nix/store/${digest}-${d.name}`;
    bytes[d.name] = out;
  }
  return path;
}

const graph = (zlibCompiler: string): Drv[] => [
  // A compiler bug-fix release that does not change zlib's machine code: same bytes either way.
  { name: "zlib-1.3", builder: zlibCompiler, args: "-O2", inputs: [], produce: () => "libz.so v1.3" },
  { name: "openssl-3.3", builder: "gcc-13", args: "-O2", inputs: ["zlib-1.3"], produce: (i) => "libssl linked " + i[0].length },
  { name: "curl-8.9", builder: "gcc-13", args: "-O2", inputs: ["openssl-3.3", "zlib-1.3"], produce: (i) => "curl " + i.join("+").length },
];

for (const mode of ["input", "content"] as const) {
  const before = realise(graph("gcc-13.2"), mode);
  const after = realise(graph("gcc-13.3"), mode);
  const moved = Object.keys(before).filter((k) => before[k] !== after[k]);
  console.log(`${mode}-addressed: curl -> ${after["curl-8.9"]}`);
  console.log(`  paths that changed after rebuilding zlib with gcc-13.3: ${moved.length ? moved.join(", ") : "none"}`);
}

// Closure: the full set of store paths needed at run time (copy these and the program works).
const refs: Record<string, string[]> = {
  "curl-8.9": ["openssl-3.3", "zlib-1.3", "glibc-2.39"],
  "openssl-3.3": ["zlib-1.3", "glibc-2.39"],
  "zlib-1.3": ["glibc-2.39"],
  "glibc-2.39": [],
};
const closure = new Set<string>();
const walk = (p: string) => { if (!closure.has(p)) { closure.add(p); refs[p].forEach(walk); } };
walk("curl-8.9");
console.log("closure of curl:", [...closure].sort().join(", "));

input-addressed: curl -> /nix/store/5f7343d2-curl-8.9
  paths that changed after rebuilding zlib with gcc-13.3: zlib-1.3, openssl-3.3, curl-8.9
content-addressed: curl -> /nix/store/fc582430-curl-8.9
  paths that changed after rebuilding zlib with gcc-13.3: none
closure of curl: curl-8.9, glibc-2.39, openssl-3.3, zlib-1.3

"""Bit-for-bit reproducibility: the same sources packed twice, naive vs normalised.

Naive packing leaks the environment into the artifact: wall-clock mtimes, the order the
filesystem lists files in, and the build user's uid/gid. Normalising those (sorted entries,
mtime clamped to SOURCE_DATE_EPOCH, fixed owner, gzip header mtime 0) gives identical bytes.
The two "machines" are simulated, so the output is deterministic.
"""
import gzip, hashlib, io, tarfile

sources = {"src/main.py": b"print('hello')\n", "src/util.py": b"X = 1\n", "README": b"demo\n"}

def pack(listing_order, mtime, uid, user, normalise, epoch=None):
    raw = io.BytesIO()
    names = sorted(listing_order) if normalise else listing_order
    with tarfile.open(fileobj=raw, mode="w", format=tarfile.PAX_FORMAT) as tar:
        for name in names:
            data = sources[name]
            info = tarfile.TarInfo(name)
            info.size = len(data)
            info.mtime = min(mtime, epoch) if normalise else mtime   # clamp to SOURCE_DATE_EPOCH
            info.uid, info.gid = (0, 0) if normalise else (uid, uid)
            info.uname, info.gname = ("", "") if normalise else (user, user)
            info.mode = 0o644
            tar.addfile(info, io.BytesIO(data))
    out = io.BytesIO()
    # gzip stores a timestamp in its header too; fix it at 0 when normalising
    with gzip.GzipFile(fileobj=out, mode="wb", mtime=0 if normalise else mtime) as gz:
        gz.write(raw.getvalue())
    return hashlib.sha256(out.getvalue()).hexdigest()[:16]

SOURCE_DATE_EPOCH = 1759795200   # e.g. the last commit time, from `git log -1 --pretty=%ct`
laptop = dict(listing_order=["src/util.py", "README", "src/main.py"], mtime=1759881600, uid=1000, user="ajay")
ci = dict(listing_order=["README", "src/main.py", "src/util.py"], mtime=1759885200, uid=1001, user="runner")

a, b = pack(**laptop, normalise=False), pack(**ci, normalise=False)
print("naive      laptop", a, " ci", b, " identical:", a == b)
a = pack(**laptop, normalise=True, epoch=SOURCE_DATE_EPOCH)
b = pack(**ci, normalise=True, epoch=SOURCE_DATE_EPOCH)
print("normalised laptop", a, " ci", b, " identical:", a == b)
print("an independent rebuilder can now compare hashes instead of trusting the CI machine")

naive      laptop 42f26564fec115eb  ci 09e1a24ccb81f2d9  identical: False
normalised laptop 2790007285e768d7  ci 2790007285e768d7  identical: True
an independent rebuilder can now compare hashes instead of trusting the CI machine