Durable Objects in Production - Resets, Deploys & Class Migrations, Observability, Testing & Interview Q&A

// SIMULATION (not Cloudflare code): during a rollout a NEW Worker can call an OLD object version
// (and vice versa) for seconds to minutes. RPC payloads must be forward and backward compatible.
type V1Msg = { user: string; text: string };
type V2Msg = { user: string; text: string; replyTo?: number };   // additive, optional => safe
type V2Bad = { author: string; body: string };                   // renamed fields => breaks old code

function oldObjectHandle(msg: any): string {                     // version 1 still running somewhere
  if (typeof msg.user !== "string" || typeof msg.text !== "string") throw new Error("400 bad message");
  return `stored "${msg.text}" from ${msg.user}`;
}
function newObjectHandle(msg: Partial<V2Msg> & Partial<V2Bad>): string { // tolerant reader in version 2
  const user = msg.user ?? msg.author, text = msg.text ?? msg.body;
  if (!user || !text) throw new Error("400 bad message");
  return `stored "${text}" from ${user}${msg.replyTo ? ` (reply to #${msg.replyTo})` : ""}`;
}
const cases: [string, unknown, (m: any) => string][] = [
  ["old Worker -> new object", { user: "a", text: "hi" } as V1Msg, newObjectHandle],
  ["new Worker (additive) -> old object", { user: "b", text: "yo", replyTo: 7 } as V2Msg, oldObjectHandle],
  ["new Worker (renamed) -> old object", { author: "c", body: "hey" } as V2Bad, oldObjectHandle],
  ["new Worker (renamed) -> new object", { author: "c", body: "hey" } as V2Bad, newObjectHandle],
];
for (const [name, msg, h] of cases) {
  try { console.log(name.padEnd(38), "OK  ", h(msg)); } catch (e) { console.log(name.padEnd(38), "FAIL", (e as Error).message); }
}
export {};

old Worker -> new object               OK   stored "hi" from a
new Worker (additive) -> old object    OK   stored "yo" from b
new Worker (renamed) -> old object     FAIL 400 bad message
new Worker (renamed) -> new object     OK   stored "hey" from c

{
  // Real Worker + Durable Objects used by the Study lesson. Runs locally with `wrangler dev`.
  "name": "do-study-lab",
  "main": "src/index.ts",
  "compatibility_date": "2026-10-01",
  "durable_objects": {
    "bindings": [
      { "name": "CHAT_ROOM", "class_name": "ChatRoom" },
      { "name": "COUNTER", "class_name": "Counter" },
      { "name": "RATE_LIMITER", "class_name": "RateLimiter" },
      { "name": "LEASE", "class_name": "LeaseLock" },
      { "name": "INVENTORY", "class_name": "Inventory" }
    ]
  },
  // Declarative class lifecycle (replaces the legacy "migrations" array).
  "exports": {
    "ChatRoom": { "type": "durable-object", "storage": "sqlite" },
    "Counter": { "type": "durable-object", "storage": "sqlite" },
    "RateLimiter": { "type": "durable-object", "storage": "sqlite" },
    "LeaseLock": { "type": "durable-object", "storage": "sqlite" },
    "Inventory": { "type": "durable-object", "storage": "sqlite" }
  },
  "observability": { "enabled": true }
}

# SIMULATION (not Cloudflare code) of alarm delivery: at-least-once, retried on uncaught exceptions
# with exponential backoff starting at 2 s, up to 6 retries (numbers from the Alarms docs).
def deliver(handler, fail_first_n, max_retries=6):
    attempts, delay, t = 0, 2.0, 0.0
    while True:
        attempts += 1
        ok = handler(attempt=attempts, will_fail=attempts <= fail_first_n)
        if ok: return f"succeeded on attempt {attempts} at t={t:.0f}s"
        if attempts > max_retries: return f"gave up after {attempts} attempts (t={t:.0f}s); alarm is gone until someone calls setAlarm again"
        t += delay; delay *= 2

charges, seen = [], set()
def naive(attempt, will_fail):
    charges.append(f"charge#{attempt}")    # side effect happens BEFORE the failure below
    return not will_fail
def idempotent(attempt, will_fail):
    if "invoice-42" not in seen:            # dedupe key stored in the object's SQLite in real code
        seen.add("invoice-42"); charges.append("charge")
    return not will_fail

print("naive     :", deliver(naive, fail_first_n=2), "| charges:", charges); charges.clear()
print("idempotent:", deliver(idempotent, fail_first_n=2), "| charges:", charges); charges.clear()
print("long outage:", deliver(idempotent, fail_first_n=99))
print("fix for long outages: catch inside alarm(), record the error, call setAlarm(now + backoff) yourself")

naive     : succeeded on attempt 3 at t=6s | charges: ['charge#1', 'charge#2', 'charge#3']
idempotent: succeeded on attempt 3 at t=6s | charges: ['charge']
long outage: gave up after 7 attempts (t=126s); alarm is gone until someone calls setAlarm again
fix for long outages: catch inside alarm(), record the error, call setAlarm(now + backoff) yourself

// Worker-side call wrapper: recreate the stub after errors, retry only retryable + idempotent calls.
type DOError = Error & { retryable?: boolean; overloaded?: boolean; remote?: boolean };

export async function callWithRetry<T>(makeStub: () => T, op: (s: T) => Promise<unknown>, idempotent: boolean, max = 4) {
  for (let attempt = 0; ; attempt++) {
    const stub = makeStub();                 // a stub that threw may be "broken": always make a fresh one
    try {
      return await op(stub);
    } catch (err) {
      const e = err as DOError;
      if (e.overloaded) throw e;             // never retry overload: it makes the hot object hotter
      if (!e.retryable || !idempotent || attempt >= max) throw e;
      const backoff = Math.min(2000, 100 * 2 ** attempt) * (0.5 + Math.random()); // exponential + jitter
      await new Promise((r) => setTimeout(r, backoff));
    }
  }
}

import { cloudflareTest } from "@cloudflare/vitest-plugin";
import { defineConfig } from "vitest/config";

export default defineConfig({
  plugins: [cloudflareTest({ wrangler: { configPath: "./wrangler.jsonc" } })],
});

// Runs inside workerd via @cloudflare/vitest-plugin: real Durable Objects, real SQLite, real alarms.
import { env } from "cloudflare:workers";
import { evictDurableObject, runDurableObjectAlarm, runInDurableObject } from "cloudflare:test";
import { describe, expect, it } from "vitest";
import type { LeaseLock } from "../src/lease";

describe("Durable Objects lab", () => {
  it("same name -> same object; 100 concurrent RPCs lose no updates", async () => {
    const stub = env.COUNTER.getByName("t-counter");
    await Promise.all(Array.from({ length: 100 }, () => stub.increment(1)));
    expect(await env.COUNTER.getByName("t-counter").get()).toBe(100);
  });

  it("eviction wipes memory but the persisted bucket survives (no free refill)", async () => {
    const stub = env.RATE_LIMITER.getByName("key:evict");
    for (let i = 0; i < 5; i++) await stub.take(5, 0.001); // drain the burst; refill is ~0
    await evictDurableObject(stub);                       // constructor runs again on next call
    const d = await stub.take(5, 0.001);
    expect(d.allowed).toBe(false);
  });

  it("racy reserve oversells, claim-first does not", async () => {
    const racy = env.INVENTORY.getByName("sku:t-racy");
    await racy.reset(3);
    const r1 = await Promise.all(Array.from({ length: 6 }, () => racy.reserveRacy()));
    expect(r1.filter((x) => x === "reserved").length).toBeGreaterThan(3); // the bug, reproduced

    const safe = env.INVENTORY.getByName("sku:t-claim");
    await safe.reset(3);
    const r2 = await Promise.all(Array.from({ length: 6 }, () => safe.reserveClaimFirst()));
    expect(r2.filter((x) => x === "reserved").length).toBe(3);
    expect(await safe.qty()).toBe(0);
  });

  it("lease alarm clears an expired holder; fencing token keeps increasing", async () => {
    const stub = env.LEASE.getByName("lock:t");
    const a = await stub.acquire("a", 60_000);
    // Simulate the TTL passing: rewrite expires_at instead of sleeping for a minute.
    await runInDurableObject(stub, (_inst: LeaseLock, state) => {
      state.storage.sql.exec("UPDATE lease SET expires_at = ? WHERE id = 1", Date.now() - 1);
    });
    expect(await runDurableObjectAlarm(stub)).toBe(true); // fire the pending alarm now
    await runInDurableObject(stub, (_inst: LeaseLock, state) => {
      const row = state.storage.sql.exec("SELECT owner, token FROM lease").one();
      expect(row.owner).toBeNull();
      expect(row.token).toBe(a.token);
    });
    const b = await stub.acquire("b", 1000);
    expect(b.token).toBe((a.token ?? 0) + 1);
  });
});

 ✓ test/do.test.ts > Durable Objects lab > same name -> same object; 100 concurrent RPCs lose no updates 86ms
 ✓ test/do.test.ts > Durable Objects lab > eviction wipes memory but the persisted bucket survives (no free refill) 22ms
 ✓ test/do.test.ts > Durable Objects lab > racy reserve oversells, claim-first does not 155ms
 ✓ test/do.test.ts > Durable Objects lab > lease alarm clears an expired holder; fencing token keeps increasing 21ms
 Test Files  1 passed (1)
      Tests  4 passed (4)