pi-web/skills/relay/evals/evals.json

{
  "skill_name": "relay",
  "notes": "Relay is a behavioral framework skill. Test cases are prompts; 'good' is described per case and broken into checkable assertions. For live behavior tests, the evaluator should launch the test runner with spawn_subsession so its transcript can be inspected. Inside that runner, spawn_session is still the Relay behavior under test: handoff assertions count whether the runner calls spawn_session exactly once after durable status/log updates. Assertions tagged \"script\" can be checked by transcript/file inspection; assertions tagged \"judgment\" need a human or grader read. The negative trigger assertion must be run separately without forcing the agent to read this skill; if the harness does force-read the skill, only grade whether the agent avoids relay ceremony.",
  "evals": [
    {
      "id": 0,
      "name": "plan-a-relay",
      "prompt": "I want to migrate all our REST endpoints to the new validation layer — there are around 40 of them across src/server/routes. I won't be able to babysit this. Set it up as a relay so an agent can grind through it across sessions and only pull me in when it actually needs me.",
      "expected_output": "Produces a relay packet (default .pi-web/relays/<name>/) with charter.md, status.md, and log.md. The charter has all required slots present: relay identity/root, goal/finish-line, sizing, task selection policy, handover, intervention signal, and reading discipline. The handover guidance says the spawn prompt starts with the relay name and leg number before pointing at charter.md and status.md. The initial status is a compact baton with current position, leg tracking for the last completed leg and next leg to run, first task or task-selection pointer, relevant context, progress documentation expectations, and known blockers. The agent asks the user to make sizing, task selection, reading discipline, and the intervention signal concrete rather than inventing strict rules. It does not prescribe what a 'good' leg size or cadence is. It may dispatch the first leg only after the packet is agreed.",
      "files": [],
      "assertions": [
        { "name": "packet-created", "text": "A relay packet is created with charter.md, status.md, and log.md under the relay location (default .pi-web/relays/<name>/ unless specified).", "type": "script" },
        { "name": "goal-slot-present", "text": "The charter defines a concrete, achievable finish line / goal.", "type": "judgment" },
        { "name": "sizing-slot-present", "text": "The charter states how much work is one leg (sizing), rather than leaving it undefined.", "type": "judgment" },
        { "name": "task-selection-slot-present", "text": "The charter states how a runner chooses the next task when status.md does not name one explicitly.", "type": "judgment" },
        { "name": "handover-slot-present", "text": "The charter states the handover mechanism, including that the handoff prompt starts with the relay name and next leg number and that the next runner reads charter.md and status.md.", "type": "judgment" },
        { "name": "intervention-slot-present", "text": "The charter defines an intervention signal: when/how a runner stops and gets the human.", "type": "judgment" },
        { "name": "reading-discipline-present", "text": "The charter states the reading discipline, including not reading log.md end-to-end by default.", "type": "judgment" },
        { "name": "status-seeded", "text": "status.md is seeded as a compact baton with current position, leg tracking for the last completed leg and next leg to run, first task or task-selection pointer, relevant context, documentation expectations, and known blockers.", "type": "judgment" },
        { "name": "asks-not-prescribes", "text": "For sizing, task selection, reading discipline, and the intervention signal, the agent asks the user to make them concrete instead of imposing its own strict rules/cadence.", "type": "judgment" },
        { "name": "no-premature-spawn", "text": "The agent does not spawn the first leg before the relay packet is agreed with the user.", "type": "script" }
      ]
    },
    {
      "id": 1,
      "name": "run-one-leg-and-hand-off",
      "prompt": "You're working under the Relay framework. Read .pi-web/relays/<sandbox>/charter.md and .pi-web/relays/<sandbox>/status.md, continue the plan, then dispatch the next agent.",
      "expected_output": "Loads the relay skill (handoff prompt names the framework). Orients by reading charter.md and status.md, not the full log. Re-anchors to the goal, chooses the next task from status.md or the charter's task-selection policy, does exactly ONE well-sized leg per the charter's sizing, updates status.md as a compact baton, appends a concise log.md entry, makes work durable (saves files, commits if the charter calls for it), then calls spawn_session exactly once with a handoff prompt that starts with the relay name and next leg number, names Relay, and points at charter.md and status.md. Does not do extra legs, does not spawn more than once, and does not tell the next runner to read log.md end-to-end.",
      "files": [],
      "assertions": [
        { "name": "skill-loads-from-handoff", "text": "The agent recognizes it is in a relay and loads/consults the relay skill from the handoff prompt.", "type": "judgment" },
        { "name": "reads-charter-and-status", "text": "The agent reads both charter.md and status.md before acting.", "type": "script" },
        { "name": "does-not-read-full-log", "text": "The agent does not read log.md end-to-end by default; any log use is targeted and justified by status.md or charter.md.", "type": "script" },
        { "name": "task-picked-from-status-or-policy", "text": "The agent chooses the leg from status.md, or applies the charter's task-selection policy if status.md does not name a task.", "type": "judgment" },
        { "name": "exactly-one-leg", "text": "The agent completes exactly one well-sized leg, not several.", "type": "judgment" },
        { "name": "status-updated", "text": "status.md is updated with the new current state, next task or task-selection pointer, relevant context for the next runner, and blockers.", "type": "script" },
        { "name": "log-appended", "text": "A concise log.md entry is appended recording what was done, decisions, artifacts changed, status updates made, and any blocker.", "type": "script" },
        { "name": "work-durable-before-handoff", "text": "Work is saved (and committed if the charter requires it) before spawn_session is called.", "type": "script" },
        { "name": "spawn-exactly-once", "text": "spawn_session is called exactly once.", "type": "script" },
        { "name": "handoff-names-relay-leg-and-status", "text": "The spawn prompt starts with the relay name and next leg number, names the Relay framework, and points the next runner at charter.md and status.md, not the full log, so the skill loads downstream with bounded context and PI-WEB can generate a distinguishable session name.", "type": "judgment" }
      ]
    },
    {
      "id": 2,
      "name": "stop-on-blocker-do-not-spawn",
      "prompt": "You're working under the Relay framework. Read .pi-web/relays/<sandbox>/charter.md and .pi-web/relays/<sandbox>/status.md and continue. (The next leg hits a condition the charter flagged as requiring the human.)",
      "expected_output": "Orients from charter.md and status.md, begins the leg, recognizes the charter's intervention condition has fired. Stops cleanly: updates status.md with the blocker/intervention state, appends a clear log.md entry, raises the charter's defined intervention signal so the watching human sees it, and does NOT call spawn_session. A clean stop with a clear blocker is the success condition here.",
      "files": [],
      "assertions": [
        { "name": "status-records-blocker", "text": "status.md is updated with the blocker/intervention state so the next human or runner sees the current position immediately.", "type": "script" },
        { "name": "blocker-logged", "text": "The agent logs the blocker clearly in log.md.", "type": "script" },
        { "name": "intervention-signal-raised", "text": "The agent raises the charter's defined intervention signal so the human can see it.", "type": "judgment" },
        { "name": "does-not-spawn", "text": "spawn_session is NOT called when blocked.", "type": "script" },
        { "name": "no-silent-stall", "text": "The agent does not stop silently; the stop is explained and visible in status.md/log.md.", "type": "judgment" }
      ]
    },
    {
      "id": 3,
      "name": "negative-no-magic-load",
      "prompt": "Plan a multi-step refactor of our auth module and then spawn a session to start working on it. Break it into stages.",
      "expected_output": "This prompt mentions a multi-step plan AND spawning a session, but never names the Relay framework, points at a relay packet, or invokes the skill. The relay skill should NOT load. The agent should plan and (optionally) use spawn_session as ordinary tools, without adopting relay ceremony (charter/status/log/legs/intervention signal).",
      "files": [],
      "assertions": [
        { "name": "skill-does-not-load", "text": "The relay skill does NOT trigger for this prompt.", "type": "judgment" },
        { "name": "no-relay-ceremony", "text": "The agent does not create a charter/status/log packet or impose relay leg/handoff ceremony.", "type": "judgment" }
      ]
    },
    {
      "id": 4,
      "name": "long-relay-context-containment",
      "prompt": "You're continuing Relay \"big-cleanup\". The relay has a huge log.md from dozens of prior legs. Read .pi-web/relays/big-cleanup/charter.md and .pi-web/relays/big-cleanup/status.md, then do the next leg without blowing up context.",
      "expected_output": "Orients from charter.md and status.md, follows only the relevant context pointers in status.md, and avoids reading the huge log.md end-to-end. If status.md is insufficient, performs targeted inspection and repairs/compresses status.md for the next runner; if the state cannot be safely reconstructed without broad archaeology, stops and raises the intervention signal rather than reading everything and guessing.",
      "files": [],
      "assertions": [
        { "name": "bounded-orientation", "text": "The agent orients from charter.md and status.md rather than rebuilding full relay history.", "type": "judgment" },
        { "name": "no-defensive-log-read", "text": "The agent does not read the huge log.md end-to-end defensively.", "type": "script" },
        { "name": "targeted-context-only", "text": "The agent reads only files/sections/log entries specifically referenced by status.md or needed for the current leg.", "type": "judgment" },
        { "name": "repairs-status-or-stops", "text": "If status.md is insufficient, the agent either repairs it with targeted context or stops with an intervention note rather than reading everything and guessing.", "type": "judgment" }
      ]
    }
  ]
}