roast_command_center/test/llm-admin.test.js

import test from "node:test";
import assert from "node:assert/strict";
import { setup, signup, password } from "./helpers.js";

const MODELS = [
	{ key: "opencode-go:minimax-m3", name: "MiniMax-M3", provider: "opencode-go" },
	{ key: "opencode-go:big-model", name: "Big Model", provider: "opencode-go" },
];

async function bootstrapAdmin(agent) {
	const response = await agent.post("/api/auth/bootstrap").send({
		email: "[email protected]",
		password,
		setupToken: "a-secure-bootstrap-token",
	});
	assert.equal(response.status, 201);
	return response.body.csrfToken;
}

function makeAlog() {
	const timex = [], temp1 = [], temp2 = [];
	for (let i = 0; i <= 20; i++) {
		timex.push(i * 30);
		temp1.push(200 + i);
		temp2.push(i < 3 ? 180 - i * 30 : 90 + (i - 3) * 7);
	}
	return JSON.stringify({ title: "T", mode: "C", weight: [250, 212, "g"], timex, temp1, temp2, timeindex: [1, 8, 14, 0, 0, 0, 20, 0] });
}

test("admin can list and change the LLM model; the setting reaches evaluations", async () => {
	const evalCalls = [];
	const { agent } = await setup(
		{},
		{
			listModels: async () => MODELS,
			evaluateRoast: async (parsed, plan, preferredModel) => {
				evalCalls.push(preferredModel);
				return { summary: "ok", grade: "good", highlights: [], concerns: [], suggestions: [], planComparison: null };
			},
		},
	);
	const csrf = await bootstrapAdmin(agent);

	const listing = await agent.get("/api/admin/llm");
	assert.equal(listing.status, 200);
	assert.equal(listing.body.current, "");
	assert.equal(listing.body.models.length, 2);
	assert.equal(listing.body.modelsError, null);

	const set = await agent
		.put("/api/admin/llm")
		.set("x-csrf-token", csrf)
		.send({ model: "opencode-go:big-model" });
	assert.equal(set.status, 200);
	assert.equal((await agent.get("/api/admin/llm")).body.current, "opencode-go:big-model");

	// An unknown model is rejected, a non-string is rejected
	assert.equal(
		(await agent.put("/api/admin/llm").set("x-csrf-token", csrf).send({ model: "nope:x" })).status,
		400,
	);
	assert.equal(
		(await agent.put("/api/admin/llm").set("x-csrf-token", csrf).send({ model: 5 })).status,
		400,
	);

	// The chosen model is what evaluations receive
	const up = await agent
		.post("/api/roasts")
		.set("x-csrf-token", csrf)
		.send({ filename: "r.alog", content: makeAlog() });
	assert.equal(up.status, 201);
	for (let i = 0; i < 100 && !evalCalls.length; i++)
		await new Promise((resolve) => setTimeout(resolve, 10));
	assert.deepEqual(evalCalls, ["opencode-go:big-model"]);

	// Back to auto
	const clear = await agent.put("/api/admin/llm").set("x-csrf-token", csrf).send({ model: "" });
	assert.equal(clear.status, 200);
	assert.equal((await agent.get("/api/admin/llm")).body.current, "");
});

test("LLM setting is admin-only and degrades when no models are configured", async () => {
	const { agent, app } = await setup(
		{},
		{
			listModels: async () => {
				throw new Error("no runtime");
			},
		},
	);
	const { csrf } = await signup(agent, "[email protected]");
	assert.equal((await agent.get("/api/admin/llm")).status, 403);
	assert.equal(
		(await agent.put("/api/admin/llm").set("x-csrf-token", csrf).send({ model: "" })).status,
		403,
	);

	const request = (await import("supertest")).default;
	const adminAgent = request.agent(app);
	const adminCsrf = await bootstrapAdmin(adminAgent);
	const listing = await adminAgent.get("/api/admin/llm");
	assert.equal(listing.status, 200);
	assert.equal(listing.body.modelsError, "llm_unavailable");
	assert.deepEqual(listing.body.models, []);
	// A concrete model can't be validated with no runtime, but clearing to auto still works
	assert.equal(
		(await adminAgent.put("/api/admin/llm").set("x-csrf-token", adminCsrf).send({ model: "a:b" })).status,
		503,
	);
	assert.equal(
		(await adminAgent.put("/api/admin/llm").set("x-csrf-token", adminCsrf).send({ model: "" })).status,
		200,
	);
});