roast_command_center/server/fetch-page.js

// Guarded HTTPS fetch + HTML->text reduction, ported from
// /Users/shane/dev/hope_roaster/electron/src/agent/fetch-guard.ts (its `fetchUrlGuarded` +
// `extractTextFromHtml`), simplified for a standalone Node server using global fetch.
//
// Guarantees kept: https only (bare host assumed https), every redirect re-validated as
// https, bounded redirects, a hard byte cap enforced while streaming, a timeout, and a
// content-type allowlist. The loopback TLS test escape hatch from the source is dropped —
// this process has no test harness that needs it.

const MAX_BYTES = 2 * 1024 * 1024; // 2 MiB
const TIMEOUT_MS = 10_000;
const MAX_REDIRECTS = 5;
const ALLOWED_CONTENT_TYPES = ["text/html", "text/plain", "application/json", "application/xhtml+xml"];
const USER_AGENT = "RoastPlannerWebapp/0.1 (+local prefill tool)";
const MAX_EXTRACTED_CHARS = 40_000;
const BROWSERLESS_URL = process.env.BROWSERLESS_URL?.replace(/\/$/, "");

function normalizeToUrl(input) {
  const trimmed = input.trim();
  const hasScheme = /^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//.test(trimmed);
  const candidate = hasScheme ? trimmed : `https://${trimmed}`;
  try {
    return new URL(candidate);
  } catch {
    return null;
  }
}

function contentTypeOf(header) {
  if (!header) return undefined;
  return header.split(";")[0].trim().toLowerCase();
}

async function readBodyCapped(response) {
  const reader = response.body?.getReader?.();
  if (!reader) {
    // Fallback for environments without a streaming body (shouldn't happen on Node 24 fetch).
    const buf = new Uint8Array(await response.arrayBuffer());
    const capped = buf.length > MAX_BYTES;
    return { text: Buffer.from(buf.subarray(0, MAX_BYTES)).toString("utf8"), truncated: capped };
  }
  const chunks = [];
  let total = 0;
  let truncated = false;
  for (;;) {
    const { done, value } = await reader.read();
    if (done) break;
    if (total + value.length > MAX_BYTES) {
      chunks.push(value.subarray(0, MAX_BYTES - total));
      truncated = true;
      try {
        await reader.cancel();
      } catch {
        /* best-effort */
      }
      break;
    }
    chunks.push(value);
    total += value.length;
  }
  return { text: Buffer.concat(chunks.map((c) => Buffer.from(c))).toString("utf8"), truncated };
}

/**
 * Fetches `rawUrl` under the guards above and returns { requestedUrl, finalUrl, httpStatus,
 * contentType, bytes, truncated, text, html }. Throws an Error with a `.code` on any failure
 * (`bad_url`, `scheme_rejected`, `redirect_downgrade_rejected`, `too_many_redirects`,
 * `content_type_rejected`, `fetch_timeout`, `fetch_failed`) — the route handler maps these to
 * HTTP status codes.
 */
export async function fetchPageText(rawUrl) {
  let url;
  try {
    url = normalizeToUrl(rawUrl);
  } catch {
    throw withCode(new Error(`"${rawUrl}" is not a valid URL.`), "bad_url");
  }
  if (!url) {
    throw withCode(new Error(`"${rawUrl}" is not a valid URL.`), "bad_url");
  }
  if (url.protocol !== "https:") {
    throw withCode(new Error(`Only https:// URLs are allowed (got ${url.protocol}).`), "scheme_rejected");
  }

  let response;
  let redirects = 0;
  let currentUrl = url;
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), TIMEOUT_MS);
  try {
    for (;;) {
      let res;
      try {
        res = await fetch(currentUrl, {
          method: "GET",
          redirect: "manual",
          signal: controller.signal,
          headers: {
            "user-agent": USER_AGENT,
            accept: "text/html,application/xhtml+xml,application/json,text/plain;q=0.9,*/*;q=0.1",
          },
        });
      } catch (err) {
        if (err.name === "AbortError") throw withCode(new Error(`Request timed out after ${TIMEOUT_MS}ms.`), "fetch_timeout");
        throw withCode(new Error(`Network error fetching ${currentUrl}: ${err.message}`), "fetch_failed");
      }

      if (res.status >= 300 && res.status < 400 && res.headers.get("location")) {
        redirects += 1;
        if (redirects > MAX_REDIRECTS) throw withCode(new Error("Too many redirects."), "too_many_redirects");
        const next = new URL(res.headers.get("location"), currentUrl);
        if (next.protocol !== "https:") {
          throw withCode(new Error(`Redirect to non-https URL rejected (${next}).`), "redirect_downgrade_rejected");
        }
        currentUrl = next;
        continue;
      }
      // Some storefronts (including Shopify sites protected by Cloudflare) rate-limit
      // server-to-server requests while allowing a normal browser. Use the operator's
      // local Browserless service only as a narrow fallback; all existing size and
      // extraction limits still apply below.
      if (res.status === 429 && BROWSERLESS_URL) {
        response = await fetchWithBrowserless(currentUrl, controller.signal);
      } else {
        response = res;
      }
      break;
    }
  } finally {
    clearTimeout(timer);
  }

  const contentType = contentTypeOf(response.headers.get("content-type"));
  if (!contentType || !ALLOWED_CONTENT_TYPES.includes(contentType)) {
    throw withCode(new Error(`Disallowed content-type: ${contentType ?? "(none)"}.`), "content_type_rejected");
  }
  if (!response.ok) {
    throw withCode(new Error(`HTTP ${response.status} fetching ${currentUrl}.`), "fetch_failed");
  }

  const { text: html, truncated } = await readBodyCapped(response);
  const extracted = extractTextFromHtml(html).slice(0, MAX_EXTRACTED_CHARS);

  return {
    requestedUrl: rawUrl,
    finalUrl: currentUrl.toString(),
    httpStatus: response.status,
    contentType,
    bytes: html.length,
    truncated,
    text: extracted,
  };
}

async function fetchWithBrowserless(url, signal) {
  try {
    const response = await fetch(`${BROWSERLESS_URL}/content`, {
      method: "POST",
      signal,
      headers: { "content-type": "application/json" },
      body: JSON.stringify({ url: url.toString(), gotoOptions: { waitUntil: "networkidle2", timeout: TIMEOUT_MS } }),
    });
    if (!response.ok) throw new Error(`Browserless returned HTTP ${response.status}.`);
    return response;
  } catch (error) {
    throw withCode(new Error(`Browser fallback failed for ${url}: ${error.message}`), "fetch_failed");
  }
}

function withCode(err, code) {
  err.code = code;
  return err;
}

/** Ported verbatim (behaviour) from fetch-guard.ts's extractTextFromHtml. */
export function extractTextFromHtml(html) {
  let s = html;
  s = s.replace(/<!--[\s\S]*?-->/g, " ");
  s = s.replace(/<(script|style|noscript|template)\b[^>]*>[\s\S]*?<\/\1>/gi, " ");
  s = s.replace(/<(nav|header|footer|aside)\b[^>]*>[\s\S]*?<\/\1>/gi, " ");
  s = s.replace(/<br\s*\/?>/gi, "\n");
  s = s.replace(/<\/(p|div|li|tr|h1|h2|h3|h4|h5|h6|section|article|table)\s*>/gi, "\n");
  s = s.replace(/<[^>]+>/g, " ");
  s = s
    .replace(/&nbsp;/gi, " ")
    .replace(/&amp;/gi, "&")
    .replace(/&lt;/gi, "<")
    .replace(/&gt;/gi, ">")
    .replace(/&quot;/gi, '"')
    .replace(/&#0?39;/gi, "'");
  s = s.replace(/[ \t]+/g, " ");
  s = s.replace(/[ \t]*\n[ \t]*/g, "\n");
  s = s.replace(/\n{3,}/g, "\n\n");
  return s.trim();
}