Rebuild printable roast plans and remove machine-based adjustments

2026-09-05 14:56:11 -04:00
1 parent e9f2393c96
commit 824a964f9d
@@ -1,19 +1,108 @@
-// Guarded HTTPS fetch + HTML->text reduction, ported from
-// /Users/shane/dev/hope_roaster/electron/src/agent/fetch-guard.ts (its `fetchUrlGuarded` +
-// `extractTextFromHtml`), simplified for a standalone Node server using global fetch.
-//
-// Guarantees kept: https only (bare host assumed https), every redirect re-validated as
-// https, bounded redirects, a hard byte cap enforced while streaming, a timeout, and a
-// content-type allowlist. The loopback TLS test escape hatch from the source is dropped —
-// this process has no test harness that needs it.
+import { lookup } from "node:dns/promises";
+import { isIP, BlockList } from "node:net";
+import { request } from "node:https";
+import { Readable } from "node:stream";
+
+const blocked = new BlockList();
+for (const [ip, prefix] of [
+  ["0.0.0.0", 8],
+  ["10.0.0.0", 8],
+  ["100.64.0.0", 10],
+  ["127.0.0.0", 8],
+  ["169.254.0.0", 16],
+  ["172.16.0.0", 12],
+  ["192.0.0.0", 24],
+  ["192.0.2.0", 24],
+  ["192.168.0.0", 16],
+  ["198.18.0.0", 15],
+  ["198.51.100.0", 24],
+  ["203.0.113.0", 24],
+  ["224.0.0.0", 3],
+])
+  blocked.addSubnet(ip, prefix, "ipv4");
+const globalV6 = new BlockList();
+globalV6.addSubnet("2000::", 3, "ipv6");
+blocked.addSubnet("2001::", 23, "ipv6");
+blocked.addSubnet("2001:db8::", 32, "ipv6");
+blocked.addSubnet("2002::", 16, "ipv6");
+blocked.addSubnet("3fff::", 20, "ipv6");
+export function isPublicAddress(address) {
+  const family = isIP(address);
+  return family === 4
+    ? !blocked.check(address, "ipv4")
+    : family === 6 &&
+        globalV6.check(address, "ipv6") &&
+        !blocked.check(address, "ipv6");
+}
+export async function resolvePublicUrl(url, resolver = lookup) {
+  if (url.protocol !== "https:")
+    throw withCode(
+      new Error("Only HTTPS product URLs are allowed."),
+      "scheme_rejected",
+    );
+  if (url.username || url.password || (url.port && url.port !== "443"))
+    throw withCode(
+      new Error("URL credentials and nonstandard ports are not allowed."),
+      "destination_rejected",
+    );
+  const host = url.hostname.replace(/^\[|\]$/g, "");
+  const addresses = isIP(host)
+    ? [{ address: host, family: isIP(host) }]
+    : await resolver(host, { all: true, verbatim: true });
+  if (!addresses.length || addresses.some((a) => !isPublicAddress(a.address)))
+    throw withCode(
+      new Error("Product URLs must resolve only to public Internet addresses."),
+      "destination_rejected",
+    );
+  return addresses[0];
+}
+// Pin the validated DNS answer to the socket lookup. A second DNS lookup would
+// allow a public hostname to rebind to a private service after validation.
+async function publicRequest(url, signal, address) {
+  if (signal.aborted) throw signal.reason;
+  return new Promise((resolve, reject) => {
+    const req = request(
+      url,
+      {
+        signal,
+        method: "GET",
+        headers: {
+          "user-agent": USER_AGENT,
+          accept: "text/html,application/xhtml+xml,application/json,text/plain",
+        },
+        lookup: (_host, options, cb) =>
+          options.all
+            ? cb(null, [address])
+            : cb(null, address.address, address.family),
+      },
+      (res) => {
+        const headers = new Headers();
+        for (const [k, v] of Object.entries(res.headers))
+          if (v != null) headers.set(k, Array.isArray(v) ? v.join(", ") : v);
+        const status = res.statusCode;
+        const empty = [204, 205, 304].includes(status);
+        if (empty) res.resume();
+        resolve(
+          new Response(empty ? null : Readable.toWeb(res), { status, headers }),
+        );
+      },
+    );
+    req.on("error", reject);
+    req.end();
+  });
+}

 const MAX_BYTES = 2 * 1024 * 1024; // 2 MiB
 const TIMEOUT_MS = 10_000;
 const MAX_REDIRECTS = 5;
-const ALLOWED_CONTENT_TYPES = ["text/html", "text/plain", "application/json", "application/xhtml+xml"];
+const ALLOWED_CONTENT_TYPES = [
+  "text/html",
+  "text/plain",
+  "application/json",
+  "application/xhtml+xml",
+];
 const USER_AGENT = "RoastPlannerWebapp/0.1 (+local prefill tool)";
 const MAX_EXTRACTED_CHARS = 40_000;
-const BROWSERLESS_URL = process.env.BROWSERLESS_URL?.replace(/\/$/, "");

 function normalizeToUrl(input) {
  const trimmed = input.trim();
@@ -37,7 +126,10 @@ async function readBodyCapped(response) {
    // Fallback for environments without a streaming body (shouldn't happen on Node 24 fetch).
    const buf = new Uint8Array(await response.arrayBuffer());
    const capped = buf.length > MAX_BYTES;
-    return { text: Buffer.from(buf.subarray(0, MAX_BYTES)).toString("utf8"), truncated: capped };
+    return {
+      text: Buffer.from(buf.subarray(0, MAX_BYTES)).toString("utf8"),
+      truncated: capped,
+    };
  }
  const chunks = [];
  let total = 0;
@@ -58,7 +150,10 @@ async function readBodyCapped(response) {
    chunks.push(value);
    total += value.length;
  }
-  return { text: Buffer.concat(chunks.map((c) => Buffer.from(c))).toString("utf8"), truncated };
+  return {
+    text: Buffer.concat(chunks.map((c) => Buffer.from(c))).toString("utf8"),
+    truncated,
+  };
 }

 /**
@@ -68,7 +163,14 @@ async function readBodyCapped(response) {
 * `content_type_rejected`, `fetch_timeout`, `fetch_failed`) — the route handler maps these to
 * HTTP status codes.
 */
-export async function fetchPageText(rawUrl) {
+export async function fetchPageText(
+  rawUrl,
+  {
+    requestPage = publicRequest,
+    resolver = lookup,
+    timeoutMs = TIMEOUT_MS,
+  } = {},
+) {
  let url;
  try {
    url = normalizeToUrl(rawUrl);
@@ -79,91 +181,104 @@ export async function fetchPageText(rawUrl) {
    throw withCode(new Error(`"${rawUrl}" is not a valid URL.`), "bad_url");
  }
  if (url.protocol !== "https:") {
-    throw withCode(new Error(`Only https:// URLs are allowed (got ${url.protocol}).`), "scheme_rejected");
+    throw withCode(
+      new Error(`Only https:// URLs are allowed (got ${url.protocol}).`),
+      "scheme_rejected",
+    );
  }

  let response;
  let redirects = 0;
  let currentUrl = url;
  const controller = new AbortController();
-  const timer = setTimeout(() => controller.abort(), TIMEOUT_MS);
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
  try {
    for (;;) {
      let res;
      try {
-        res = await fetch(currentUrl, {
-          method: "GET",
-          redirect: "manual",
-          signal: controller.signal,
-          headers: {
-            "user-agent": USER_AGENT,
-            accept: "text/html,application/xhtml+xml,application/json,text/plain;q=0.9,*/*;q=0.1",
-          },
-        });
+        const address = await Promise.race([
+          resolvePublicUrl(currentUrl, resolver),
+          new Promise((_, reject) =>
+            controller.signal.addEventListener(
+              "abort",
+              () =>
+                reject(
+                  withCode(new Error("Request timed out."), "fetch_timeout"),
+                ),
+              { once: true },
+            ),
+          ),
+        ]);
+        res = await requestPage(currentUrl, controller.signal, address);
      } catch (err) {
-        if (err.name === "AbortError") throw withCode(new Error(`Request timed out after ${TIMEOUT_MS}ms.`), "fetch_timeout");
-        throw withCode(new Error(`Network error fetching ${currentUrl}: ${err.message}`), "fetch_failed");
+        if (["destination_rejected", "fetch_timeout"].includes(err.code))
+          throw err;
+        if (err.name === "AbortError")
+          throw withCode(
+            new Error(`Request timed out after ${TIMEOUT_MS}ms.`),
+            "fetch_timeout",
+          );
+        throw withCode(
+          new Error(`Network error fetching ${currentUrl}: ${err.message}`),
+          "fetch_failed",
+        );
      }

-      if (res.status >= 300 && res.status < 400 && res.headers.get("location")) {
+      if (
+        res.status >= 300 &&
+        res.status < 400 &&
+        res.headers.get("location")
+      ) {
+        await res.body?.cancel();
        redirects += 1;
-        if (redirects > MAX_REDIRECTS) throw withCode(new Error("Too many redirects."), "too_many_redirects");
+        if (redirects > MAX_REDIRECTS)
+          throw withCode(
+            new Error("Too many redirects."),
+            "too_many_redirects",
+          );
        const next = new URL(res.headers.get("location"), currentUrl);
        if (next.protocol !== "https:") {
-          throw withCode(new Error(`Redirect to non-https URL rejected (${next}).`), "redirect_downgrade_rejected");
+          throw withCode(
+            new Error(`Redirect to non-https URL rejected (${next}).`),
+            "redirect_downgrade_rejected",
+          );
        }
        currentUrl = next;
        continue;
      }
-      // Some storefronts (including Shopify sites protected by Cloudflare) rate-limit
-      // server-to-server requests while allowing a normal browser. Use the operator's
-      // local Browserless service only as a narrow fallback; all existing size and
-      // extraction limits still apply below.
-      if (res.status === 429 && BROWSERLESS_URL) {
-        response = await fetchWithBrowserless(currentUrl, controller.signal);
-      } else {
-        response = res;
-      }
+      response = res;
      break;
    }
+
+    const contentType = contentTypeOf(response.headers.get("content-type"));
+    if (!contentType || !ALLOWED_CONTENT_TYPES.includes(contentType)) {
+      throw withCode(
+        new Error(`Disallowed content-type: ${contentType ?? "(none)"}.`),
+        "content_type_rejected",
+      );
+    }
+    if (!response.ok) {
+      throw withCode(
+        new Error(`HTTP ${response.status} fetching ${currentUrl}.`),
+        "fetch_failed",
+      );
+    }
+
+    const { text: html, truncated } = await readBodyCapped(response);
+    const extracted = extractTextFromHtml(html).slice(0, MAX_EXTRACTED_CHARS);
+
+    return {
+      requestedUrl: rawUrl,
+      finalUrl: currentUrl.toString(),
+      httpStatus: response.status,
+      contentType,
+      bytes: Buffer.byteLength(html),
+      truncated,
+      text: extracted,
+    };
  } finally {
    clearTimeout(timer);
-  }
-
-  const contentType = contentTypeOf(response.headers.get("content-type"));
-  if (!contentType || !ALLOWED_CONTENT_TYPES.includes(contentType)) {
-    throw withCode(new Error(`Disallowed content-type: ${contentType ?? "(none)"}.`), "content_type_rejected");
-  }
-  if (!response.ok) {
-    throw withCode(new Error(`HTTP ${response.status} fetching ${currentUrl}.`), "fetch_failed");
-  }
-
-  const { text: html, truncated } = await readBodyCapped(response);
-  const extracted = extractTextFromHtml(html).slice(0, MAX_EXTRACTED_CHARS);
-
-  return {
-    requestedUrl: rawUrl,
-    finalUrl: currentUrl.toString(),
-    httpStatus: response.status,
-    contentType,
-    bytes: html.length,
-    truncated,
-    text: extracted,
-  };
-}
-
-async function fetchWithBrowserless(url, signal) {
-  try {
-    const response = await fetch(`${BROWSERLESS_URL}/content`, {
-      method: "POST",
-      signal,
-      headers: { "content-type": "application/json" },
-      body: JSON.stringify({ url: url.toString(), gotoOptions: { waitUntil: "networkidle2", timeout: TIMEOUT_MS } }),
-    });
-    if (!response.ok) throw new Error(`Browserless returned HTTP ${response.status}.`);
-    return response;
-  } catch (error) {
-    throw withCode(new Error(`Browser fallback failed for ${url}: ${error.message}`), "fetch_failed");
+    await response?.body?.cancel().catch(() => {});
  }
 }

@@ -176,10 +291,16 @@ function withCode(err, code) {
 export function extractTextFromHtml(html) {
  let s = html;
  s = s.replace(/<!--[\s\S]*?-->/g, " ");
-  s = s.replace(/<(script|style|noscript|template)\b[^>]*>[\s\S]*?<\/\1>/gi, " ");
+  s = s.replace(
+    /<(script|style|noscript|template)\b[^>]*>[\s\S]*?<\/\1>/gi,
+    " ",
+  );
  s = s.replace(/<(nav|header|footer|aside)\b[^>]*>[\s\S]*?<\/\1>/gi, " ");
  s = s.replace(/<br\s*\/?>/gi, "\n");
-  s = s.replace(/<\/(p|div|li|tr|h1|h2|h3|h4|h5|h6|section|article|table)\s*>/gi, "\n");
+  s = s.replace(
+    /<\/(p|div|li|tr|h1|h2|h3|h4|h5|h6|section|article|table)\s*>/gi,
+    "\n",
+  );
  s = s.replace(/<[^>]+>/g, " ");
  s = s
    .replace(/&nbsp;/gi, " ")