Fallback to Browserless for rate-limited prefills

docker-compose.yml

@@ -10,6 +10,10 @@ services:
       BOOTSTRAP_SETUP_TOKEN: ${BOOTSTRAP_SETUP_TOKEN:-}
       # Leave unset unless a known reverse-proxy address/CIDR is configured.
       TRUST_PROXY: ${TRUST_PROXY:-}
+      # Browser-rendered fallback for storefronts that rate-limit server fetches.
+      BROWSERLESS_URL: ${BROWSERLESS_URL:-http://host.docker.internal:9085}
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     volumes:
       # Mount only non-secret Pi agent model/auth configuration; keep it read-only.
       - ${PI_AGENT_CONFIG_DIR:-./appdata/pi-agent}:/home/node/.pi/agent:ro

server/fetch-page.js

@@ -13,12 +13,17 @@ const MAX_REDIRECTS = 5;
 const ALLOWED_CONTENT_TYPES = ["text/html", "text/plain", "application/json", "application/xhtml+xml"];
 const USER_AGENT = "RoastPlannerWebapp/0.1 (+local prefill tool)";
 const MAX_EXTRACTED_CHARS = 40_000;
+const BROWSERLESS_URL = process.env.BROWSERLESS_URL?.replace(/\/$/, "");
 
 function normalizeToUrl(input) {
   const trimmed = input.trim();
   const hasScheme = /^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//.test(trimmed);
   const candidate = hasScheme ? trimmed : `https://${trimmed}`;
+  try {
     return new URL(candidate);
+  } catch {
+    return null;
+  }
 }
 
 function contentTypeOf(header) {
@@ -70,6 +75,9 @@ export async function fetchPageText(rawUrl) {
   } catch {
     throw withCode(new Error(`"${rawUrl}" is not a valid URL.`), "bad_url");
   }
+  if (!url) {
+    throw withCode(new Error(`"${rawUrl}" is not a valid URL.`), "bad_url");
+  }
   if (url.protocol !== "https:") {
     throw withCode(new Error(`Only https:// URLs are allowed (got ${url.protocol}).`), "scheme_rejected");
   }
@@ -107,7 +115,15 @@ export async function fetchPageText(rawUrl) {
         currentUrl = next;
         continue;
       }
+      // Some storefronts (including Shopify sites protected by Cloudflare) rate-limit
+      // server-to-server requests while allowing a normal browser. Use the operator's
+      // local Browserless service only as a narrow fallback; all existing size and
+      // extraction limits still apply below.
+      if (res.status === 429 && BROWSERLESS_URL) {
+        response = await fetchWithBrowserless(currentUrl, controller.signal);
+      } else {
         response = res;
+      }
       break;
     }
   } finally {
@@ -136,6 +152,21 @@ export async function fetchPageText(rawUrl) {
   };
 }
 
+async function fetchWithBrowserless(url, signal) {
+  try {
+    const response = await fetch(`${BROWSERLESS_URL}/content`, {
+      method: "POST",
+      signal,
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ url: url.toString(), gotoOptions: { waitUntil: "networkidle2", timeout: TIMEOUT_MS } }),
+    });
+    if (!response.ok) throw new Error(`Browserless returned HTTP ${response.status}.`);
+    return response;
+  } catch (error) {
+    throw withCode(new Error(`Browser fallback failed for ${url}: ${error.message}`), "fetch_failed");
+  }
+}
+
 function withCode(err, code) {
   err.code = code;
   return err;