diff --git a/docker-compose.yml b/docker-compose.yml index aac5b13..9c74a96 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,10 @@ services: BOOTSTRAP_SETUP_TOKEN: ${BOOTSTRAP_SETUP_TOKEN:-} # Leave unset unless a known reverse-proxy address/CIDR is configured. TRUST_PROXY: ${TRUST_PROXY:-} + # Browser-rendered fallback for storefronts that rate-limit server fetches. + BROWSERLESS_URL: ${BROWSERLESS_URL:-http://host.docker.internal:9085} + extra_hosts: + - "host.docker.internal:host-gateway" volumes: # Mount only non-secret Pi agent model/auth configuration; keep it read-only. - ${PI_AGENT_CONFIG_DIR:-./appdata/pi-agent}:/home/node/.pi/agent:ro diff --git a/server/fetch-page.js b/server/fetch-page.js index d353a5f..de7e5b1 100644 --- a/server/fetch-page.js +++ b/server/fetch-page.js @@ -13,12 +13,17 @@ const MAX_REDIRECTS = 5; const ALLOWED_CONTENT_TYPES = ["text/html", "text/plain", "application/json", "application/xhtml+xml"]; const USER_AGENT = "RoastPlannerWebapp/0.1 (+local prefill tool)"; const MAX_EXTRACTED_CHARS = 40_000; +const BROWSERLESS_URL = process.env.BROWSERLESS_URL?.replace(/\/$/, ""); function normalizeToUrl(input) { const trimmed = input.trim(); const hasScheme = /^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//.test(trimmed); const candidate = hasScheme ? trimmed : `https://${trimmed}`; - return new URL(candidate); + try { + return new URL(candidate); + } catch { + return null; + } } function contentTypeOf(header) { @@ -70,6 +75,9 @@ export async function fetchPageText(rawUrl) { } catch { throw withCode(new Error(`"${rawUrl}" is not a valid URL.`), "bad_url"); } + if (!url) { + throw withCode(new Error(`"${rawUrl}" is not a valid URL.`), "bad_url"); + } if (url.protocol !== "https:") { throw withCode(new Error(`Only https:// URLs are allowed (got ${url.protocol}).`), "scheme_rejected"); } @@ -107,7 +115,15 @@ export async function fetchPageText(rawUrl) { currentUrl = next; continue; } - response = res; + // Some storefronts (including Shopify sites protected by Cloudflare) rate-limit + // server-to-server requests while allowing a normal browser. Use the operator's + // local Browserless service only as a narrow fallback; all existing size and + // extraction limits still apply below. + if (res.status === 429 && BROWSERLESS_URL) { + response = await fetchWithBrowserless(currentUrl, controller.signal); + } else { + response = res; + } break; } } finally { @@ -136,6 +152,21 @@ export async function fetchPageText(rawUrl) { }; } +async function fetchWithBrowserless(url, signal) { + try { + const response = await fetch(`${BROWSERLESS_URL}/content`, { + method: "POST", + signal, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ url: url.toString(), gotoOptions: { waitUntil: "networkidle2", timeout: TIMEOUT_MS } }), + }); + if (!response.ok) throw new Error(`Browserless returned HTTP ${response.status}.`); + return response; + } catch (error) { + throw withCode(new Error(`Browser fallback failed for ${url}: ${error.message}`), "fetch_failed"); + } +} + function withCode(err, code) { err.code = code; return err;