317 lines
9.2 KiB
JavaScript
317 lines
9.2 KiB
JavaScript
import { lookup } from "node:dns/promises";
|
|
import { isIP, BlockList } from "node:net";
|
|
import { request } from "node:https";
|
|
import { Readable } from "node:stream";
|
|
|
|
const blocked = new BlockList();
|
|
for (const [ip, prefix] of [
|
|
["0.0.0.0", 8],
|
|
["10.0.0.0", 8],
|
|
["100.64.0.0", 10],
|
|
["127.0.0.0", 8],
|
|
["169.254.0.0", 16],
|
|
["172.16.0.0", 12],
|
|
["192.0.0.0", 24],
|
|
["192.0.2.0", 24],
|
|
["192.168.0.0", 16],
|
|
["198.18.0.0", 15],
|
|
["198.51.100.0", 24],
|
|
["203.0.113.0", 24],
|
|
["224.0.0.0", 3],
|
|
])
|
|
blocked.addSubnet(ip, prefix, "ipv4");
|
|
const globalV6 = new BlockList();
|
|
globalV6.addSubnet("2000::", 3, "ipv6");
|
|
blocked.addSubnet("2001::", 23, "ipv6");
|
|
blocked.addSubnet("2001:db8::", 32, "ipv6");
|
|
blocked.addSubnet("2002::", 16, "ipv6");
|
|
blocked.addSubnet("3fff::", 20, "ipv6");
|
|
export function isPublicAddress(address) {
|
|
const family = isIP(address);
|
|
return family === 4
|
|
? !blocked.check(address, "ipv4")
|
|
: family === 6 &&
|
|
globalV6.check(address, "ipv6") &&
|
|
!blocked.check(address, "ipv6");
|
|
}
|
|
export async function resolvePublicUrl(url, resolver = lookup) {
|
|
if (url.protocol !== "https:")
|
|
throw withCode(
|
|
new Error("Only HTTPS product URLs are allowed."),
|
|
"scheme_rejected",
|
|
);
|
|
if (url.username || url.password || (url.port && url.port !== "443"))
|
|
throw withCode(
|
|
new Error("URL credentials and nonstandard ports are not allowed."),
|
|
"destination_rejected",
|
|
);
|
|
const host = url.hostname.replace(/^\[|\]$/g, "");
|
|
const addresses = isIP(host)
|
|
? [{ address: host, family: isIP(host) }]
|
|
: await resolver(host, { all: true, verbatim: true });
|
|
if (!addresses.length || addresses.some((a) => !isPublicAddress(a.address)))
|
|
throw withCode(
|
|
new Error("Product URLs must resolve only to public Internet addresses."),
|
|
"destination_rejected",
|
|
);
|
|
return addresses[0];
|
|
}
|
|
// Pin the validated DNS answer to the socket lookup. A second DNS lookup would
|
|
// allow a public hostname to rebind to a private service after validation.
|
|
async function publicRequest(url, signal, address) {
|
|
if (signal.aborted) throw signal.reason;
|
|
return new Promise((resolve, reject) => {
|
|
const req = request(
|
|
url,
|
|
{
|
|
signal,
|
|
method: "GET",
|
|
headers: {
|
|
"user-agent": USER_AGENT,
|
|
accept: "text/html,application/xhtml+xml,application/json,text/plain",
|
|
},
|
|
lookup: (_host, options, cb) =>
|
|
options.all
|
|
? cb(null, [address])
|
|
: cb(null, address.address, address.family),
|
|
},
|
|
(res) => {
|
|
const headers = new Headers();
|
|
for (const [k, v] of Object.entries(res.headers))
|
|
if (v != null) headers.set(k, Array.isArray(v) ? v.join(", ") : v);
|
|
const status = res.statusCode;
|
|
const empty = [204, 205, 304].includes(status);
|
|
if (empty) res.resume();
|
|
resolve(
|
|
new Response(empty ? null : Readable.toWeb(res), { status, headers }),
|
|
);
|
|
},
|
|
);
|
|
req.on("error", reject);
|
|
req.end();
|
|
});
|
|
}
|
|
|
|
const MAX_BYTES = 2 * 1024 * 1024; // 2 MiB
|
|
const TIMEOUT_MS = 10_000;
|
|
const MAX_REDIRECTS = 5;
|
|
const ALLOWED_CONTENT_TYPES = [
|
|
"text/html",
|
|
"text/plain",
|
|
"application/json",
|
|
"application/xhtml+xml",
|
|
];
|
|
const USER_AGENT = "RoastPlannerWebapp/0.1 (+local prefill tool)";
|
|
const MAX_EXTRACTED_CHARS = 40_000;
|
|
|
|
function normalizeToUrl(input) {
|
|
const trimmed = input.trim();
|
|
const hasScheme = /^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//.test(trimmed);
|
|
const candidate = hasScheme ? trimmed : `https://${trimmed}`;
|
|
try {
|
|
return new URL(candidate);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function contentTypeOf(header) {
|
|
if (!header) return undefined;
|
|
return header.split(";")[0].trim().toLowerCase();
|
|
}
|
|
|
|
async function readBodyCapped(response) {
|
|
const reader = response.body?.getReader?.();
|
|
if (!reader) {
|
|
// Fallback for environments without a streaming body (shouldn't happen on Node 24 fetch).
|
|
const buf = new Uint8Array(await response.arrayBuffer());
|
|
const capped = buf.length > MAX_BYTES;
|
|
return {
|
|
text: Buffer.from(buf.subarray(0, MAX_BYTES)).toString("utf8"),
|
|
truncated: capped,
|
|
};
|
|
}
|
|
const chunks = [];
|
|
let total = 0;
|
|
let truncated = false;
|
|
for (;;) {
|
|
const { done, value } = await reader.read();
|
|
if (done) break;
|
|
if (total + value.length > MAX_BYTES) {
|
|
chunks.push(value.subarray(0, MAX_BYTES - total));
|
|
truncated = true;
|
|
try {
|
|
await reader.cancel();
|
|
} catch {
|
|
/* best-effort */
|
|
}
|
|
break;
|
|
}
|
|
chunks.push(value);
|
|
total += value.length;
|
|
}
|
|
return {
|
|
text: Buffer.concat(chunks.map((c) => Buffer.from(c))).toString("utf8"),
|
|
truncated,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Fetches `rawUrl` under the guards above and returns { requestedUrl, finalUrl, httpStatus,
|
|
* contentType, bytes, truncated, text, html }. Throws an Error with a `.code` on any failure
|
|
* (`bad_url`, `scheme_rejected`, `redirect_downgrade_rejected`, `too_many_redirects`,
|
|
* `content_type_rejected`, `fetch_timeout`, `fetch_failed`) — the route handler maps these to
|
|
* HTTP status codes.
|
|
*/
|
|
export async function fetchPageText(
|
|
rawUrl,
|
|
{
|
|
requestPage = publicRequest,
|
|
resolver = lookup,
|
|
timeoutMs = TIMEOUT_MS,
|
|
} = {},
|
|
) {
|
|
let url;
|
|
try {
|
|
url = normalizeToUrl(rawUrl);
|
|
} catch {
|
|
throw withCode(new Error(`"${rawUrl}" is not a valid URL.`), "bad_url");
|
|
}
|
|
if (!url) {
|
|
throw withCode(new Error(`"${rawUrl}" is not a valid URL.`), "bad_url");
|
|
}
|
|
if (url.protocol !== "https:") {
|
|
throw withCode(
|
|
new Error(`Only https:// URLs are allowed (got ${url.protocol}).`),
|
|
"scheme_rejected",
|
|
);
|
|
}
|
|
|
|
let response;
|
|
let redirects = 0;
|
|
let currentUrl = url;
|
|
const controller = new AbortController();
|
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
try {
|
|
for (;;) {
|
|
let res;
|
|
try {
|
|
const address = await Promise.race([
|
|
resolvePublicUrl(currentUrl, resolver),
|
|
new Promise((_, reject) =>
|
|
controller.signal.addEventListener(
|
|
"abort",
|
|
() =>
|
|
reject(
|
|
withCode(new Error("Request timed out."), "fetch_timeout"),
|
|
),
|
|
{ once: true },
|
|
),
|
|
),
|
|
]);
|
|
res = await requestPage(currentUrl, controller.signal, address);
|
|
} catch (err) {
|
|
if (["destination_rejected", "fetch_timeout"].includes(err.code))
|
|
throw err;
|
|
if (err.name === "AbortError")
|
|
throw withCode(
|
|
new Error(`Request timed out after ${TIMEOUT_MS}ms.`),
|
|
"fetch_timeout",
|
|
);
|
|
throw withCode(
|
|
new Error(`Network error fetching ${currentUrl}: ${err.message}`),
|
|
"fetch_failed",
|
|
);
|
|
}
|
|
|
|
if (
|
|
res.status >= 300 &&
|
|
res.status < 400 &&
|
|
res.headers.get("location")
|
|
) {
|
|
await res.body?.cancel();
|
|
redirects += 1;
|
|
if (redirects > MAX_REDIRECTS)
|
|
throw withCode(
|
|
new Error("Too many redirects."),
|
|
"too_many_redirects",
|
|
);
|
|
const next = new URL(res.headers.get("location"), currentUrl);
|
|
if (next.protocol !== "https:") {
|
|
throw withCode(
|
|
new Error(`Redirect to non-https URL rejected (${next}).`),
|
|
"redirect_downgrade_rejected",
|
|
);
|
|
}
|
|
currentUrl = next;
|
|
continue;
|
|
}
|
|
response = res;
|
|
break;
|
|
}
|
|
|
|
const contentType = contentTypeOf(response.headers.get("content-type"));
|
|
if (!contentType || !ALLOWED_CONTENT_TYPES.includes(contentType)) {
|
|
throw withCode(
|
|
new Error(`Disallowed content-type: ${contentType ?? "(none)"}.`),
|
|
"content_type_rejected",
|
|
);
|
|
}
|
|
if (!response.ok) {
|
|
throw withCode(
|
|
new Error(`HTTP ${response.status} fetching ${currentUrl}.`),
|
|
"fetch_failed",
|
|
);
|
|
}
|
|
|
|
const { text: html, truncated } = await readBodyCapped(response);
|
|
const extracted = extractTextFromHtml(html).slice(0, MAX_EXTRACTED_CHARS);
|
|
|
|
return {
|
|
requestedUrl: rawUrl,
|
|
finalUrl: currentUrl.toString(),
|
|
httpStatus: response.status,
|
|
contentType,
|
|
bytes: Buffer.byteLength(html),
|
|
truncated,
|
|
text: extracted,
|
|
};
|
|
} finally {
|
|
clearTimeout(timer);
|
|
await response?.body?.cancel().catch(() => {});
|
|
}
|
|
}
|
|
|
|
function withCode(err, code) {
|
|
err.code = code;
|
|
return err;
|
|
}
|
|
|
|
/** Ported verbatim (behaviour) from fetch-guard.ts's extractTextFromHtml. */
|
|
export function extractTextFromHtml(html) {
|
|
let s = html;
|
|
s = s.replace(/<!--[\s\S]*?-->/g, " ");
|
|
s = s.replace(
|
|
/<(script|style|noscript|template)\b[^>]*>[\s\S]*?<\/\1>/gi,
|
|
" ",
|
|
);
|
|
s = s.replace(/<(nav|header|footer|aside)\b[^>]*>[\s\S]*?<\/\1>/gi, " ");
|
|
s = s.replace(/<br\s*\/?>/gi, "\n");
|
|
s = s.replace(
|
|
/<\/(p|div|li|tr|h1|h2|h3|h4|h5|h6|section|article|table)\s*>/gi,
|
|
"\n",
|
|
);
|
|
s = s.replace(/<[^>]+>/g, " ");
|
|
s = s
|
|
.replace(/ /gi, " ")
|
|
.replace(/&/gi, "&")
|
|
.replace(/</gi, "<")
|
|
.replace(/>/gi, ">")
|
|
.replace(/"/gi, '"')
|
|
.replace(/�?39;/gi, "'");
|
|
s = s.replace(/[ \t]+/g, " ");
|
|
s = s.replace(/[ \t]*\n[ \t]*/g, "\n");
|
|
s = s.replace(/\n{3,}/g, "\n\n");
|
|
return s.trim();
|
|
}
|