fix: background tasks now fetch real page content; add task-status tools

Background research tasks were producing garbage or failing because the
self-hosted Firecrawl /v1/scrape defaults to a ~30s timeout that's too short
for heavy news/research sites, and the small LLM won't reliably chain a
second scrape call after a search that only returns snippets.

task_worker.py:
- web_search now auto-fetches the top results directly from the container
  (which has working outbound internet) and returns their real text, so the
  model gets actual page content in one call instead of snippet echoes.
- web_scrape requests timeout: 90000 (Firecrawl renders heavy sites but needs
  >30s), and falls back to a direct container fetch with tag-stripping if
  Firecrawl times out or returns nothing.

task_registry.py / dispatch_mcp.py:
- Add task_status(id) and list_tasks() tools so Hope can check the live state
  of any dispatched task (pending / running / completed / failed / STALLED).
- Track last_activity per task and surface a stall heuristic.
- Move the status summary into task_registry (no mcp dependency) so it's
  unit-testable.

agent.py: prompt tells Hope about task_status / list_tasks.
tests: add task-status summary and last-activity regression tests.

agent/task_worker.py

@@ -16,6 +16,35 @@ from datetime import datetime, timezone
 logger = logging.getLogger("voice-agent.tasks")
 
 
+def _html_to_text(html: str) -> str:
+    """Strip a raw HTML page to readable plain text (no external deps)."""
+    if not html:
+        return ""
+    # Drop non-content blocks first.
+    html = re.sub(
+        r"(?is)<(script|style|noscript|head|title|svg|nav|footer|form|iframe)[^>]*>.*?</\1>",
+        " ",
+        html,
+    )
+    # Turn block-level tags into paragraph breaks.
+    html = re.sub(
+        r"(?i)</?(?:p|div|li|ul|ol|h[1-6]|br|tr|td|th|article|section|"
+        r"blockquote|header|figure|table|hr)[^>]*>",
+        "\n",
+        html,
+    )
+    text = re.sub(r"<[^>]+>", " ", html)
+    for ent, ch in (
+        ("&nbsp;", " "), ("&amp;", "&"), ("&lt;", "<"), ("&gt;", ">"),
+        ("&quot;", '"'), ("&#39;", "'"), ("&apos;", "'"), ("&#x27;", "'"),
+        ("&hellip;", "..."), ("&mdash;", "-"), ("&ndash;", "-"),
+    ):
+        text = text.replace(ent, ch)
+    text = re.sub(r"[ \t]+", " ", text)
+    text = re.sub(r"\n\s*\n+", "\n", text)
+    return text.strip()
+
+
 class TaskWorker:
     """Runs a background research/lookup task using the LLM + tools."""
 
@@ -289,6 +318,36 @@ class TaskWorker:
             },
         ]
 
+    async def _fetch_page_text(self, url: str, timeout: float = 20.0) -> str:
+        """Fetch a page directly from the container and reduce it to plain text.
+
+        Used as a fallback when Firecrawl's scrape times out or returns nothing.
+        The container has outbound internet (the self-hosted Firecrawl does not
+        reliably, and its 30s default scrape limit is too short for news sites).
+        """
+        if not url:
+            return "No URL provided."
+        import httpx
+
+        try:
+            async with httpx.AsyncClient(
+                timeout=timeout,
+                follow_redirects=True,
+                headers={
+                    "User-Agent": (
+                        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+                        "(KHTML, like Gecko) Chrome/124.0 Safari/537.36"
+                    )
+                },
+            ) as client:
+                resp = await client.get(url)
+                if resp.status_code >= 400:
+                    return f"Fetch failed (HTTP {resp.status_code})."
+                text = _html_to_text(resp.text)
+                return text[:8000]
+        except httpx.HTTPError as exc:
+            return f"Fetch failed: {exc}"
+
     async def _execute_tool(self, name: str, args: dict) -> str:
         import httpx
 
@@ -351,19 +410,24 @@ class TaskWorker:
                         f"{firecrawl_base}/v1/search",
                         json={"query": query, "limit": 5},
                     )
-                    if resp.status_code == 200:
-                        data = resp.json()
-                        items = data.get("data", [])
-                        lines = []
-                        for item in items[:5]:
-                            title = (item.get("title") or "Untitled").strip()
-                            description = (item.get("description") or "").strip()
-                            url = (item.get("url") or "").strip()
-                            lines.append(f"{title}: {description}")
-                            if url:
-                                lines.append(f"URL: {url}")
-                        return "\n".join(lines) or "No results found."
-                    return f"Search failed (HTTP {resp.status_code})."
+                    if resp.status_code != 200:
+                        return f"Search failed (HTTP {resp.status_code})."
+                    data = resp.json()
+                    items = data.get("data", [])
+                    if not items:
+                        return "No results found for that query."
+                    # Search results only carry titles/descriptions. Fetch the
+                    # top pages ourselves (container has outbound internet) so the
+                    # model gets real content instead of just snippets — small
+                    # models won't reliably chain a second scrape call.
+                    blocks = []
+                    for item in items[:3]:
+                        title = (item.get("title") or "Untitled").strip()
+                        url = (item.get("url") or "").strip()
+                        desc = (item.get("description") or "").strip()
+                        body = await self._fetch_page_text(url) if url else (desc or "No URL.")
+                        blocks.append(f"{title}\n{url}\n{body[:1800]}")
+                    return "\n\n---\n\n".join(blocks) or "No results found."
 
             elif name == "web_scrape":
                 url = args.get("url", "")
@@ -372,22 +436,29 @@ class TaskWorker:
                 firecrawl_base = os.environ.get("FIRECRAWL_BASE", "")
                 if not firecrawl_base:
                     return "Web scraping is not configured."
-                async with httpx.AsyncClient(timeout=90) as client:
-                    resp = await client.post(
-                        f"{firecrawl_base}/v1/scrape",
-                        json={"url": url, "formats": ["markdown"], "timeout": 60000},
-                    )
-                    if resp.status_code != 200:
-                        return f"Scrape failed (HTTP {resp.status_code})."
-                    data = resp.json()
-                    if not data.get("success"):
-                        return f"Scrape failed: {data.get('error', 'unknown error')}"
-                    result = data.get("data", {})
-                    markdown = (result.get("markdown") or "").strip()
-                    if not markdown:
-                        return "No page content was extracted."
-                    title = (result.get("metadata", {}).get("title") or "").strip()
-                    return f"Page: {title}\n\n{markdown[:8000]}"
+                # Firecrawl renders JS-heavy pages with a real browser, but its
+                # default scrape timeout (~30s) is too short for big news sites.
+                # Ask for 90s and give the HTTP client the same headroom; if it
+                # returns nothing, fall back to a direct fetch from the container.
+                try:
+                    async with httpx.AsyncClient(timeout=120) as client:
+                        resp = await client.post(
+                            f"{firecrawl_base}/v1/scrape",
+                            json={"url": url, "formats": ["markdown"], "timeout": 90000},
+                        )
+                        if resp.status_code == 200:
+                            data = resp.json()
+                            if data.get("success"):
+                                result = data.get("data", {})
+                                markdown = (result.get("markdown") or "").strip()
+                                if markdown:
+                                    title = (result.get("metadata", {}).get("title") or url).strip()
+                                    return f"Page: {title}\n\n{markdown[:8000]}"
+                        # fall through to direct fetch on empty/failed scrape
+                except (httpx.HTTPError, Exception):
+                    pass
+                body = await self._fetch_page_text(url)
+                return body or f"Could not extract content from {url}."
 
             else:
                 return f"Unknown tool: {name}"