fix(processors): URL pattern bugs, pipeline hardening, quality polish

- Fix Instagram not matching /{username}/p/ and /{username}/reel/ URLs
- Fix Substack not matching substack.com/home/post/ aggregator URLs
- Fix Reddit not matching hyphenated subreddit names (r/some-sub)
- Fix SE not matching hyphenated site names (code-review.stackexchange)
- Simplify Mastodon pattern (drop protocol requirement for consistency)
- YouTube/TikTok: always set item_type="video" even in fallback paths
- LinkedIn: lower limited-extraction threshold from 250→150 chars
- Threads: lower limited threshold from 50→30 chars (short by design)
- GenericURL: emit structured sections via trafilatura/readability
- Meilisearch: index section_kind/role/author in chunk documents
- Pipeline: catch all exceptions in section re-hydration, preserve provenance
- Add 48 new tests: URL matching (42 URLs × 14 platforms), pipeline
  integration (round-trip provenance, heading paths, boilerplate filtering)

304 tests pass, lint clean, frontend builds clean.

Author: prakersh

src/fourdpocket/processors/generic_url.py

@@ -1,15 +1,15 @@
-"""Generic URL processor - fallback for any URL."""
+"""Generic URL processor - fallback for any URL with section support."""
 
 import httpx
-from readability import Document
 
 from fourdpocket.processors.base import BaseProcessor, ProcessorResult, ProcessorStatus
+from fourdpocket.processors.medium import _trafilatura_or_readability_sections
 from fourdpocket.processors.registry import register_processor
 
 
 @register_processor
 class GenericURLProcessor(BaseProcessor):
-    """Extract content from any URL using readability and metadata parsing."""
+    """Extract content from any URL using trafilatura/readability as sections."""
 
     url_patterns = []  # matches nothing - used as fallback
     priority = -1  # lowest priority
@@ -47,23 +47,15 @@ async def process(self, url: str, **kwargs) -> ProcessorResult:
                 status=ProcessorStatus.partial,
             )
 
-        # Extract readable content via readability
-        try:
-            doc = Document(raw_html)
-            readable_title = doc.title()
-            readable_content = doc.summary()
-            doc.short_title()
-        except Exception:
-            readable_title = None
-            readable_content = None
-
         # Extract OG metadata
         og_meta = self._extract_og_metadata(raw_html)
 
+        # Emit structured sections via trafilatura/readability
+        sections = _trafilatura_or_readability_sections(raw_html, url, og_meta)
+
         # Determine best title
         title = (
             og_meta.get("og_title")
-            or readable_title
             or og_meta.get("html_title")
             or url
         )
@@ -99,11 +91,12 @@ async def process(self, url: str, **kwargs) -> ProcessorResult:
         return ProcessorResult(
             title=title,
             description=description,
-            content=readable_content,
-            raw_content=raw_html[:100000],  # cap raw HTML at 100KB
+            content=None,
+            raw_content=raw_html[:100000],
             media=media,
             metadata=metadata,
             source_platform="generic",
             item_type="url",
             status=ProcessorStatus.success,
+            sections=sections,
         )

src/fourdpocket/processors/instagram.py

@@ -26,7 +26,7 @@
 
 
 def _extract_shortcode(url: str) -> str | None:
-    m = re.search(r"instagram\.com/(?:p|reel|reels)/([A-Za-z0-9_-]+)", url)
+    m = re.search(r"instagram\.com/(?:[^/]+/)?(?:p|reel|reels)/([A-Za-z0-9_-]+)", url)
     return m.group(1) if m else None
 
 
@@ -38,6 +38,8 @@ class InstagramProcessor(BaseProcessor):
         r"instagram\.com/p/[A-Za-z0-9_-]+",
         r"instagram\.com/reel/[A-Za-z0-9_-]+",
         r"instagram\.com/reels/[A-Za-z0-9_-]+",
+        r"instagram\.com/[^/]+/p/[A-Za-z0-9_-]+",
+        r"instagram\.com/[^/]+/reel/[A-Za-z0-9_-]+",
     ]
     priority = 10
 

src/fourdpocket/processors/linkedin.py

@@ -38,7 +38,7 @@
     "Chrome/124.0.0.0 Safari/537.36"
 )
 
-_MIN_BODY_CHARS = 250  # below this we still ship but flag as limited
+_MIN_BODY_CHARS = 150  # below this we still ship but flag as limited
 
 
 @register_processor

src/fourdpocket/processors/mastodon.py

@@ -59,7 +59,7 @@ class MastodonProcessor(BaseProcessor):
     """Mastodon status + reply tree as sections."""
 
     url_patterns = [
-        r"https?://[^/]+/@[^/]+/\d+",
+        r"/@[^/]+/\d+",
     ]
     priority = 8
 

src/fourdpocket/processors/reddit.py

@@ -84,8 +84,8 @@ class RedditProcessor(BaseProcessor):
     """Extract a Reddit submission with threaded comments as sections."""
 
     url_patterns = [
-        r"reddit\.com/r/\w+/comments/",
-        r"old\.reddit\.com/r/\w+/comments/",
+        r"reddit\.com/r/[\w-]+/comments/",
+        r"old\.reddit\.com/r/[\w-]+/comments/",
         r"redd\.it/\w+",
     ]
     priority = 10

src/fourdpocket/processors/stackoverflow.py

@@ -66,7 +66,7 @@ class StackOverflowProcessor(BaseProcessor):
         r"serverfault\.com/questions/\d+",
         r"superuser\.com/questions/\d+",
         r"askubuntu\.com/questions/\d+",
-        r"\w+\.stackexchange\.com/questions/\d+",
+        r"[\w-]+\.stackexchange\.com/questions/\d+",
     ]
     priority = 10
 

src/fourdpocket/processors/substack.py

@@ -21,10 +21,15 @@
 
 
 def _extract_pub_and_slug(url: str) -> tuple[str | None, str | None]:
+    # Standard: https://{pub}.substack.com/p/{slug}
     m = re.match(r"https?://([^.]+)\.substack\.com/p/([\w\-]+)", url)
-    if not m:
-        return None, None
-    return m.group(1), m.group(2)
+    if m:
+        return m.group(1), m.group(2)
+    # Aggregator: https://substack.com/home/post/p-{id}
+    m2 = re.match(r"https?://substack\.com/home/post/([\w\-]+)", url)
+    if m2:
+        return None, m2.group(1)
+    return None, None
 
 
 def _try_substack_api(pub: str, slug: str) -> dict | None:
@@ -99,7 +104,10 @@ def _html_to_sections(html: str, url: str, start_order: int) -> tuple[list[Secti
 class SubstackProcessor(BaseProcessor):
     """Substack post → typed sections via API; trafilatura fallback."""
 
-    url_patterns = [r"[a-z0-9-]+\.substack\.com/p/"]
+    url_patterns = [
+        r"[a-z0-9-]+\.substack\.com/p/",
+        r"substack\.com/home/post/",
+    ]
     priority = 10
 
     async def process(self, url: str, **kwargs) -> ProcessorResult:

src/fourdpocket/processors/threads.py

@@ -96,7 +96,7 @@ async def process(self, url: str, **kwargs) -> ProcessorResult:
             body_text = og_desc
 
         body_text = body_text.strip()
-        limited = len(body_text) < 50
+        limited = len(body_text) < 30
 
         # ─── Sections ───
         sections: list[Section] = []

src/fourdpocket/processors/tiktok.py

@@ -86,7 +86,7 @@ async def _og_fallback(self, url: str, reason: str) -> ProcessorResult:
             media=media,
             metadata={"url": url, "fallback": "og_metadata", "limited_extraction": True},
             source_platform="tiktok",
-            item_type="url",
+            item_type="video",
             status=ProcessorStatus.partial,
             error=reason,
             sections=sections,

src/fourdpocket/processors/youtube.py

@@ -264,7 +264,7 @@ def _flush():
             media=media,
             metadata=metadata,
             source_platform="youtube",
-            item_type="video" if media else "url",
+            item_type="video",
             status=status,
             error=error,
             sections=sections,