324 lines
14 KiB
Python
324 lines
14 KiB
Python
|
"""
|
||
|
|
Copyright 2024-2026 ReplyPier
|
||
|
|
|
||
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||
|
|
you may not use this file except in compliance with the License.
|
||
|
|
You may obtain a copy of the License at
|
||
|
|
|
||
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
||
|
|
|
||
|
|
Unless required by applicable law or agreed to in writing, software
|
||
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
|
|
See the License for the specific language governing permissions and
|
||
|
|
limitations under the License.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import asyncio
|
||
|
|
from typing import Optional, Tuple
|
||
|
|
from bs4 import BeautifulSoup
|
||
|
|
from app.core.logger import get_logger
|
||
|
|
from app.knowledge.crawler_identity import CRAWLER_USER_AGENT
|
||
|
|
from app.knowledge.url_safety import guard_url
|
||
|
|
|
||
|
|
logger = get_logger(__name__)
|
||
|
|
|
||
|
|
# Try to import crawl4ai
|
||
|
|
try:
|
||
|
|
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
|
||
|
|
CRAWL4AI_AVAILABLE = True
|
||
|
|
logger.info("✓ Crawl4AI module loaded successfully")
|
||
|
|
except ImportError:
|
||
|
|
CRAWL4AI_AVAILABLE = False
|
||
|
|
logger.warning("⚠️ Crawl4AI not installed. Install with: pip install crawl4ai>=0.9")
|
||
|
|
|
||
|
|
# Note: We use threading for async compatibility instead of nest_asyncio
|
||
|
|
# This avoids conflicts with uvloop and other event loop implementations
|
||
|
|
|
||
|
|
|
||
|
|
class Crawl4AIFallback:
|
||
|
|
"""
|
||
|
|
Fallback crawler using Crawl4AI with browser rendering for JavaScript-heavy websites.
|
||
|
|
This provides an alternative when standard HTTP requests fail to extract content.
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self, timeout: int = 30, verify_ssl: bool = True):
|
||
|
|
"""
|
||
|
|
Initialize the Crawl4AI fallback crawler.
|
||
|
|
|
||
|
|
:param timeout: Request timeout in seconds
|
||
|
|
:param verify_ssl: Whether to verify SSL certificates
|
||
|
|
"""
|
||
|
|
self.timeout = timeout
|
||
|
|
self.verify_ssl = verify_ssl
|
||
|
|
self._is_available = CRAWL4AI_AVAILABLE
|
||
|
|
|
||
|
|
@property
|
||
|
|
def is_available(self) -> bool:
|
||
|
|
"""Check if Crawl4AI is available."""
|
||
|
|
return self._is_available
|
||
|
|
|
||
|
|
async def _async_fetch(self, url: str, take_screenshot: bool = False) -> Tuple[Optional[str], Optional[str], Optional[str]]:
|
||
|
|
"""
|
||
|
|
Async method to fetch content using Crawl4AI.
|
||
|
|
|
||
|
|
:param url: The URL to fetch
|
||
|
|
:param take_screenshot: Whether to take a screenshot of the rendered page
|
||
|
|
:return: Tuple of (html, markdown, screenshot_base64) or (None, None, None) if failed
|
||
|
|
:raises BlockedHostError: if the URL targets an internal/blocked host
|
||
|
|
"""
|
||
|
|
# SSRF guard: the httpx crawl path validates through url_safety, but this
|
||
|
|
# browser fallback drives a real Chromium and previously fetched any URL.
|
||
|
|
# Note this covers the initial URL only - redirects and in-page JS navigation
|
||
|
|
# the browser follows on its own need network-layer egress policy too.
|
||
|
|
guard_url(url)
|
||
|
|
|
||
|
|
screenshot_base64 = None
|
||
|
|
html_content = None
|
||
|
|
markdown_content = None
|
||
|
|
|
||
|
|
# If screenshot is needed, use Playwright directly for better control
|
||
|
|
if take_screenshot:
|
||
|
|
try:
|
||
|
|
logger.info(f"📸 Using Playwright directly for viewport screenshot of {url}")
|
||
|
|
from playwright.async_api import async_playwright
|
||
|
|
import base64
|
||
|
|
|
||
|
|
async with async_playwright() as p:
|
||
|
|
browser = await p.chromium.launch(
|
||
|
|
headless=True,
|
||
|
|
args=[
|
||
|
|
"--disable-blink-features=AutomationControlled",
|
||
|
|
"--disable-dev-shm-usage",
|
||
|
|
"--no-sandbox",
|
||
|
|
]
|
||
|
|
)
|
||
|
|
|
||
|
|
context = await browser.new_context(
|
||
|
|
viewport={'width': 1280, 'height': 1024},
|
||
|
|
user_agent=CRAWLER_USER_AGENT,
|
||
|
|
ignore_https_errors=not self.verify_ssl
|
||
|
|
)
|
||
|
|
|
||
|
|
page = await context.new_page()
|
||
|
|
await page.goto(url, wait_until="domcontentloaded", timeout=self.timeout * 1000)
|
||
|
|
|
||
|
|
# Wait for content to render
|
||
|
|
await page.wait_for_timeout(3000) # 3 seconds for JS to execute
|
||
|
|
|
||
|
|
# Take viewport-only screenshot
|
||
|
|
screenshot_bytes = await page.screenshot(type='png', full_page=False)
|
||
|
|
screenshot_base64 = base64.b64encode(screenshot_bytes).decode('utf-8')
|
||
|
|
|
||
|
|
screenshot_size = len(screenshot_base64)
|
||
|
|
logger.info(f"✅ Viewport screenshot captured: {screenshot_size} chars ({screenshot_size // 1024}KB)")
|
||
|
|
|
||
|
|
# Also get the HTML content
|
||
|
|
html_content = await page.content()
|
||
|
|
|
||
|
|
await browser.close()
|
||
|
|
|
||
|
|
except Exception as screenshot_error:
|
||
|
|
logger.error(f"❌ Error with Playwright screenshot: {str(screenshot_error)}", exc_info=True)
|
||
|
|
screenshot_base64 = None
|
||
|
|
|
||
|
|
# Use Crawl4AI for content extraction (more reliable for markdown)
|
||
|
|
try:
|
||
|
|
browser_config = BrowserConfig(
|
||
|
|
headless=True,
|
||
|
|
java_script_enabled=True,
|
||
|
|
text_mode=False,
|
||
|
|
verbose=False,
|
||
|
|
ignore_https_errors=not self.verify_ssl,
|
||
|
|
user_agent=CRAWLER_USER_AGENT,
|
||
|
|
viewport_width=1280,
|
||
|
|
viewport_height=1024,
|
||
|
|
extra_args=[
|
||
|
|
"--disable-blink-features=AutomationControlled",
|
||
|
|
"--disable-dev-shm-usage",
|
||
|
|
"--no-sandbox",
|
||
|
|
]
|
||
|
|
)
|
||
|
|
|
||
|
|
# Wait out short bot-check interstitials (WordPress.com / Cloudflare
|
||
|
|
# "Checking your browser… will only take a few seconds") which
|
||
|
|
# JS-redirect to the real page after a few seconds. Resolve as soon as
|
||
|
|
# the page has substantial real content (>800 chars — fast for normal
|
||
|
|
# pages), OR, on a still-short page, once the visible interstitial
|
||
|
|
# phrases are gone. This actually waits through the redirect instead
|
||
|
|
# of returning the challenge immediately; if it never clears it times
|
||
|
|
# out and the reader skips the page.
|
||
|
|
wait_for_real_content = (
|
||
|
|
"js:() => { const t = document.body ? document.body.innerText : ''; "
|
||
|
|
"if (t.length > 800) return true; "
|
||
|
|
"const low = t.toLowerCase(); "
|
||
|
|
"return t.length > 0 "
|
||
|
|
"&& !low.includes('checking your browser') "
|
||
|
|
"&& !low.includes('just a moment') "
|
||
|
|
"&& !low.includes('secured by wp.com') "
|
||
|
|
"&& !low.includes('verifying you are human'); }"
|
||
|
|
)
|
||
|
|
crawler_config = CrawlerRunConfig(
|
||
|
|
cache_mode=CacheMode.BYPASS,
|
||
|
|
wait_until="domcontentloaded",
|
||
|
|
page_timeout=self.timeout * 1000,
|
||
|
|
wait_for=wait_for_real_content,
|
||
|
|
delay_before_return_html=3.0,
|
||
|
|
screenshot=False,
|
||
|
|
# Anti-bot: simulate a real user and hide automation flags so
|
||
|
|
# WordPress.com / Cloudflare interstitials clear for the headless
|
||
|
|
# browser instead of looping on "Checking your browser…".
|
||
|
|
magic=True,
|
||
|
|
simulate_user=True,
|
||
|
|
override_navigator=True,
|
||
|
|
)
|
||
|
|
|
||
|
|
logger.info(f"🌐 Starting Crawl4AI for content extraction: {url}")
|
||
|
|
|
||
|
|
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||
|
|
result = await crawler.arun(url=url, config=crawler_config)
|
||
|
|
|
||
|
|
if result.success:
|
||
|
|
html_length = len(result.html) if result.html else 0
|
||
|
|
markdown_length = len(result.markdown.raw_markdown) if result.markdown and result.markdown.raw_markdown else 0
|
||
|
|
|
||
|
|
logger.info(f"✓ Crawl4AI content extraction successful - HTML: {html_length} chars, Markdown: {markdown_length} chars")
|
||
|
|
|
||
|
|
# Use Crawl4AI's HTML if we didn't get it from Playwright
|
||
|
|
if not html_content:
|
||
|
|
html_content = result.html
|
||
|
|
|
||
|
|
markdown_content = result.markdown.raw_markdown if result.markdown else None
|
||
|
|
|
||
|
|
return html_content, markdown_content, screenshot_base64
|
||
|
|
else:
|
||
|
|
logger.error(f"❌ Crawl4AI failed: {result.error_message}")
|
||
|
|
# Return whatever we got from Playwright
|
||
|
|
return html_content, None, screenshot_base64
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
logger.error(f"❌ Crawl4AI async fetch error: {str(e)}", exc_info=True)
|
||
|
|
return None, None, None
|
||
|
|
|
||
|
|
def fetch_with_browser(self, url: str, take_screenshot: bool = False) -> Tuple[Optional[str], Optional[BeautifulSoup], Optional[str]]:
|
||
|
|
"""
|
||
|
|
Fetch content using Crawl4AI with browser rendering.
|
||
|
|
This method handles async execution in a thread-safe manner.
|
||
|
|
|
||
|
|
:param url: The URL to fetch
|
||
|
|
:param take_screenshot: Whether to take a screenshot of the rendered page
|
||
|
|
:return: Tuple of (content, soup, screenshot_base64) where content is the extracted text
|
||
|
|
"""
|
||
|
|
if not self._is_available:
|
||
|
|
logger.error(f"❌ Crawl4AI not available. Cannot fetch {url}")
|
||
|
|
return None, None, None
|
||
|
|
|
||
|
|
try:
|
||
|
|
logger.info(f"🔄 Using Crawl4AI fallback for JavaScript-heavy page: {url}")
|
||
|
|
|
||
|
|
# Handle async execution in thread-safe way
|
||
|
|
html, markdown, screenshot = self._run_async_safely(url, take_screenshot)
|
||
|
|
|
||
|
|
if not html and not markdown:
|
||
|
|
logger.warning(f"⚠️ Crawl4AI returned no content for {url}")
|
||
|
|
return None, None, None
|
||
|
|
|
||
|
|
# Parse HTML with BeautifulSoup if available
|
||
|
|
soup = None
|
||
|
|
if html:
|
||
|
|
soup = BeautifulSoup(html, 'html.parser')
|
||
|
|
logger.debug(f"Parsed HTML into BeautifulSoup object ({len(html)} chars)")
|
||
|
|
|
||
|
|
# Prefer markdown if available and substantial
|
||
|
|
if markdown and len(markdown.strip()) > 100:
|
||
|
|
logger.info(f"✓ Using Crawl4AI markdown output ({len(markdown)} chars)")
|
||
|
|
return markdown.strip(), soup, screenshot
|
||
|
|
|
||
|
|
# Otherwise return None for content (will trigger extraction from soup)
|
||
|
|
return None, soup, screenshot
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
logger.error(f"❌ Crawl4AI fetch error for {url}: {str(e)}", exc_info=True)
|
||
|
|
return None, None, None
|
||
|
|
|
||
|
|
def _run_async_safely(self, url: str, take_screenshot: bool = False) -> Tuple[Optional[str], Optional[str], Optional[str]]:
|
||
|
|
"""
|
||
|
|
Run async crawl in a thread-safe manner, handling various event loop scenarios.
|
||
|
|
|
||
|
|
:param url: The URL to fetch
|
||
|
|
:param take_screenshot: Whether to take a screenshot
|
||
|
|
:return: Tuple of (html, markdown, screenshot_base64)
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
# Try to get the current event loop
|
||
|
|
try:
|
||
|
|
loop = asyncio.get_event_loop()
|
||
|
|
loop_is_running = loop.is_running()
|
||
|
|
except RuntimeError:
|
||
|
|
loop_is_running = False
|
||
|
|
|
||
|
|
if loop_is_running:
|
||
|
|
# Event loop is already running (e.g., in FastAPI/uvloop context)
|
||
|
|
# Always use a separate thread for running Crawl4AI to avoid conflicts
|
||
|
|
logger.debug("Event loop running, creating new loop in separate thread")
|
||
|
|
|
||
|
|
import concurrent.futures
|
||
|
|
import threading
|
||
|
|
|
||
|
|
result = [None, None, None]
|
||
|
|
exception = [None]
|
||
|
|
|
||
|
|
def run_in_new_loop():
|
||
|
|
"""Run async fetch in a new event loop in a separate thread"""
|
||
|
|
try:
|
||
|
|
# Create a new event loop for this thread
|
||
|
|
new_loop = asyncio.new_event_loop()
|
||
|
|
asyncio.set_event_loop(new_loop)
|
||
|
|
try:
|
||
|
|
result[0], result[1], result[2] = new_loop.run_until_complete(self._async_fetch(url, take_screenshot))
|
||
|
|
finally:
|
||
|
|
new_loop.close()
|
||
|
|
except Exception as e:
|
||
|
|
exception[0] = e
|
||
|
|
|
||
|
|
# Run in a separate thread
|
||
|
|
thread = threading.Thread(target=run_in_new_loop)
|
||
|
|
thread.start()
|
||
|
|
thread.join(timeout=self.timeout + 10)
|
||
|
|
|
||
|
|
if thread.is_alive():
|
||
|
|
logger.error(f"Thread timeout after {self.timeout + 10}s")
|
||
|
|
return None, None, None
|
||
|
|
|
||
|
|
if exception[0]:
|
||
|
|
raise exception[0]
|
||
|
|
|
||
|
|
return result[0], result[1], result[2]
|
||
|
|
else:
|
||
|
|
# No event loop running, safe to use asyncio.run
|
||
|
|
logger.debug("No event loop running, using asyncio.run")
|
||
|
|
return asyncio.run(self._async_fetch(url, take_screenshot))
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
logger.error(f"❌ Error running async fetch: {str(e)}", exc_info=True)
|
||
|
|
return None, None, None
|
||
|
|
|
||
|
|
|
||
|
|
# Singleton instance for convenience
|
||
|
|
_crawl4ai_fallback_instance = None
|
||
|
|
|
||
|
|
def get_crawl4ai_fallback(timeout: int = 30, verify_ssl: bool = True) -> Crawl4AIFallback:
|
||
|
|
"""
|
||
|
|
Get or create a Crawl4AI fallback instance (singleton pattern).
|
||
|
|
|
||
|
|
:param timeout: Request timeout in seconds
|
||
|
|
:param verify_ssl: Whether to verify SSL certificates
|
||
|
|
:return: Crawl4AIFallback instance
|
||
|
|
"""
|
||
|
|
global _crawl4ai_fallback_instance
|
||
|
|
|
||
|
|
if _crawl4ai_fallback_instance is None:
|
||
|
|
_crawl4ai_fallback_instance = Crawl4AIFallback(timeout=timeout, verify_ssl=verify_ssl)
|
||
|
|
|
||
|
|
return _crawl4ai_fallback_instance
|
||
|
|
|