replypier/backend/tests/knowledge/test_crawl4ai_fallback.py

"""
Copyright 2024-2026 ReplyPier

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import asyncio
import os
import socket
import threading
import pytest
from unittest.mock import MagicMock, patch, AsyncMock
from bs4 import BeautifulSoup

from app.knowledge.crawl4ai_fallback import (
    Crawl4AIFallback,
    get_crawl4ai_fallback,
    CRAWL4AI_AVAILABLE
)


# Live integration tests below hit the real network and launch a real Chromium
# browser. On CI (Python 3.12) the event-loop teardown of fetch_with_browser can
# raise "Event loop is closed" during GC and leave a coroutine un-awaited, which
# then fails an unrelated test. Skip them in CI by default; run them locally with
# RUN_LIVE_CRAWL_TESTS=1 to exercise the real path.
_SKIP_LIVE = os.getenv("CI") is not None and os.getenv("RUN_LIVE_CRAWL_TESTS") != "1"
_LIVE_REASON = "live crawl4ai network/browser test (set RUN_LIVE_CRAWL_TESTS=1 to run; skipped in CI)"


@pytest.fixture(autouse=True)
def offline_crawler_dns(monkeypatch):
    """Keep offline cases synthetic without changing the live-test opt-in."""
    if not _SKIP_LIVE:
        return

    def unresolved_host(*args, **kwargs):
        raise socket.gaierror("synthetic test host")

    monkeypatch.setattr(socket, "getaddrinfo", unresolved_host)


@pytest.mark.parametrize("skip_live", [True, False])
def test_offline_dns_fixture_respects_existing_live_mode(monkeypatch, skip_live):
    monkeypatch.setitem(globals(), "_SKIP_LIVE", skip_live)
    resolver = MagicMock(return_value=[])
    monkeypatch.setattr(socket, "getaddrinfo", resolver)

    offline_crawler_dns.__wrapped__(monkeypatch)

    if skip_live:
        with pytest.raises(socket.gaierror, match="synthetic test host"):
            socket.getaddrinfo("crawl.test", 443)
        resolver.assert_not_called()
    else:
        assert socket.getaddrinfo is resolver


class TestCrawl4AIFallback:
    """Test suite for Crawl4AIFallback class"""
    
    def test_init_default_params(self):
        """Test initialization with default parameters"""
        fallback = Crawl4AIFallback()
        
        assert fallback.timeout == 30
        assert fallback.verify_ssl is True
        assert fallback._is_available == CRAWL4AI_AVAILABLE
    
    def test_init_custom_params(self):
        """Test initialization with custom parameters"""
        fallback = Crawl4AIFallback(timeout=60, verify_ssl=False)
        
        assert fallback.timeout == 60
        assert fallback.verify_ssl is False
    
    def test_is_available_property(self):
        """Test is_available property"""
        fallback = Crawl4AIFallback()
        
        assert fallback.is_available == CRAWL4AI_AVAILABLE
    
    @pytest.mark.asyncio
    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
    @patch('app.knowledge.crawl4ai_fallback.AsyncWebCrawler')
    async def test_async_fetch_success(self, mock_crawler_class):
        """Test successful async fetch without screenshot"""
        # Arrange
        url = "https://crawl.test"
        mock_result = MagicMock()
        mock_result.success = True
        mock_result.html = "<html><body>Test content</body></html>"
        mock_result.markdown = MagicMock()
        mock_result.markdown.raw_markdown = "# Test content"
        mock_result.error_message = None
        
        mock_crawler_instance = AsyncMock()
        mock_crawler_instance.arun.return_value = mock_result
        mock_crawler_instance.__aenter__.return_value = mock_crawler_instance
        mock_crawler_instance.__aexit__.return_value = None
        mock_crawler_class.return_value = mock_crawler_instance
        
        fallback = Crawl4AIFallback()
        fallback._is_available = True
        
        # Act
        html, markdown, screenshot = await fallback._async_fetch(url, take_screenshot=False)
        
        # Assert
        assert html == "<html><body>Test content</body></html>"
        assert markdown == "# Test content"
        assert screenshot is None
        mock_crawler_instance.arun.assert_called_once()
    
    @pytest.mark.asyncio
    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
    @patch('app.knowledge.crawl4ai_fallback.AsyncWebCrawler')
    async def test_async_fetch_failure(self, mock_crawler_class):
        """Test failed async fetch"""
        # Arrange
        url = "https://crawl.test"
        mock_result = MagicMock()
        mock_result.success = False
        mock_result.error_message = "Connection timeout"
        
        mock_crawler_instance = AsyncMock()
        mock_crawler_instance.arun.return_value = mock_result
        mock_crawler_instance.__aenter__.return_value = mock_crawler_instance
        mock_crawler_instance.__aexit__.return_value = None
        mock_crawler_class.return_value = mock_crawler_instance
        
        fallback = Crawl4AIFallback()
        fallback._is_available = True
        
        # Act
        html, markdown, screenshot = await fallback._async_fetch(url, take_screenshot=False)
        
        # Assert
        assert html is None
        assert markdown is None
        assert screenshot is None
    
    @pytest.mark.asyncio
    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
    @patch('playwright.async_api.async_playwright')
    @patch('app.knowledge.crawl4ai_fallback.AsyncWebCrawler')
    async def test_async_fetch_with_screenshot(self, mock_crawler_class, mock_playwright):
        """Test async fetch with screenshot enabled"""
        # Arrange
        url = "https://crawl.test"
        
        # Mock Playwright screenshot
        mock_page = AsyncMock()
        mock_page.goto = AsyncMock()
        mock_page.wait_for_timeout = AsyncMock()
        mock_page.screenshot.return_value = b'fake_screenshot_bytes'
        mock_page.content.return_value = "<html><body>Playwright content</body></html>"
        
        mock_context = AsyncMock()
        mock_context.new_page.return_value = mock_page
        
        mock_browser = AsyncMock()
        mock_browser.new_context.return_value = mock_context
        mock_browser.close = AsyncMock()
        
        mock_playwright_instance = AsyncMock()
        mock_playwright_instance.chromium.launch.return_value = mock_browser
        mock_playwright_instance.__aenter__.return_value = mock_playwright_instance
        mock_playwright_instance.__aexit__.return_value = None
        mock_playwright.return_value = mock_playwright_instance
        
        # Mock Crawl4AI
        mock_result = MagicMock()
        mock_result.success = True
        mock_result.html = "<html><body>Crawl4AI content</body></html>"
        mock_result.markdown = MagicMock()
        mock_result.markdown.raw_markdown = "# Crawl4AI content"
        
        mock_crawler_instance = AsyncMock()
        mock_crawler_instance.arun.return_value = mock_result
        mock_crawler_instance.__aenter__.return_value = mock_crawler_instance
        mock_crawler_instance.__aexit__.return_value = None
        mock_crawler_class.return_value = mock_crawler_instance
        
        fallback = Crawl4AIFallback()
        fallback._is_available = True
        
        # Act
        html, markdown, screenshot = await fallback._async_fetch(url, take_screenshot=True)
        
        # Assert
        assert html == "<html><body>Playwright content</body></html>"  # Playwright HTML is used
        assert markdown == "# Crawl4AI content"
        assert screenshot is not None
        assert isinstance(screenshot, str)  # Base64 encoded
        mock_page.screenshot.assert_called_once()
    
    @pytest.mark.asyncio
    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
    @patch('app.knowledge.crawl4ai_fallback.AsyncWebCrawler')
    async def test_async_fetch_exception(self, mock_crawler_class):
        """Test async fetch with exception"""
        # Arrange
        url = "https://crawl.test"
        mock_crawler_class.side_effect = Exception("Network error")
        
        fallback = Crawl4AIFallback()
        fallback._is_available = True
        
        # Act
        html, markdown, screenshot = await fallback._async_fetch(url, take_screenshot=False)
        
        # Assert
        assert html is None
        assert markdown is None
        assert screenshot is None
    
    @pytest.mark.asyncio
    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
    @patch('app.knowledge.crawl4ai_fallback.AsyncWebCrawler')
    async def test_async_fetch_blocks_internal_hosts(self, mock_crawler_class):
        """The browser fallback must refuse internal/link-local targets. Without this
        guard it is an SSRF primitive: a real Chromium fetching cloud metadata and
        internal services, bypassing the httpx path's url_safety checks."""
        from app.knowledge.url_safety import BlockedHostError

        fallback = Crawl4AIFallback()
        fallback._is_available = True

        for url in ("http://169.254.169.254/latest/meta-data/",
                    "http://127.0.0.1:8000/internal",
                    "http://10.0.0.5/admin"):
            with pytest.raises(BlockedHostError):
                await fallback._async_fetch(url, take_screenshot=False)

        mock_crawler_class.assert_not_called()

    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', False)
    def test_fetch_with_browser_not_available(self):
        """Test fetch_with_browser when Crawl4AI is not available"""
        fallback = Crawl4AIFallback()
        fallback._is_available = False
        
        content, soup, screenshot = fallback.fetch_with_browser("https://example.com")
        
        assert content is None
        assert soup is None
        assert screenshot is None
    
    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
    def test_fetch_with_browser_with_markdown(self):
        """Test fetch_with_browser returns markdown when available"""
        fallback = Crawl4AIFallback()
        fallback._is_available = True
        
        # Mock _run_async_safely to return markdown
        html_content = "<html><body>Test</body></html>"
        markdown_content = "# Test Markdown\n\nThis is a long markdown content with more than 100 characters to ensure it's substantial enough."
        
        with patch.object(fallback, '_run_async_safely', return_value=(html_content, markdown_content, None)):
            content, soup, screenshot = fallback.fetch_with_browser("https://example.com")
            
            assert content == markdown_content.strip()
            assert isinstance(soup, BeautifulSoup)
            assert screenshot is None
    
    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
    def test_fetch_with_browser_without_markdown(self):
        """Test fetch_with_browser when markdown is not substantial"""
        fallback = Crawl4AIFallback()
        fallback._is_available = True
        
        # Mock _run_async_safely to return short markdown
        html_content = "<html><body>Test</body></html>"
        markdown_content = "Short"  # Less than 100 chars
        
        with patch.object(fallback, '_run_async_safely', return_value=(html_content, markdown_content, None)):
            content, soup, screenshot = fallback.fetch_with_browser("https://example.com")
            
            assert content is None  # Should return None for extraction from soup
            assert isinstance(soup, BeautifulSoup)
            assert screenshot is None
    
    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
    def test_fetch_with_browser_no_content(self):
        """Test fetch_with_browser when no content is returned"""
        fallback = Crawl4AIFallback()
        fallback._is_available = True
        
        with patch.object(fallback, '_run_async_safely', return_value=(None, None, None)):
            content, soup, screenshot = fallback.fetch_with_browser("https://example.com")
            
            assert content is None
            assert soup is None
            assert screenshot is None
    
    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
    def test_fetch_with_browser_exception(self):
        """Test fetch_with_browser handles exceptions gracefully"""
        fallback = Crawl4AIFallback()
        fallback._is_available = True
        
        with patch.object(fallback, '_run_async_safely', side_effect=Exception("Test error")):
            content, soup, screenshot = fallback.fetch_with_browser("https://example.com")
            
            assert content is None
            assert soup is None
            assert screenshot is None
    
    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
    @patch('asyncio.get_event_loop')
    def test_run_async_safely_no_running_loop(self, mock_get_loop):
        """Test _run_async_safely when no event loop is running"""
        mock_loop = MagicMock()
        mock_loop.is_running.return_value = False
        mock_get_loop.return_value = mock_loop
        
        fallback = Crawl4AIFallback()
        fallback._is_available = True

        expected = ("<html>Test</html>", "# Test", None)
        with patch.object(fallback, '_async_fetch', new=AsyncMock(return_value=expected)) as fetch:
            result = fallback._run_async_safely("https://crawl.test", False)

        assert result == expected
        fetch.assert_awaited_once_with("https://crawl.test", False)
    
    @pytest.mark.asyncio
    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
    async def test_run_async_safely_with_running_loop(self):
        """A running caller loop uses another thread and loop, then joins it."""
        fallback = Crawl4AIFallback()
        fallback._is_available = True
        caller_loop = asyncio.get_running_loop()
        caller_thread = threading.get_ident()
        expected = ("<html>Test</html>", "# Test", None)

        async def fetch_in_worker(url, take_screenshot):
            assert threading.get_ident() != caller_thread
            assert asyncio.get_running_loop() is not caller_loop
            return expected

        with patch.object(fallback, '_async_fetch', new=AsyncMock(side_effect=fetch_in_worker)) as fetch:
            html, markdown, screenshot = fallback._run_async_safely("https://crawl.test", False)

        assert (html, markdown, screenshot) == expected
        fetch.assert_awaited_once_with("https://crawl.test", False)
    
    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
    @patch('asyncio.get_event_loop')
    def test_run_async_safely_exception(self, mock_get_loop):
        """Test _run_async_safely handles exceptions"""
        mock_loop = MagicMock()
        mock_loop.is_running.return_value = False
        mock_get_loop.return_value = mock_loop
        
        fallback = Crawl4AIFallback()
        fallback._is_available = True

        with patch.object(fallback, '_async_fetch', new=AsyncMock(side_effect=RuntimeError("Async execution failed"))):
            html, markdown, screenshot = fallback._run_async_safely("https://crawl.test", False)

        assert html is None
        assert markdown is None
        assert screenshot is None


    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
    def test_run_async_safely_threaded_exception(self):
        """An exception inside the worker thread is re-raised and handled gracefully."""
        fallback = Crawl4AIFallback()
        fallback._is_available = True

        fake_loop = MagicMock()
        fake_loop.is_running.return_value = True

        with patch('asyncio.get_event_loop', return_value=fake_loop), \
             patch.object(fallback, '_async_fetch', new=AsyncMock(side_effect=RuntimeError("boom"))):
            html, markdown, screenshot = fallback._run_async_safely("https://example.com", False)

        assert html is None
        assert markdown is None
        assert screenshot is None


class TestGetCrawl4AIFallback:
    """Test suite for get_crawl4ai_fallback singleton function"""
    
    def test_get_crawl4ai_fallback_creates_instance(self):
        """Test that get_crawl4ai_fallback creates an instance"""
        # Reset singleton
        import app.knowledge.crawl4ai_fallback as module
        module._crawl4ai_fallback_instance = None
        
        instance = get_crawl4ai_fallback()
        
        assert isinstance(instance, Crawl4AIFallback)
        assert instance.timeout == 30
        assert instance.verify_ssl is True
    
    def test_get_crawl4ai_fallback_returns_same_instance(self):
        """Test that get_crawl4ai_fallback returns the same instance (singleton)"""
        # Reset singleton
        import app.knowledge.crawl4ai_fallback as module
        module._crawl4ai_fallback_instance = None
        
        instance1 = get_crawl4ai_fallback()
        instance2 = get_crawl4ai_fallback()
        
        assert instance1 is instance2
    
    def test_get_crawl4ai_fallback_custom_params(self):
        """Test get_crawl4ai_fallback with custom parameters"""
        # Reset singleton
        import app.knowledge.crawl4ai_fallback as module
        module._crawl4ai_fallback_instance = None
        
        instance = get_crawl4ai_fallback(timeout=60, verify_ssl=False)
        
        assert instance.timeout == 60
        assert instance.verify_ssl is False


class TestCrawl4AIIntegration:
    """Integration tests for Crawl4AI (only run if Crawl4AI is available)"""
    
    @pytest.mark.skipif(not CRAWL4AI_AVAILABLE or _SKIP_LIVE, reason=_LIVE_REASON)
    @pytest.mark.asyncio
    async def test_real_async_fetch_simple_page(self):
        """Test real async fetch with a simple page (requires Crawl4AI)"""
        fallback = Crawl4AIFallback(timeout=10)
        
        # Use a simple, reliable test URL
        url = "https://example.com"
        
        try:
            html, markdown, screenshot = await fallback._async_fetch(url, take_screenshot=False)
            
            # Basic assertions
            assert html is not None or markdown is not None
            if html:
                assert len(html) > 0
                assert "example" in html.lower()
        except Exception as e:
            pytest.skip(f"Network or Crawl4AI issue: {str(e)}")
    
    @pytest.mark.skipif(not CRAWL4AI_AVAILABLE or _SKIP_LIVE, reason=_LIVE_REASON)
    def test_real_fetch_with_browser(self):
        """Test real fetch_with_browser (requires Crawl4AI)"""
        fallback = Crawl4AIFallback(timeout=10)
        
        if not fallback.is_available:
            pytest.skip("Crawl4AI not available")
        
        url = "https://example.com"
        
        try:
            content, soup, screenshot = fallback.fetch_with_browser(url, take_screenshot=False)
            
            # Basic assertions
            assert content is not None or soup is not None
            if soup:
                assert isinstance(soup, BeautifulSoup)
        except Exception as e:
            pytest.skip(f"Network or Crawl4AI issue: {str(e)}")