""" Copyright 2024-2026 ReplyPier Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ import random import re import socket import threading import time from copy import deepcopy from datetime import datetime from dataclasses import dataclass, field from typing import Dict, List, Set, Tuple, Optional, Callable from urllib.parse import urljoin, urlparse, urldefrag, urlunparse from app.knowledge.crawler_identity import CRAWLER_USER_AGENT from app.knowledge.url_safety import BlockedHostError from app.knowledge.deadline import Deadline from app.knowledge.page_fetch import NotAPage, fetch_page_text import concurrent.futures from concurrent.futures import ThreadPoolExecutor, as_completed from bs4 import BeautifulSoup, Tag import httpx from agno.document.base import Document from agno.document.reader.website_reader import WebsiteReader from app.core.logger import get_logger from app.knowledge.chunking import chunk_page from app.knowledge.crawl4ai_fallback import get_crawl4ai_fallback from app.knowledge.content_summarizer import get_content_summarizer from app.knowledge.crawl_scope import ( DEFAULT_CRAWL_SCOPE, CrawlScope, is_crawlable_url, url_scheme, ) from app.knowledge.domains import registrable_domain from app.knowledge.failure_advice import FAILURE_ADVICE, FailureCause from app.knowledge.main_content import select_main_node # Initialize logger for this module logger = get_logger(__name__) class BotProtectionError(Exception): """Raised when a crawl produced no content because the site's bot protection (Cloudflare / WordPress.com "Checking your browser…" interstitial) blocked the automated browser. Signals the user should add the content manually.""" class EmptyCrawlError(Exception): """Raised when a crawl stored no pages for a reason other than bot protection. A run that indexes nothing is a failure, not a success: without this the queue item completes and the source appears in the dashboard as a healthy source with zero pages, so nobody learns the agent has no content to answer from.""" def _is_name_resolution_error(error: Exception) -> bool: """True when a connection failed because the name has no address. Walks the cause chain for socket.gaierror rather than matching on the message, which would also catch any other error whose text happens to mention a "name" and wrongly skip the browser fallback for it. """ seen = set() current = error while current is not None and id(current) not in seen: seen.add(id(current)) if isinstance(current, socket.gaierror): return True current = current.__cause__ or current.__context__ text = str(error).lower() return 'nodename' in text or 'name or service not known' in text or 'name resolution' in text @dataclass class EnhancedWebsiteReader(WebsiteReader): """Enhanced Reader for Websites with more robust content extraction""" debug_on = True # Additional configuration options min_content_length: int = 100 # Minimum length of text to be considered meaningful content # Only blacklist truly problematic tags - keep structural elements that may have useful content blacklist_tags: List[str] = field(default_factory=lambda: [ 'script', 'style', 'noscript', 'iframe', 'head' ]) user_agent: str = CRAWLER_USER_AGENT headers: Dict[str, str] = field(default_factory=lambda: { 'User-Agent': CRAWLER_USER_AGENT, 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.9', 'Accept-Encoding': 'gzip, deflate, br', 'Connection': 'keep-alive', 'Upgrade-Insecure-Requests': '1', 'DNT': '1', 'Sec-Fetch-Dest': 'document', 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-Site': 'none', 'Sec-Fetch-User': '?1', 'Cache-Control': 'max-age=0' }) timeout: int = 30 # Request timeout in seconds max_retries: int = 3 # Maximum number of retries for failed requests respect_robots_txt: bool = True # Whether to respect robots.txt max_workers: int = 10 # Maximum number of parallel workers for crawling # How far the crawl may wander from its seed URL: 'host' (default), 'path' # (seed host + seed path prefix) or 'domain' (every host on the registrable # domain). See app/knowledge/crawl_scope.py. crawl_scope: str = DEFAULT_CRAWL_SCOPE verify_ssl: bool = True # Whether to verify SSL certificates (set to False for self-signed certs) # Track crawling statistics _crawled_pages_count: int = 0 _successful_crawls: int = 0 _failed_crawls: int = 0 _challenge_blocked: int = 0 # Pages skipped because a bot-check couldn't be cleared # Why pages failed, as {FailureCause: count}. One string covering "empty, # JavaScript, or unreachable" told the user nothing and told us nothing; # _process_url already knows which of those it hit, so keep it. _failure_causes: Dict[str, int] = field(default_factory=dict) # Pages are crawled on up to max_workers threads, and "read, add one, write # back" loses increments when two of them fail at once. _failure_lock: threading.Lock = field(default_factory=threading.Lock, repr=False) _current_url: str = None # Track current URL being processed for link resolution # Set by the knowledge base for the source being read: no new pages are # fetched once it passes (see app/knowledge/deadline.py). deadline: Optional[Deadline] = None @staticmethod def _classify_transport_error(error: Exception) -> Tuple[str, bool, bool]: """Map a transport failure to (cause, retry_is_useful, browser_may_help). Retrying only pays when the failure could plausibly differ next time. A domain with no address and a host that drops the SYN both fail the same way every attempt, so the old ladder of three tries with backoff plus a headless browser spent 130 seconds on mateusoliveira.com and ~10 on each dead domain, holding one of only two queue slots the whole time. The browser shares our DNS and TCP stack, so it cannot reach what we could not connect to — it is worth a turn only when the connection itself was refused, which can be a TLS quirk rather than a dead host. """ if isinstance(error, httpx.ConnectTimeout): # No SYN-ACK. Waiting longer only waits longer. return FailureCause.TIMEOUT, False, False if isinstance(error, httpx.ConnectError): if _is_name_resolution_error(error): # Chromium reports ERR_NAME_NOT_RESOLVED for exactly these. return FailureCause.DNS, False, False return FailureCause.CONNECTION, False, True if isinstance(error, httpx.PoolTimeout): # Our own connection pool was saturated; the site never heard from # us. Retry, but do not report it as the site being slow. return FailureCause.UNKNOWN, True, True if isinstance(error, httpx.TimeoutException): # Connected, then went quiet: a slow page really may load next time. return FailureCause.TIMEOUT, True, True if isinstance(error, httpx.RequestError): return FailureCause.CONNECTION, True, True # Not a transport failure: a bug in our own parsing, most likely. Retry # and let the browser try, but do not tell the user their site is down. return FailureCause.UNKNOWN, True, True @property def out_of_time(self) -> bool: return self.deadline is not None and self.deadline.expired def _record_failure(self, cause: str) -> None: """Remember why a page failed so the queue item can be specific.""" with self._failure_lock: self._failure_causes[cause] = self._failure_causes.get(cause, 0) + 1 def dominant_failure_cause(self) -> Optional[str]: """The cause that accounts for the most failed pages, if any.""" if not self._failure_causes: return None return max(self._failure_causes.items(), key=lambda item: item[1])[0] def _normalize_url(self, url: str) -> str: """ Normalize URL by ensuring it has a proper protocol (http:// or https://). If no protocol is present, https:// is added by default. A URL that already carries a scheme is returned untouched — including a non-HTTP one such as ``mailto:``. Prefixing those produced ``https://mailto:a@example.com``, which parses as host ``example.com`` with ``mailto:a`` as userinfo and so was crawled as a real page. :param url: The URL to normalize. :return: The normalized URL with protocol. """ if not url: return url url = url.strip() # Check if URL already has a protocol if url_scheme(url): return url # Add https:// by default if no protocol is present logger.debug(f"URL '{url}' is missing protocol, adding 'https://'") return f"https://{url}" @staticmethod def _host_resolves(host: str) -> bool: """Whether a hostname has an address. False for any failure to find one. Strict on purpose: the two callers want opposite defaults from a lookup that errors oddly (a malformed IDN host, say), and treating "could not tell" as success would let the apex check pass a host it never verified. Failing closed means an unverifiable pair is simply left alone. """ try: socket.getaddrinfo(host, None) return True except Exception: return False def _seed_with_resolvable_host(self, url: str) -> str: """Swap a dead ``www`` seed for its apex when only the apex resolves. utsabzone.store serves its site on the apex but has a broken CNAME on ``www``, so a customer who pasted the ``www`` address was told the domain does not resolve — true of the name they gave us, useless as advice, and the site was up the whole time. Only the seed is rewritten. CrawlScope already treats ``www.x`` and ``x`` as one host, so the crawl stays inside the same boundary, and the fetch still runs the SSRF guard on whichever host we end up asking for. """ parsed = urlparse(url) host = (parsed.hostname or "").lower() if not host.startswith("www."): return url # Only pay for a lookup once we know there is an alternative to try. if self._host_resolves(host): return url apex = host[4:] if not apex or not self._host_resolves(apex): return url netloc = parsed.netloc if "@" in netloc or not netloc.lower().startswith(host): # Credentials or an unexpected shape: do not rewrite it. return url swapped = parsed._replace(netloc=apex + netloc[len(host):]).geturl() logger.warning( f"{host} has no address; crawling {apex} instead (the apex resolves)" ) return swapped def _canonical_url(self, url: str) -> str: """Canonicalize a URL so variants of the same page collapse to one id. Drops the ``#fragment`` and any query string, and strips a trailing slash (the root path stays as-is). Without this the same page linked as ``/``, ``/#features`` and ``/#pricing`` would be crawled and stored three times under distinct ids, producing duplicate sub-pages. """ if not url: return url url = urldefrag(url).url # remove #fragment parsed = urlparse(url) path = parsed.path.rstrip('/') or ('/' if not parsed.netloc else '') return urlunparse((parsed.scheme, parsed.netloc, path, '', '', '')) # Unambiguous bot-challenge / interstitial signatures. _STRONG_CHALLENGE_MARKERS = ( "secured by wp.com", "cf-browser-verification", "cf_chl_", "ddos protection by cloudflare", "verifying you are human", "attention required! | cloudflare", ) # Weaker signatures — only treated as a challenge on a short page, since these # phrases can legitimately appear inside real content. _WEAK_CHALLENGE_MARKERS = ( "checking your browser", "just a moment", "please enable javascript", "please turn javascript on", "enable cookies and reload", "please stand by, while we are checking", ) def _looks_like_bot_challenge(self, text: str) -> bool: """True if the extracted text is a bot-check/interstitial, not real content. Sites behind Cloudflare / WordPress.com serve a short "Checking your browser…" page to non-browser clients. Its text is just long enough to pass the min-length check, so without this it would be stored as the page's content. """ if not text: return False lowered = text.lower() if any(marker in lowered for marker in self._STRONG_CHALLENGE_MARKERS): return True if len(text) < 500 and any(marker in lowered for marker in self._WEAK_CHALLENGE_MARKERS): return True return False def _get_primary_domain(self, url: str) -> str: """ Extract primary domain from the given URL. Overrides the parent method to handle domains more effectively. :param url: The URL to extract the primary domain from. :return: The primary domain. """ # Delegate to the shared registrable-domain helper (ccTLD-aware, # port/userinfo stripped via hostname) so every same-domain check in # the codebase agrees. parsed_url = urlparse(self._normalize_url(url)) return registrable_domain(parsed_url.hostname or parsed_url.netloc or "") @staticmethod def _page_title(soup: BeautifulSoup) -> str: """The page's own title,

first and second. Must run BEFORE _clean_soup, which strips <head> and takes <title> with it — a page with no <h1> would otherwise have no title to fall back to. """ heading = soup.find('h1') if heading: title = heading.get_text(" ", strip=True) if title: return title tag = soup.find('title') return tag.get_text(strip=True) if tag else "" @staticmethod def _prepend_title(title: str, content: str) -> str: """Keep the title when selection left it outside the content container. Mintlify and similar docs themes put the <h1> in a page <header>, which selection correctly treats as chrome — but the title is the strongest retrieval signal a page has, so dropping it makes the page harder for an agent to find rather than merely shorter. """ if title and title not in content: return f"{title}\n\n{content}" return content def _extract_main_content(self, soup: BeautifulSoup) -> str: """ Extracts the main content from a BeautifulSoup object. Selection is shared with the help-center importer (see main_content.py): the deepest container that still holds essentially the whole page. If it yields too little text the page is thin or JavaScript-rendered, and the progressively looser fallbacks below get a turn — falling through here is also what routes such a page to the browser fallback in _process_url. :param soup: The BeautifulSoup object to extract the main content from. :return: The main content as a string. """ # Read the title before cleaning: _clean_soup strips <head>. page_title = self._page_title(soup) # Remove undesirable elements first self._clean_soup(soup) main_node = select_main_node(soup, min_chars=self.min_content_length) if main_node is None: # Nothing but navigation chrome. The looser fallbacks below would # happily return that chrome as the page's content — which is how # www.solcontrol.ca came to store "Welcome, Guest - Login" and, worse, # cleared the floor and so never reached the browser fallback that can # actually render it. Report empty and let _process_url try Crawl4AI. logger.warning("No readable content on the page (navigation chrome only)") return "" content = self._get_clean_text(main_node, include_links=True, base_url=self._current_url) if len(content) >= self.min_content_length: content = self._prepend_title(page_title, content) logger.info(f"✓ Content extracted from <{main_node.name}> ({len(content)} chars)") return content logger.debug(f" Main content node too short: {len(content)} chars") # Fallback 1: Density-based content extraction logger.debug(f"Trying fallback: text density") density_content = self._extract_by_text_density(soup) if density_content and len(density_content) >= self.min_content_length: logger.info(f"✓ Content extracted using density strategy ({len(density_content)} chars)") return density_content # Fallback 2: Collect all meaningful text elements (headings, paragraphs, lists, etc.) logger.debug(f"Trying fallback: all meaningful content (tables, headings, paragraphs, lists)") meaningful_content = self._extract_all_meaningful_content(soup) if meaningful_content and len(meaningful_content) >= self.min_content_length: logger.debug(f"✓ Content extracted using meaningful content strategy ({len(meaningful_content)} chars)") return meaningful_content else: logger.debug(f" Meaningful content too short: {len(meaningful_content) if meaningful_content else 0} chars") # Fallback 3: Entire body content with minimal cleaning logger.debug(f"Trying fallback: body") body = soup.find('body') if body: content = self._get_clean_text(body, include_links=True, base_url=self._current_url) if content: logger.info(f"✓ Content extracted using body fallback strategy (length: {len(content)})") return content else: logger.warning(f"Body element found but no text extracted") else: logger.warning(f"No body element found in HTML") # Last resort: just get all text from the document logger.debug(f"Trying last resort: all text from document") content = soup.get_text(strip=True, separator=" ") if content: logger.info(f"✓ Content extracted using last resort strategy (length: {len(content)})") else: logger.error(f"Failed to extract any text from HTML document") return content def _clean_soup(self, soup: BeautifulSoup) -> None: """ Removes undesirable elements from the soup. IMPORTANT: Only remove elements that are truly undesirable (scripts, styles, etc.) Keep structural elements like nav, footer, header, aside as they may contain useful content. :param soup: The BeautifulSoup object to clean. """ removed_count = 0 # Remove comments for comment in soup.find_all(string=lambda text: isinstance(text, str) and text.strip().startswith('<!--')): comment.extract() removed_count += 1 # Only remove truly problematic tags (scripts, styles, iframes) # DO NOT remove structural tags like nav, footer, header, aside minimal_blacklist = ['script', 'style', 'noscript', 'iframe', 'head'] for tag in minimal_blacklist: elements = soup.find_all(tag) removed_count += len(elements) for element in elements: element.extract() # Remove hidden elements with style attribute (but be careful) hidden_elements = soup.find_all(style=re.compile(r'display:\s*none|visibility:\s*hidden')) removed_count += len(hidden_elements) for element in hidden_elements: element.extract() # Remove elements with hidden or screen-reader-text class for class_name in ['hidden', 'screen-reader-text']: hidden_class_elements = soup.find_all(class_=class_name) removed_count += len(hidden_class_elements) for element in hidden_class_elements: element.extract() # Only remove obvious ad/tracking elements for class_name in ['ads', 'advertisement', 'cookie']: elements = soup.find_all(class_=re.compile(class_name, re.IGNORECASE)) removed_count += len(elements) for element in elements: element.extract() # Remove comment sections - these often contain user comments that aren't main content # Look for common comment container patterns comment_selectors = [ {'id': re.compile(r'comment', re.IGNORECASE)}, {'class_': re.compile(r'comment', re.IGNORECASE)}, {'id': 'respond'}, {'class_': 'comment-respond'}, {'class_': 'comment-form'}, {'class_': 'comment-list'}, {'class_': 'comment-body'}, {'class_': 'comments-area'}, ] for selector in comment_selectors: comment_elements = soup.find_all(**selector) if comment_elements: logger.debug(f" Removing {len(comment_elements)} comment elements matching {selector}") removed_count += len(comment_elements) for element in comment_elements: element.extract() # Remove navigation menus - these are not main content nav_selectors = [ 'nav', {'role': 'navigation'}, {'class_': re.compile(r'navigation|menu|nav-', re.IGNORECASE)}, {'id': re.compile(r'navigation|menu|nav-', re.IGNORECASE)}, ] for selector in nav_selectors: if isinstance(selector, str): nav_elements = soup.find_all(selector) else: nav_elements = soup.find_all(**selector) if nav_elements: logger.debug(f" Removing {len(nav_elements)} navigation elements matching {selector}") removed_count += len(nav_elements) for element in nav_elements: element.extract() # Remove sidebars and widgets - often contain non-essential content sidebar_selectors = [ {'id': re.compile(r'sidebar|widget', re.IGNORECASE)}, {'class_': re.compile(r'sidebar|widget', re.IGNORECASE)}, ] for selector in sidebar_selectors: sidebar_elements = soup.find_all(**selector) if sidebar_elements: logger.debug(f" Removing {len(sidebar_elements)} sidebar/widget elements matching {selector}") removed_count += len(sidebar_elements) for element in sidebar_elements: element.extract() logger.debug(f"Cleaned soup: removed {removed_count} elements (minimal cleaning mode)") def _get_clean_text(self, element: Tag, include_links: bool = True, base_url: str = None) -> str: """ Gets clean text from a BeautifulSoup element, optionally including URLs. :param element: The BeautifulSoup element to get text from. :param include_links: If True, appends URLs after link text in format [text](url) :param base_url: Base URL for converting relative URLs to absolute URLs :return: The clean text. """ if not element: return "" if include_links: # Create a deep copy to avoid modifying the original element_copy = deepcopy(element) # Find all links and append their URLs to the text for link in element_copy.find_all('a', href=True): href = link.get('href', '') # Skip anchor links and javascript links if href and not href.startswith('#') and not href.startswith('javascript:'): # Convert relative URLs to absolute URLs if base_url is provided if base_url and not href.startswith(('http://', 'https://', 'mailto:')): href = urljoin(base_url, href) # Get the link text link_text = link.get_text(strip=True) if link_text: # Only add URL if there's text # Create a new text node with the URL appended # Format: "text (URL: https://...)" new_text = f"{link_text} (URL: {href})" link.string = new_text # Get text with preserved whitespace text = element_copy.get_text(separator=" ", strip=True) else: # Get text with preserved whitespace text = element.get_text(separator=" ", strip=True) # Normalize whitespace text = re.sub(r'\s+', ' ', text) # Remove excessive newlines text = re.sub(r'\n{2,}', '\n', text) return text.strip() def _extract_all_meaningful_content(self, soup: BeautifulSoup) -> str: """ Extracts all meaningful content from the page (headings, paragraphs, lists, tables, etc.). This is useful for pages that don't have clear content containers. :param soup: The BeautifulSoup object to extract content from. :return: The extracted meaningful content. """ # Focus on block-level content elements to avoid duplication from nested inline elements meaningful_tags = ['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'li', 'td', 'th', 'blockquote', 'pre'] content_parts = [] tag_counts = {} for tag in meaningful_tags: elements = soup.find_all(tag) count = 0 for element in elements: text = element.get_text(strip=True) # Only include text that's substantial (more than 10 characters) if len(text) > 10: content_parts.append(text) count += 1 if count > 0: tag_counts[tag] = count logger.debug(f" Meaningful elements found: {tag_counts}") logger.debug(f" Total text parts collected: {len(content_parts)}") if not content_parts: return "" # Join all parts with space and clean up full_content = " ".join(content_parts) full_content = re.sub(r'\s+', ' ', full_content) return full_content.strip() def _extract_by_text_density(self, soup: BeautifulSoup) -> str: """ Extracts content based on text density (paragraphs with substantial text). :param soup: The BeautifulSoup object to extract content from. :return: The extracted content. """ paragraphs = soup.find_all('p') if not paragraphs: return "" # Find paragraphs with substantial text good_paragraphs = [] for p in paragraphs: text = p.get_text(strip=True) # Consider paragraphs with more than 20 characters if len(text) > 20: good_paragraphs.append(p) if not good_paragraphs: return "" # Find the most common parent that contains a good number of paragraphs parent_counts = {} for p in good_paragraphs: for parent in p.parents: if parent.name not in ['html', 'body']: parent_counts[parent] = parent_counts.get(parent, 0) + 1 # Get the parent with the most children if parent_counts: best_parent = max(parent_counts.items(), key=lambda x: x[1])[0] return self._get_clean_text(best_parent, include_links=True, base_url=self._current_url) # If no good parent found, just concatenate the good paragraphs return " ".join([self._get_clean_text(p, include_links=True, base_url=self._current_url) for p in good_paragraphs]) def _process_url(self, url_info: Tuple[str, int], scope: CrawlScope) -> Optional[Tuple[str, str, List[str]]]: """ Process a single URL - fetch content and extract links. This is used for parallel processing of URLs. :param url_info: Tuple of (URL, depth) :param scope: Crawl boundary the URL must fall inside :return: Tuple of (URL, content, new_links) or None if failed """ current_url, current_depth = url_info # Normalize URL to ensure it has a protocol current_url = self._normalize_url(current_url) # Skip if URL meets any skip conditions if current_url in self._visited: return None # In-scope http(s) URLs only — the per-URL guard that also protects # sitemap-seeded crawls and anything left over in an older queue. if not scope.allows(current_url): logger.debug(f"Skipping out-of-scope URL {current_url} ({scope.describe()})") return None if current_depth > self.max_depth: return None # Mark as visited before processing self._visited.add(current_url) self.delay() # Increment crawled pages counter self._crawled_pages_count += 1 page_start_time = time.time() logger.info(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Crawling page {self._crawled_pages_count}: {current_url} (depth: {current_depth})") # Try to get content from URL with retries content = None new_links = [] retry_count = 0 last_error = None last_cause = None browser_can_help = True is_javascript_heavy = False soup = None while retry_count < self.max_retries and content is None: try: logger.info(f"Attempt {retry_count + 1}/{self.max_retries} to fetch {current_url}") # Warn if SSL verification is disabled if not self.verify_ssl and retry_count == 0: logger.warning(f"⚠️ SSL verification is disabled for {current_url} - This may pose security risks") # Make the request with httpx.Client( timeout=self.timeout, follow_redirects=True, headers=self.headers, verify=self.verify_ssl ) as client: logger.debug(f"Making HTTP GET request to {current_url}") # Follows redirects manually, re-validating each hop's host # (SSRF), and reads the body only if it is a page. html = fetch_page_text(client, current_url) # Parse HTML with BeautifulSoup soup = BeautifulSoup(html, 'html.parser') logger.info(f"HTML length: {len(html)} characters") # Check if the page is JavaScript-heavy # Modern sites use many external scripts, so count both inline and external script_tags = soup.find_all('script') inline_script_length = sum(len(str(script)) for script in script_tags) external_scripts = [script for script in script_tags if script.get('src')] logger.info(f"Found {len(script_tags)} script tags ({len(external_scripts)} external, {len(script_tags) - len(external_scripts)} inline), inline content: {inline_script_length} characters") # Log page structure for debugging body = soup.find('body') if body: body_children = list(body.children) logger.info(f"Body found with {len(body_children)} direct children") # Log child element types child_types = {} for child in body_children: if hasattr(child, 'name') and child.name: child_types[child.name] = child_types.get(child.name, 0) + 1 logger.info(f"Body children types: {child_types}") else: logger.warning(f"No body tag found in HTML for {current_url}") # Detect JavaScript-heavy pages using multiple criteria: # 1. High ratio of inline script content (>30% of HTML) # 2. Many external script files (>15 scripts typically indicates a JS framework) # 3. High script tag density (>10 scripts per 10KB of HTML) script_content_ratio = inline_script_length / len(html) if html else 0 html_size_kb = len(html) / 10000 # Convert to 10KB units script_density = len(script_tags) / max(html_size_kb, 1) # Scripts per 10KB is_javascript_heavy = ( script_content_ratio > 0.3 or # >30% inline scripts len(external_scripts) >= 15 or # Many external scripts (React/Vue apps) script_density > 10 # High density of scripts ) if is_javascript_heavy: detection_reason = [] if script_content_ratio > 0.3: detection_reason.append(f"high inline script ratio ({script_content_ratio*100:.1f}%)") if len(external_scripts) >= 15: detection_reason.append(f"many external scripts ({len(external_scripts)})") if script_density > 10: detection_reason.append(f"high script density ({script_density:.1f} per 10KB)") logger.warning(f"⚠️ JavaScript-heavy website detected: {', '.join(detection_reason)}") # Set current URL for link resolution self._current_url = current_url content = self._extract_main_content(soup) # Log extracted content length for debugging logger.info(f"Extracted content length: {len(content) if content else 0} characters from {current_url}") # A bot-challenge interstitial (wp.com / Cloudflare "Checking your # browser…") is short enough to pass the length check, so detect it # explicitly and route it through the JS browser, which can clear it. is_challenge = self._looks_like_bot_challenge(content) if is_challenge: logger.warning(f"🛡️ Bot-challenge interstitial detected for {current_url} - routing through Crawl4AI") # For JavaScript-heavy sites (or a detected challenge), try Crawl4AI # for real content. Even static extraction may just be interstitials. if is_javascript_heavy or is_challenge: logger.info(f"🔄 Attempting Crawl4AI for better content extraction: {current_url}") crawl4ai = get_crawl4ai_fallback(timeout=self.timeout, verify_ssl=self.verify_ssl) if crawl4ai.is_available: logger.info(f"🔄 Using Crawl4AI for JavaScript rendering: {current_url}") # Note: fetch_with_browser returns (content, soup, screenshot) crawl4ai_content, crawl4ai_soup, _ = crawl4ai.fetch_with_browser(current_url, take_screenshot=False) if crawl4ai_content and len(crawl4ai_content) >= self.min_content_length: logger.info(f"✓ Crawl4AI extracted {len(crawl4ai_content)} chars (vs {len(content) if content else 0} from BeautifulSoup)") content = crawl4ai_content soup = crawl4ai_soup if crawl4ai_soup else soup elif crawl4ai_soup: # Try to extract from crawl4ai soup self._current_url = current_url crawl4ai_extracted = self._extract_main_content(crawl4ai_soup) if crawl4ai_extracted and len(crawl4ai_extracted) >= self.min_content_length: logger.info(f"✓ Extracted {len(crawl4ai_extracted)} chars from Crawl4AI HTML") content = crawl4ai_extracted soup = crawl4ai_soup else: logger.warning(f"Crawl4AI didn't improve content extraction, keeping BeautifulSoup result") else: logger.warning(f"Crawl4AI extraction failed, keeping BeautifulSoup result") else: logger.warning(f"⚠️ Crawl4AI not available for JS-heavy site. Install with: pip install crawl4ai>=0.9") logger.warning(f" Falling back to BeautifulSoup (may have incomplete content)") # If the content is still a bot-challenge interstitial (the browser # fallback couldn't clear it or wasn't available), skip the page # rather than storing the "Checking your browser…" text as content. if self._looks_like_bot_challenge(content): logger.warning(f"⚠️ Skipping {current_url}: bot-challenge interstitial could not be cleared") self._failed_crawls += 1 self._challenge_blocked += 1 self._record_failure(FailureCause.BOT_BLOCKED) return None # Check content quality if not content or len(content) < self.min_content_length: logger.warning(f"Content too short or empty ({len(content) if content else 0} chars) for {current_url}. Min required: {self.min_content_length}") if content: logger.warning(f"Content preview: {content[:200]}...") else: logger.warning(f"Content is None or empty. HTML preview: {html[:500]}...") # Try to extract any text as last resort fallback_content = soup.get_text(strip=True, separator=" ") if fallback_content and len(fallback_content) >= self.min_content_length: logger.info(f"Using fallback extraction: {len(fallback_content)} characters") content = fallback_content else: logger.warning(f"Standard extraction failed. Fallback content: {len(fallback_content) if fallback_content else 0} chars") # Try Crawl4AI fallback whenever content extraction fails # (regardless of whether page is JavaScript-heavy or not) crawl4ai = get_crawl4ai_fallback(timeout=self.timeout, verify_ssl=self.verify_ssl) if crawl4ai.is_available: # Log reason for using Crawl4AI if is_javascript_heavy: logger.info(f"🔄 Attempting Crawl4AI fallback - JavaScript-heavy page detected: {current_url}") else: logger.info(f"🔄 Attempting Crawl4AI fallback - Standard extraction failed: {current_url}") # Note: fetch_with_browser returns (content, soup, screenshot) # We don't need screenshot for backend crawling, so ignore it crawl4ai_content, crawl4ai_soup, _ = crawl4ai.fetch_with_browser(current_url, take_screenshot=False) if crawl4ai_content and len(crawl4ai_content) >= self.min_content_length: logger.info(f"✓ Crawl4AI successfully extracted {len(crawl4ai_content)} chars") content = crawl4ai_content soup = crawl4ai_soup if crawl4ai_soup else soup elif crawl4ai_soup: # Try to extract from crawl4ai soup self._current_url = current_url content = self._extract_main_content(crawl4ai_soup) if content and len(content) >= self.min_content_length: logger.info(f"✓ Extracted {len(content)} chars from Crawl4AI HTML") soup = crawl4ai_soup else: logger.error(f"Crawl4AI extraction also failed for {current_url}") else: logger.error(f"Crawl4AI fallback failed for {current_url}") else: logger.warning(f"⚠️ Crawl4AI not available. Install with: pip install crawl4ai>=0.9") # Final check if not content or len(content) < self.min_content_length: logger.error(f"All extraction strategies failed for {current_url}. Final content: {len(content) if content else 0} chars") self._failed_crawls += 1 self._record_failure( FailureCause.EMPTY_RESPONSE if not html else FailureCause.THIN_CONTENT ) return None self._successful_crawls += 1 logger.info(f"✓ Successfully extracted {len(content)} chars from {current_url}") # Extract new links if not at max depth if current_depth < self.max_depth: links = self._extract_links(soup, current_url, scope) next_depth = current_depth + 1 new_links = [(link, next_depth) for link in links if link not in self._visited] except NotAPage as e: # A download the page linked to (an installer, an archive): not # content, and not the site failing. Never worth the browser. logger.info(f"Skipping {current_url}: {e}") if e.download and current_depth == 1: # The address the user gave (or a sitemap listed): if # nothing else is read, this is why. self._record_failure(FailureCause.NOT_A_PAGE) return None except BlockedHostError as e: # A redirect pointed at an internal host — abort, don't retry or # fall back to the browser (which would also reach it). logger.warning(str(e)) self._failed_crawls += 1 self._record_failure(FailureCause.BLOCKED_HOST) return None except httpx.HTTPStatusError as e: retry_count += 1 status_code = e.response.status_code last_error = f"HTTP {status_code}: {e.response.reason_phrase}" logger.warning(f"HTTP error on attempt {retry_count}/{self.max_retries} for {current_url}: {last_error}") # 401/403/429 are the shapes bot protection takes; they need the # "add it manually" advice rather than "check the URL". last_cause = FailureCause.BOT_BLOCKED if status_code in (401, 403, 429) else FailureCause.HTTP_ERROR # Special handling for 403 Forbidden (bot detection) if status_code == 403: logger.warning(f"⚠️ 403 Forbidden - Server may be blocking bot requests") # Don't retry httpx for 403, go straight to Crawl4AI fallback retry_count = self.max_retries elif retry_count < self.max_retries: # Exponential backoff for other errors sleep_time = 2 ** retry_count logger.info(f"Retrying in {sleep_time} seconds...") time.sleep(sleep_time) except Exception as e: retry_count += 1 last_cause, retry_is_useful, browser_may_help = self._classify_transport_error(e) browser_can_help = browser_may_help if isinstance(e, httpx.TimeoutException): last_error = f"Request timeout after {self.timeout}s" logger.warning(f"Timeout on attempt {retry_count}/{self.max_retries} for {current_url}") else: last_error = f"{type(e).__name__}: {str(e)}" logger.error( f"Error on attempt {retry_count}/{self.max_retries} for {current_url}: {last_error}", exc_info=True, ) if not retry_is_useful: logger.info(f"Not retrying {current_url}: {last_cause} fails the same way every attempt") break # Exponential backoff if retry_count < self.max_retries: sleep_time = 2 ** retry_count logger.info(f"Retrying in {sleep_time} seconds...") time.sleep(sleep_time) # If all retries failed, try Crawl4AI fallback as last resort — but not # when we never reached the host: Chromium shares our resolver and TCP # stack, so it fails the same way after another browser launch. if content is None and last_error and not browser_can_help: self._failed_crawls += 1 self._record_failure(last_cause or FailureCause.CONNECTION) logger.error(f"❌ {current_url} is unreachable ({last_error}); skipping the browser fallback") return None if content is None and last_error: logger.warning(f"⚠️ All extraction attempts failed. Last error: {last_error}") # Try Crawl4AI fallback for HTTP errors (403, 429, etc.) and other failures crawl4ai = get_crawl4ai_fallback(timeout=self.timeout, verify_ssl=self.verify_ssl) if crawl4ai.is_available: logger.info(f"🔄 Attempting Crawl4AI fallback after HTTP/extraction failure: {current_url}") logger.info(f" Reason: {last_error}") try: # Note: fetch_with_browser returns (content, soup, screenshot) # We don't need screenshot for backend crawling, so ignore it crawl4ai_content, crawl4ai_soup, _ = crawl4ai.fetch_with_browser(current_url, take_screenshot=False) if crawl4ai_content and len(crawl4ai_content) >= self.min_content_length: logger.info(f"✓ Crawl4AI successfully bypassed error and extracted {len(crawl4ai_content)} chars") content = crawl4ai_content soup = crawl4ai_soup if crawl4ai_soup else soup # Extract links if we got content if current_depth < self.max_depth and soup: links = self._extract_links(soup, current_url, scope) next_depth = current_depth + 1 new_links = [(link, next_depth) for link in links if link not in self._visited] self._successful_crawls += 1 page_end_time = time.time() page_duration = page_end_time - page_start_time logger.info(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Completed crawling page {self._crawled_pages_count} with Crawl4AI (Time taken: {page_duration:.2f}s)") return (current_url, content, new_links) elif crawl4ai_soup: # Try to extract from crawl4ai soup self._current_url = current_url content = self._extract_main_content(crawl4ai_soup) if content and len(content) >= self.min_content_length: logger.info(f"✓ Extracted {len(content)} chars from Crawl4AI HTML") soup = crawl4ai_soup # Extract links if current_depth < self.max_depth: links = self._extract_links(soup, current_url, scope) next_depth = current_depth + 1 new_links = [(link, next_depth) for link in links if link not in self._visited] self._successful_crawls += 1 page_end_time = time.time() page_duration = page_end_time - page_start_time logger.info(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Completed crawling page {self._crawled_pages_count} with Crawl4AI (Time taken: {page_duration:.2f}s)") return (current_url, content, new_links) logger.error(f"Crawl4AI also failed to extract content from {current_url}") except Exception as crawl_error: logger.error(f"Crawl4AI fallback error: {str(crawl_error)}", exc_info=True) else: logger.warning(f"⚠️ Crawl4AI not available to retry failed request. Install with: pip install crawl4ai>=0.9") # Final failure self._failed_crawls += 1 self._record_failure(last_cause or FailureCause.CONNECTION) logger.error(f"❌ Failed to extract content from {current_url} after all attempts. Last error: {last_error}") return None page_end_time = time.time() page_duration = page_end_time - page_start_time logger.info(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Completed crawling page {self._crawled_pages_count} (Time taken: {page_duration:.2f}s)") return (current_url, content, new_links) def crawl(self, url: str, starting_depth: int = 1, on_document_callback: Optional[Callable[[str, str], None]] = None, on_url_crawled_callback: Optional[Callable[[str], None]] = None) -> Dict[str, str]: """ Enhanced crawl method with parallel processing and immediate vector DB insertion. :param url: The URL to crawl. :param starting_depth: The starting depth level for the crawl. :param on_document_callback: Callback function that receives (url, content) for immediate processing :param on_url_crawled_callback: Callback function that receives (url) when a page is successfully crawled :return: Dictionary of URLs and their corresponding content. """ # Normalize URL to ensure it has a protocol url = self._normalize_url(url) # A www host with no address of its own falls back to the apex. url = self._seed_with_resolvable_host(url) # Reset visited and urls_to_crawl for fresh crawl self._visited = set() self._urls_to_crawl = [] # Reset crawling statistics self._crawled_pages_count = 0 self._successful_crawls = 0 self._failed_crawls = 0 self._challenge_blocked = 0 self._failure_causes = {} crawl_start_time = time.time() logger.info(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Starting parallel crawl of {url} with max_depth={self.max_depth}, max_links={self.max_links}, and max_workers={self.max_workers}") # Bound the crawl to the seed's own host by default — see crawl_scope.py. scope = CrawlScope.for_seed(url, self.crawl_scope) logger.info(f"Crawl scope for {url}: {scope.describe()}") # Add starting URL with its depth to the queue urls_to_process = [(url, starting_depth)] logger.info(f"Added starting URL to crawl queue: {url} (depth: {starting_depth})") crawler_result = self._run_crawl_queue( urls_to_process, scope, on_document_callback, on_url_crawled_callback ) # Log crawling summary crawl_end_time = time.time() crawl_duration = crawl_end_time - crawl_start_time logger.info(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Parallel crawl completed: {len(crawler_result)} pages crawled successfully") logger.info(f"Crawling statistics: Total: {self._crawled_pages_count}, Successful: {self._successful_crawls}, Failed: {self._failed_crawls}") logger.info(f"Total crawling time: {crawl_duration:.2f}s, Average time per page: {crawl_duration/max(1, self._crawled_pages_count):.2f}s") return crawler_result def _run_crawl_queue( self, urls_to_process: List[Tuple[str, int]], scope: CrawlScope, on_document_callback: Optional[Callable[[str, str], None]] = None, on_url_crawled_callback: Optional[Callable[[str], None]] = None, ) -> Dict[str, str]: """Process a seed queue of (url, depth) in parallel until it drains or the ``max_links`` cap is hit, extracting content per URL and enqueuing any new links a page yields. Shared by website crawling (seeds one URL, follows links) and sitemap crawling (seeds the listed pages, which yield no links). """ crawler_result: Dict[str, str] = {} urls_to_process = list(urls_to_process) with ThreadPoolExecutor(max_workers=self.max_workers) as executor: # Process URLs in batches until no more URLs or max_links reached while urls_to_process and len(crawler_result) < self.max_links: if self.out_of_time: logger.warning(f"Out of time: stopping with {len(urls_to_process)} link(s) not read") break # Take a batch of URLs to process batch_size = min(self.max_workers, len(urls_to_process)) current_batch = urls_to_process[:batch_size] urls_to_process = urls_to_process[batch_size:] # Submit all URLs in the current batch for parallel processing future_to_url = { executor.submit(self._process_url, url_info, scope): url_info[0] for url_info in current_batch } # Process completed futures as they finish for future in as_completed(future_to_url): url = future_to_url[future] try: result = future.result() if result: processed_url, content, new_links = result # Add the content to our results crawler_result[processed_url] = content # Call the URL crawled callback first (for progress tracking) if on_url_crawled_callback: logger.debug(f"📞 Calling URL crawled callback for: {processed_url}") on_url_crawled_callback(processed_url) # Call the callback for immediate processing if provided if on_document_callback: on_document_callback(processed_url, content) # Add new links to the processing queue for new_link, depth in new_links: if new_link not in self._visited and (new_link, depth) not in urls_to_process: urls_to_process.append((new_link, depth)) # If we've reached max_links, break early if len(crawler_result) >= self.max_links: logger.info(f"Reached maximum number of links ({self.max_links}), stopping further crawling") break except Exception as exc: logger.error(f"URL {url} generated an exception: {exc}") self._failed_crawls += 1 return crawler_result def _extract_links( self, soup: BeautifulSoup, base_url: str, scope: Optional[CrawlScope] = None ) -> List[str]: """ Extract links from a BeautifulSoup object. :param soup: The BeautifulSoup object. :param base_url: The base URL to resolve relative URLs. :param scope: Crawl boundary links must fall inside; defaults to this reader's configured scope seeded at ``base_url``. :return: A list of absolute URLs. """ links = [] seen = set() if scope is None: scope = CrawlScope.for_seed(base_url, self.crawl_scope) base_canonical = scope.canonical(self._canonical_url(base_url)) all_links = soup.find_all("a", href=True) for link in all_links: if not isinstance(link, Tag): continue href_str = str(link["href"]) # Drop mailto:/tel:/sms:/javascript:/data: hrefs before urljoin — # it passes them through untouched, and they are not pages. if not is_crawlable_url(href_str): continue full_url = urljoin(base_url, href_str) if not isinstance(full_url, str): continue # Skip query-string URLs before canonicalizing (kept for simplicity). if "?" in full_url: continue # Canonicalize (drop #fragment / trailing slash, put the seed's own # www/bare spelling on the host) so page variants collapse to a # single link and don't get crawled/stored twice. full_url = scope.canonical(self._canonical_url(full_url)) # Filter out unwanted URLs parsed_url = urlparse(full_url) # Ignore self-links and already-collected duplicates if full_url == base_canonical or full_url in seen: continue if ( scope.allows(full_url) and not any(parsed_url.path.endswith(ext) for ext in [ ".pdf", ".jpg", ".png", ".gif", ".zip", ".mp3", ".mp4", ".exe", ".dll" ]) and not parsed_url.path.startswith("#") # Skip anchors ): seen.add(full_url) links.append(full_url) logger.info(f"Extracted {len(links)} valid links from {base_url}") return links def _create_document_from_content(self, page_url: str, content: str, source_url: str, index: int) -> Document: """ Create a Document object from page content with proper metadata. Optionally summarizes content to reduce token usage. :param page_url: The URL of the page :param content: The page content :param source_url: Original source URL :param index: The document index :return: Document object """ # Extract page identifier without protocol and domain for ID parsed_url = urlparse(page_url) path = parsed_url.path # Handle fragment identifiers (#section) fragment = parsed_url.fragment if fragment: path = f"{path}#{fragment}" # Clean up path for ID generation if not path or path == "/": path = "index" else: # Remove leading/trailing slashes and replace special chars path = path.strip("/") path = re.sub(r'[^\w\-]', '_', path) # Summarize content if enabled to reduce token usage original_length = len(content) try: summarizer = get_content_summarizer() content = summarizer.summarize(content, page_url) if len(content) < original_length: logger.info(f"Content summarized: {original_length} -> {len(content)} chars ({100*len(content)/original_length:.1f}%)") except Exception as e: logger.error(f"Error during content summarization: {str(e)}") # Continue with original content if summarization fails # Create metadata with relevant information metadata = { "url": page_url, "chunk": index, "chunk_size": len(content), "original_size": original_length } # Create document with content and metadata document = Document( content=content, meta_data=metadata ) # Set document ID and name document.id = page_url # Set name to full page URL for matching with knowledge_queue document.name = source_url return document def read(self, url: str, vector_db_callback: Optional[Callable[[Document], None]] = None, url_crawled_callback: Optional[Callable[[str], None]] = None) -> List[Document]: """ Read content from a URL, crawl related pages, and convert the content into Documents. Optionally sends documents to vector DB as they are created. :param url: The URL to read from. :param vector_db_callback: Optional callback to send documents to vector DB as they're created :param url_crawled_callback: Optional callback called when each URL is successfully crawled :return: A list of Document objects. """ # Normalize URL to ensure it has a protocol url = self._normalize_url(url) # Get timestamp for tracking start_time = time.time() logger.info(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Starting to read from {url} with parallel processing") documents = [] # Create a callback for immediate document processing. A page becomes # one document per chunk (see app.knowledge.chunking); each is queued # for embedding on its own so the whole page is searchable. def on_document_created(page_url: str, content: str): index = len(documents) + 1 page = self._create_document_from_content(page_url, content, url, index) for document in chunk_page(page): documents.append(document) # Call vector DB callback if provided if vector_db_callback: try: vector_db_callback(document) logger.info(f"✓ Document {document.id} successfully sent to vector DB") except Exception as e: logger.error(f"Error sending document {document.id} to vector DB: {str(e)}") # Crawl website with the callback for immediate document processing self.crawl(url, on_document_callback=on_document_created, on_url_crawled_callback=url_crawled_callback) end_time = time.time() duration = end_time - start_time logger.info(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Completed reading from {url} - Created {len(documents)} documents (Total time: {duration:.2f}s)") return documents