""" Copyright 2024-2026 ChatterMate Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ import asyncio from app.database import SessionLocal from app.repositories.knowledge_queue import KnowledgeQueueRepository from app.repositories.knowledge import KnowledgeRepository from app.knowledge.knowledge_base import KnowledgeManager from app.knowledge import page_editor from app.services import knowledge_vector_links from app.models.knowledge_queue import QueueStatus, ProcessingStage from app.core.logger import get_logger import os from app.core.config import settings from app.core.processor import PROCESSOR_STATUS from app.knowledge.failure_advice import FAILURE_ADVICE, FailureCause from datetime import datetime, timedelta, timezone from app.models.notification import Notification, NotificationType from app.services.user import send_fcm_notification from urllib.parse import urlparse, unquote logger = get_logger(__name__) def get_user_friendly_filename(source: str, source_type: str) -> str: """Extract a user-friendly filename from the source URL or path""" try: if source_type in ('website', 'sitemap'): # For websites/sitemaps, just return the domain parsed = urlparse(source) return parsed.netloc or source # For files (PDF, etc.), extract filename from URL or path if source.startswith('http'): # Parse URL to get the path parsed = urlparse(source) path = parsed.path # Extract filename from path if path: # Get the last part of the path (filename) filename = path.split('/')[-1] # URL decode the filename to handle encoded characters like %20 filename = unquote(filename) # Remove query parameters if any filename = filename.split('?')[0] if filename: return filename # Fallback to domain if no filename found return parsed.netloc or source else: # For local file paths, just get the basename return os.path.basename(source) except Exception as e: logger.warning(f"Error extracting filename from {source}: {e}") return source def _sync_vector_agent_links(db, queue_item) -> None: """Point the ingested chunks at every agent the source is linked to. Ingestion writes ``filters.agent_id`` from the queue row's single optional ``agent_id``, so a run queued without one leaves every chunk with an empty list. Retrieval matches on that filter, so the source is invisible to the agent while the dashboard still shows it as linked — the failure that left two production sources unreadable until the links were rebuilt by hand. The relational links are the authority, so re-derive from them. Idempotent, and non-fatal: the content is already ingested, and losing the run over a filter refresh would be worse than the stale filter. """ try: knowledge_repo = KnowledgeRepository(db) sources = knowledge_repo.get_by_sources( queue_item.organization_id, [queue_item.source] ) if not sources: return # An org can hold more than one knowledge row for the same URL, and they # share one set of vector chunks (keyed on the source name). Sync the # union of their links once, rather than per row where the last write # would drop the other rows' agents. agent_ids = sorted({ agent_id for knowledge in sources for agent_id in page_editor.agent_ids_for(knowledge) }) knowledge_vector_links.sync_agent_ids(db, sources[0], agent_ids) db.commit() except Exception as e: db.rollback() logger.error( f"Could not refresh vector agent filters for {queue_item.source} " f"(non-fatal, content was ingested): {e}" ) async def _notify(db, queue_item, kind: NotificationType, title: str, message: str) -> None: """Tell whoever added the source, in the app and by push. Never fatal: the source's own status must not depend on it. A source added without a user (a Shopify install queues the shop's site) has no one to tell.""" if not queue_item.user_id: logger.info(f"Queue item {queue_item.id} has no user to notify ({kind.value})") return try: notification = Notification( user_id=queue_item.user_id, type=kind, title=title, message=message, # The column is notification_metadata; the old metadata= # kwarg silently shadowed Base.metadata and stored nothing. notification_metadata={"queue_id": queue_item.id} ) db.add(notification) db.commit() except Exception as e: db.rollback() logger.error(f"Could not create {kind.value} notification for queue item {queue_item.id}: {e}") return await send_fcm_notification(queue_item.user_id, notification, db) async def process_queue_item(queue_item_id: int): """Process a single queue item""" with SessionLocal() as db: queue_item = None try: queue_repo = KnowledgeQueueRepository(db) queue_item = queue_repo.get_by_id(queue_item_id) if not queue_item: logger.error(f"Queue item {queue_item_id} not found") return # Get knowledge manager instance knowledge = KnowledgeManager( org_id=queue_item.organization_id, agent_id=queue_item.agent_id ) # Process if status is pending if queue_item.status == QueueStatus.PENDING: # Update to processing queue_item.status = QueueStatus.PROCESSING queue_item.processing_stage = ProcessingStage.NOT_STARTED queue_item.progress_percentage = 0.0 db.commit() await knowledge.process_knowledge(queue_item) _sync_vector_agent_links(db, queue_item) user_friendly_name = get_user_friendly_filename(queue_item.source, queue_item.source_type) await _notify(db, queue_item, NotificationType.KNOWLEDGE_PROCESSED, "Knowledge Processing Complete", f"Successfully processed {user_friendly_name}") # Auto-draft FAQs from the new source for orgs using the help # center. Fully non-fatal — even an import failure of the FAQ # stack must not flip an already-successful run to FAILED. try: from app.services.faq_generation import maybe_enqueue_auto_faq_job maybe_enqueue_auto_faq_job(db, queue_item) except Exception as faq_hook_err: logger.error(f"Auto FAQ hook unavailable (non-fatal): {faq_hook_err}") queue_item.status = QueueStatus.COMPLETED db.commit() except Exception as e: logger.error(f"Error processing queue item {queue_item_id}: {str(e)}") if queue_item is None: raise # The session may hold a failed flush: start clean, record the # failure first, and only then tell anyone about it. db.rollback() try: queue_item.status = QueueStatus.FAILED # Persist the reason so the UI can show why it failed. queue_item.error = str(e) db.commit() except Exception as status_err: db.rollback() logger.error(f"Could not mark queue item {queue_item_id} failed: {status_err}") else: await _tell_failed(db, queue_item) raise async def _tell_failed(db, queue_item) -> None: user_friendly_name = get_user_friendly_filename(queue_item.source, queue_item.source_type) await _notify(db, queue_item, NotificationType.KNOWLEDGE_FAILED, "Knowledge Processing Failed", f"Failed to process {user_friendly_name}: {queue_item.error}") # A source still 'processing' with no progress for this long was abandoned: # its worker died partway (out of memory, a restart) and nothing will pick it # up again, so it would spin in the dashboard for good. Longer than a live # run can go without progress (websites stop at their time limit; files are # size-capped and take about a minute), so it never fails one another # process is still reading. ABANDONED_AFTER = timedelta(seconds=settings.KB_SOURCE_TIME_LIMIT_SECONDS) + timedelta(minutes=15) def _shown_as(queue_item) -> str: """A website by its address, as every other failure names it; a file by its name, not the path it was uploaded to on our server.""" if queue_item.source_type in ('website', 'sitemap'): return queue_item.source return get_user_friendly_filename(queue_item.source, queue_item.source_type) async def fail_abandoned_items() -> None: """Fail sources left 'processing' by a worker that died, and tell whoever added them.""" with SessionLocal() as db: cutoff = datetime.now(timezone.utc) - ABANDONED_AFTER abandoned = KnowledgeQueueRepository(db).get_abandoned(cutoff) for item in abandoned: item.status = QueueStatus.FAILED item.error = FAILURE_ADVICE[FailureCause.INTERRUPTED].format(url=_shown_as(item)) db.commit() for item in abandoned: logger.warning(f"Queue item {item.id} was abandoned mid-run; marked failed") await _tell_failed(db, item) async def run_processor(): """Single run of the processor""" try: PROCESSOR_STATUS["is_running"] = True PROCESSOR_STATUS["error"] = None try: await fail_abandoned_items() except Exception as e: # Never at the cost of processing what is pending logger.error(f"Could not check for abandoned queue items: {e}") # Get pending items with proper connection handling with SessionLocal() as db: queue_repo = KnowledgeQueueRepository(db) pending_items = queue_repo.get_pending() # Extract IDs before closing the session pending_item_ids = [item.id for item in pending_items] if pending_items else [] if pending_item_ids: # Reduce concurrent processing to 2 to conserve connections on t3.micro semaphore = asyncio.Semaphore(2) async def process_with_semaphore(item_id): async with semaphore: await process_queue_item(item_id) # Process items with semaphore control await asyncio.gather(*[process_with_semaphore(item_id) for item_id in pending_item_ids]) PROCESSOR_STATUS["last_run"] = datetime.utcnow().isoformat() except Exception as e: logger.error(f"Error in knowledge processor: {str(e)}") PROCESSOR_STATUS["error"] = str(e) raise finally: PROCESSOR_STATUS["is_running"] = False # Between runs. A new source waits up to this long before its crawl starts, # and someone in onboarding is watching it: keep it short. An idle run is one # small query. IDLE_POLL_SECONDS = 10 def run_service(): """Run the knowledge and FAQ loops as a standalone service (what the knowledge_processor container runs: python -m app.workers.knowledge_processor).""" logger.info("Starting knowledge processor service") # Pushes for processed sources go out from this process: without this the # first one failed with "The default Firebase app does not exist" # (run_knowledge_processor.py did it; this entry point, which prod runs, didn't). from app.services.firebase import initialize_firebase try: initialize_firebase() except Exception as e: logger.error(f"Firebase unavailable, push notifications are off: {e}") async def processor_loop(): while True: try: logger.debug("Running knowledge processor") await run_processor() logger.debug(f"Knowledge processor completed, sleeping for {IDLE_POLL_SECONDS} seconds") except Exception as e: logger.error(f"Error in knowledge processor loop: {str(e)}") await asyncio.sleep(IDLE_POLL_SECONDS) async def main(): """Knowledge and FAQ loops run as independent tasks in this container: a long FAQ generation job never delays knowledge ingestion, and an import failure of the FAQ stack only disables the FAQ half.""" tasks = [processor_loop()] try: from app.workers.faq_processor import run_faq_processor_loop tasks.append(run_faq_processor_loop()) except Exception as e: logger.error(f"FAQ processor unavailable, running knowledge only: {e}") await asyncio.gather(*tasks) asyncio.run(main()) # Main entry point for running as a standalone service if __name__ == "__main__": run_service()