replypier/backend/app/workers/knowledge_processor.py

"""
Copyright 2024-2026 ReplyPier

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import asyncio
from app.database import SessionLocal
from app.repositories.knowledge_queue import KnowledgeQueueRepository
from app.repositories.knowledge import KnowledgeRepository
from app.knowledge.knowledge_base import KnowledgeManager
from app.knowledge import page_editor
from app.services import knowledge_vector_links
from app.models.knowledge_queue import QueueStatus, ProcessingStage
from app.core.logger import get_logger
import os
from app.core.config import settings
from app.core.processor import PROCESSOR_STATUS
from app.knowledge.failure_advice import FAILURE_ADVICE, FailureCause
from datetime import datetime, timedelta, timezone
from app.models.notification import Notification, NotificationType
from app.services.user import send_fcm_notification
from urllib.parse import urlparse, unquote

logger = get_logger(__name__)


def get_user_friendly_filename(source: str, source_type: str) -> str:
    """Extract a user-friendly filename from the source URL or path"""
    try:
        if source_type in ('website', 'sitemap'):
            # For websites/sitemaps, just return the domain
            parsed = urlparse(source)
            return parsed.netloc or source
        
        # For files (PDF, etc.), extract filename from URL or path
        if source.startswith('http'):
            # Parse URL to get the path
            parsed = urlparse(source)
            path = parsed.path
            
            # Extract filename from path
            if path:
                # Get the last part of the path (filename)
                filename = path.split('/')[-1]
                # URL decode the filename to handle encoded characters like %20
                filename = unquote(filename)
                # Remove query parameters if any
                filename = filename.split('?')[0]
                if filename:
                    return filename
            
            # Fallback to domain if no filename found
            return parsed.netloc or source
        else:
            # For local file paths, just get the basename
            return os.path.basename(source)
            
    except Exception as e:
        logger.warning(f"Error extracting filename from {source}: {e}")
        return source


def _sync_vector_agent_links(db, queue_item) -> None:
    """Point the ingested chunks at every agent the source is linked to.

    Ingestion writes ``filters.agent_id`` from the queue row's single optional
    ``agent_id``, so a run queued without one leaves every chunk with an empty
    list. Retrieval matches on that filter, so the source is invisible to the
    agent while the dashboard still shows it as linked — the failure that left
    two production sources unreadable until the links were rebuilt by hand.

    The relational links are the authority, so re-derive from them. Idempotent,
    and non-fatal: the content is already ingested, and losing the run over a
    filter refresh would be worse than the stale filter.
    """
    try:
        knowledge_repo = KnowledgeRepository(db)
        sources = knowledge_repo.get_by_sources(
            queue_item.organization_id, [queue_item.source]
        )
        if not sources:
            return

        # An org can hold more than one knowledge row for the same URL, and they
        # share one set of vector chunks (keyed on the source name). Sync the
        # union of their links once, rather than per row where the last write
        # would drop the other rows' agents.
        agent_ids = sorted({
            agent_id
            for knowledge in sources
            for agent_id in page_editor.agent_ids_for(knowledge)
        })
        knowledge_vector_links.sync_agent_ids(db, sources[0], agent_ids)
        db.commit()
    except Exception as e:
        db.rollback()
        logger.error(
            f"Could not refresh vector agent filters for {queue_item.source} "
            f"(non-fatal, content was ingested): {e}"
        )


async def _notify(db, queue_item, kind: NotificationType, title: str, message: str) -> None:
    """Tell whoever added the source, in the app and by push. Never fatal: the
    source's own status must not depend on it. A source added without a user
    (a Shopify install queues the shop's site) has no one to tell."""
    if not queue_item.user_id:
        logger.info(f"Queue item {queue_item.id} has no user to notify ({kind.value})")
        return
    try:
        notification = Notification(
            user_id=queue_item.user_id,
            type=kind,
            title=title,
            message=message,
            # The column is notification_metadata; the old metadata=
            # kwarg silently shadowed Base.metadata and stored nothing.
            notification_metadata={"queue_id": queue_item.id}
        )
        db.add(notification)
        db.commit()
    except Exception as e:
        db.rollback()
        logger.error(f"Could not create {kind.value} notification for queue item {queue_item.id}: {e}")
        return
    await send_fcm_notification(queue_item.user_id, notification, db)


async def process_queue_item(queue_item_id: int):
    """Process a single queue item"""
    with SessionLocal() as db:
        queue_item = None
        try:
            queue_repo = KnowledgeQueueRepository(db)
            queue_item = queue_repo.get_by_id(queue_item_id)

            if not queue_item:
                logger.error(f"Queue item {queue_item_id} not found")
                return

            # Get knowledge manager instance
            knowledge = KnowledgeManager(
                org_id=queue_item.organization_id,
                agent_id=queue_item.agent_id
            )

            # Process if status is pending
            if queue_item.status == QueueStatus.PENDING:
                # Update to processing
                queue_item.status = QueueStatus.PROCESSING
                queue_item.processing_stage = ProcessingStage.NOT_STARTED
                queue_item.progress_percentage = 0.0
                db.commit()

                await knowledge.process_knowledge(queue_item)

                _sync_vector_agent_links(db, queue_item)

                user_friendly_name = get_user_friendly_filename(queue_item.source, queue_item.source_type)
                await _notify(db, queue_item, NotificationType.KNOWLEDGE_PROCESSED,
                              "Knowledge Processing Complete", f"Successfully processed {user_friendly_name}")

                # Auto-draft FAQs from the new source for orgs using the help
                # center. Fully non-fatal — even an import failure of the FAQ
                # stack must not flip an already-successful run to FAILED.
                try:
                    from app.services.faq_generation import maybe_enqueue_auto_faq_job
                    maybe_enqueue_auto_faq_job(db, queue_item)
                except Exception as faq_hook_err:
                    logger.error(f"Auto FAQ hook unavailable (non-fatal): {faq_hook_err}")

            queue_item.status = QueueStatus.COMPLETED
            db.commit()

        except Exception as e:
            logger.error(f"Error processing queue item {queue_item_id}: {str(e)}")
            if queue_item is None:
                raise
            # The session may hold a failed flush: start clean, record the
            # failure first, and only then tell anyone about it.
            db.rollback()
            try:
                queue_item.status = QueueStatus.FAILED
                # Persist the reason so the UI can show why it failed.
                queue_item.error = str(e)
                db.commit()
            except Exception as status_err:
                db.rollback()
                logger.error(f"Could not mark queue item {queue_item_id} failed: {status_err}")
            else:
                await _tell_failed(db, queue_item)

            raise


async def _tell_failed(db, queue_item) -> None:
    user_friendly_name = get_user_friendly_filename(queue_item.source, queue_item.source_type)
    await _notify(db, queue_item, NotificationType.KNOWLEDGE_FAILED,
                  "Knowledge Processing Failed", f"Failed to process {user_friendly_name}: {queue_item.error}")


# A source still 'processing' with no progress for this long was abandoned:
# its worker died partway (out of memory, a restart) and nothing will pick it
# up again, so it would spin in the dashboard for good. Longer than a live
# run can go without progress (websites stop at their time limit; files are
# size-capped and take about a minute), so it never fails one another
# process is still reading.
ABANDONED_AFTER = timedelta(seconds=settings.KB_SOURCE_TIME_LIMIT_SECONDS) + timedelta(minutes=15)


def _shown_as(queue_item) -> str:
    """A website by its address, as every other failure names it; a file by
    its name, not the path it was uploaded to on our server."""
    if queue_item.source_type in ('website', 'sitemap'):
        return queue_item.source
    return get_user_friendly_filename(queue_item.source, queue_item.source_type)


async def fail_abandoned_items() -> None:
    """Fail sources left 'processing' by a worker that died, and tell whoever
    added them."""
    with SessionLocal() as db:
        cutoff = datetime.now(timezone.utc) - ABANDONED_AFTER
        abandoned = KnowledgeQueueRepository(db).get_abandoned(cutoff)
        for item in abandoned:
            item.status = QueueStatus.FAILED
            item.error = FAILURE_ADVICE[FailureCause.INTERRUPTED].format(url=_shown_as(item))
        db.commit()
        for item in abandoned:
            logger.warning(f"Queue item {item.id} was abandoned mid-run; marked failed")
            await _tell_failed(db, item)


async def run_processor():
    """Single run of the processor"""
    try:
        PROCESSOR_STATUS["is_running"] = True
        PROCESSOR_STATUS["error"] = None

        try:
            await fail_abandoned_items()
        except Exception as e:
            # Never at the cost of processing what is pending
            logger.error(f"Could not check for abandoned queue items: {e}")

        # Get pending items with proper connection handling
        with SessionLocal() as db:
            queue_repo = KnowledgeQueueRepository(db)
            pending_items = queue_repo.get_pending()
            # Extract IDs before closing the session
            pending_item_ids = [item.id for item in pending_items] if pending_items else []

        if pending_item_ids:
            # Reduce concurrent processing to 2 to conserve connections on t3.micro
            semaphore = asyncio.Semaphore(2)

            async def process_with_semaphore(item_id):
                async with semaphore:
                    await process_queue_item(item_id)

            # Process items with semaphore control
            await asyncio.gather(*[process_with_semaphore(item_id) for item_id in pending_item_ids])

        PROCESSOR_STATUS["last_run"] = datetime.utcnow().isoformat()

    except Exception as e:
        logger.error(f"Error in knowledge processor: {str(e)}")
        PROCESSOR_STATUS["error"] = str(e)
        raise

    finally:
        PROCESSOR_STATUS["is_running"] = False


# Between runs. A new source waits up to this long before its crawl starts,
# and someone in onboarding is watching it: keep it short. An idle run is one
# small query.
IDLE_POLL_SECONDS = 10


def run_service():
    """Run the knowledge and FAQ loops as a standalone service (what the
    knowledge_processor container runs: python -m app.workers.knowledge_processor)."""
    logger.info("Starting knowledge processor service")

    # Pushes for processed sources go out from this process: without this the
    # first one failed with "The default Firebase app does not exist"
    # (run_knowledge_processor.py did it; this entry point, which prod runs, didn't).
    from app.services.firebase import initialize_firebase
    try:
        initialize_firebase()
    except Exception as e:
        logger.error(f"Firebase unavailable, push notifications are off: {e}")

    async def processor_loop():
        while True:
            try:
                logger.debug("Running knowledge processor")
                await run_processor()
                logger.debug(f"Knowledge processor completed, sleeping for {IDLE_POLL_SECONDS} seconds")
            except Exception as e:
                logger.error(f"Error in knowledge processor loop: {str(e)}")

            await asyncio.sleep(IDLE_POLL_SECONDS)

    async def main():
        """Knowledge and FAQ loops run as independent tasks in this container:
        a long FAQ generation job never delays knowledge ingestion, and an
        import failure of the FAQ stack only disables the FAQ half."""
        tasks = [processor_loop()]
        try:
            from app.workers.faq_processor import run_faq_processor_loop
            tasks.append(run_faq_processor_loop())
        except Exception as e:
            logger.error(f"FAQ processor unavailable, running knowledge only: {e}")
        await asyncio.gather(*tasks)

    asyncio.run(main())


# Main entry point for running as a standalone service
if __name__ == "__main__":
    run_service()