From 7937de5d7d20a58cb1b30717287a2e9d05c03857 Mon Sep 17 00:00:00 2001 From: admin Date: Tue, 29 Sep 2026 20:35:00 +0800 Subject: [PATCH] =?UTF-8?q?feat(knowledge):=20=E7=9F=A5=E8=AF=86=E6=BA=90?= =?UTF-8?q?=E7=BB=B4=E6=8A=A4=E4=B8=8E=20TXT/Markdown/PDF=20=E5=AF=BC?= =?UTF-8?q?=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 支持文本、Markdown 与非扫描 PDF 上传解析，展示提取文本与切块结果。 --- backend/app/api/file_upload.py | 281 +++ backend/app/api/knowledge.py | 1628 +++++++++++++++++ backend/app/services/file_storage.py | 117 ++ backend/app/services/file_upload_service.py | 384 ++++ backend/app/services/image_security.py | 117 ++ .../services/test_file_upload_service.py | 512 ++++++ backend/tests/services/test_image_security.py | 94 + backend/tests/services/test_upload_paths.py | 88 + .../KnowledgeAddSourceModal.scope.spec.ts | 71 + .../useKnowledgeManagement.url.spec.ts | 116 ++ .../composables/useStagedSourceCap.spec.ts | 55 + .../knowledge/KnowledgeAddSourceModal.vue | 597 ++++++ .../knowledge/KnowledgeExplorer.vue | 505 +++++ .../knowledge/KnowledgeLinkModal.vue | 308 ++++ .../knowledge/KnowledgePageDetail.vue | 377 ++++ .../knowledge/KnowledgePageEditor.vue | 216 +++ .../knowledge/KnowledgePlanMeters.vue | 171 ++ .../knowledge/KnowledgeSourceTree.vue | 501 +++++ .../src/composables/useKnowledgeExplorer.ts | 669 +++++++ .../src/composables/useKnowledgeManagement.ts | 506 +++++ .../src/composables/useStagedSourceCap.ts | 69 + frontend/src/utils/knowledgePages.ts | 112 ++ frontend/src/views/KnowledgeView.vue | 51 + 23 files changed, 7545 insertions(+) create mode 100644 backend/app/api/file_upload.py create mode 100644 backend/app/api/knowledge.py create mode 100644 backend/app/services/file_storage.py create mode 100644 backend/app/services/file_upload_service.py create mode 100644 backend/app/services/image_security.py create mode 100644 backend/tests/services/test_file_upload_service.py create mode 100644 backend/tests/services/test_image_security.py create mode 100644 backend/tests/services/test_upload_paths.py create mode 100644 frontend/src/__tests__/components/knowledge/KnowledgeAddSourceModal.scope.spec.ts create mode 100644 frontend/src/__tests__/composables/useKnowledgeManagement.url.spec.ts create mode 100644 frontend/src/__tests__/composables/useStagedSourceCap.spec.ts create mode 100644 frontend/src/components/knowledge/KnowledgeAddSourceModal.vue create mode 100644 frontend/src/components/knowledge/KnowledgeExplorer.vue create mode 100644 frontend/src/components/knowledge/KnowledgeLinkModal.vue create mode 100644 frontend/src/components/knowledge/KnowledgePageDetail.vue create mode 100644 frontend/src/components/knowledge/KnowledgePageEditor.vue create mode 100644 frontend/src/components/knowledge/KnowledgePlanMeters.vue create mode 100644 frontend/src/components/knowledge/KnowledgeSourceTree.vue create mode 100644 frontend/src/composables/useKnowledgeExplorer.ts create mode 100644 frontend/src/composables/useKnowledgeManagement.ts create mode 100644 frontend/src/composables/useStagedSourceCap.ts create mode 100644 frontend/src/utils/knowledgePages.ts create mode 100644 frontend/src/views/KnowledgeView.vue diff --git a/backend/app/api/file_upload.py b/backend/app/api/file_upload.py new file mode 100644 index 0000000..817695c --- /dev/null +++ b/backend/app/api/file_upload.py @@ -0,0 +1,281 @@ +""" +Copyright 2024-2026 ReplyPier + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from fastapi import APIRouter, UploadFile, File, HTTPException, Depends, status, Header, Request, Query +from fastapi.responses import FileResponse, JSONResponse +from typing import Optional, Tuple +import uuid +import os +from datetime import datetime +from urllib.parse import quote +from sqlalchemy.orm import Session +from app.core.config import settings +from app.core.s3 import upload_file_to_s3, get_s3_signed_url +from app.core.local_file_urls import verify_local_signature +from app.core.logger import get_logger +from app.core.security import verify_conversation_token +from app.models import User +from app.core.auth import get_current_user +from app.database import get_db + +logger = get_logger(__name__) + +router = APIRouter() + +# Allowed file types and size limits +ALLOWED_IMAGE_TYPES = {'image/jpeg', 'image/jpg', 'image/png', 'image/gif', 'image/webp', 'image/svg+xml'} +ALLOWED_DOCUMENT_TYPES = {'application/pdf', 'application/msword', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'text/plain', 'text/csv', 'application/vnd.ms-excel', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'} +ALLOWED_FILE_TYPES = ALLOWED_IMAGE_TYPES | ALLOWED_DOCUMENT_TYPES + +MAX_FILE_SIZE = 10 * 1024 * 1024 # 10MB +MAX_IMAGE_SIZE = 5 * 1024 * 1024 # 5MB + + +async def get_current_user_or_widget( + request: Request, + db: Session = Depends(get_db), + authorization: Optional[str] = Header(None), + x_conversation_token: Optional[str] = Header(None) +) -> dict: + """ + Get current user from JWT token, cookies, or widget conversation token + Returns dict with user info or widget info + """ + # Try to get user from cookies or Authorization header (admin/agent users) + try: + # Call get_current_user with the request - it handles both cookies and headers + user = await get_current_user(request, None, db) + if user: + return { + "type": "user", + "user_id": str(user.id), + "org_id": str(user.organization_id) + } + except HTTPException: + # User auth failed, continue to try other methods + pass + except Exception as e: + logger.error(f"Error authenticating user: {str(e)}") + + # Try to get widget info from conversation token + if x_conversation_token: + try: + payload = verify_conversation_token(x_conversation_token) + if payload: + return { + "type": "widget", + "widget_id": payload.get("widget_id"), + "org_id": payload.get("org_id"), + "customer_id": payload.get("customer_id") + } + except Exception as e: + logger.error(f"Error verifying conversation token: {str(e)}") + + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Authentication required" + ) + + +def _not_found() -> HTTPException: + return HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found") + + +def attachment_key(file_path: str) -> Tuple[str, str]: + """Parse a download path into (org_id, storage key), or 404. + + Uploads only ever land at `chat_attachments//.` (see + FileUploadService.upload_file), so anything else is not a servable + attachment. Matching that shape exactly rules out traversal: no `..` + segment can survive a three-part match on a UUID directory. + """ + # Accept the stored form ("/uploads/chat_attachments/...") and the bare key. + clean_path = file_path.lstrip('/') + if clean_path.startswith('uploads/'): + clean_path = clean_path[len('uploads/'):] + + parts = clean_path.split('/') + if len(parts) != 3 or parts[0] != 'chat_attachments' or not parts[2] or parts[2] in ('.', '..'): + raise _not_found() + try: + path_org_id = str(uuid.UUID(parts[1])) + except ValueError: + raise _not_found() + return path_org_id, '/'.join(('chat_attachments', path_org_id, parts[2])) + + +def authorized_attachment_key(file_path: str, org_id: Optional[str]) -> str: + """Resolve a download path to a storage key the caller's org may read.""" + if not org_id: + raise _not_found() + path_org_id, key = attachment_key(file_path) + # 404 rather than 403: whether another org's attachment exists is itself + # something the caller shouldn't learn. + if path_org_id != str(org_id): + raise _not_found() + return key + + +# Visitor-uploaded files (SVG included) are served from the API origin, so one +# opened directly must not run script there. Only scripts are blocked: a +# sandbox CSP would also stop the browser's PDF viewer. +ATTACHMENT_HEADERS = { + "Content-Security-Policy": "script-src 'none'", + "X-Content-Type-Options": "nosniff", + "Cache-Control": "private, max-age=300", +} + + +def _inline_attachment(path: str, filename: str) -> FileResponse: + return FileResponse(path, content_disposition_type="inline", filename=filename, + headers=ATTACHMENT_HEADERS) + + +@router.get("/local/{file_path:path}") +async def serve_signed_local_file( + file_path: str, + exp: Optional[int] = Query(None), + sig: Optional[str] = Query(None), +): + """Serve a local-storage attachment through a link from sign_local_key. + + The signature is the authorization, as with an S3 presigned URL, because + the widget loads attachments with plain / tags. + """ + _, key = attachment_key(file_path) + if not verify_local_signature(key, exp, sig): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Invalid or expired link") + path = os.path.join("uploads", key) + if not os.path.isfile(path): + raise _not_found() + return _inline_attachment(path, os.path.basename(key)) + + +@router.get("/download/{file_path:path}") +async def download_file( + file_path: str, + request: Request, + db: Session = Depends(get_db), + authorization: Optional[str] = Header(None), + x_conversation_token: Optional[str] = Header(None) +): + """ + Download/serve a file from AWS S3 or local storage + + Usage: GET /api/v1/files/download/chat_attachments/org-id/filename.jpg + """ + try: + from fastapi.responses import StreamingResponse + from app.core.s3 import get_s3_client + + # Authentication is required, and the result is what authorizes the + # read below. This used to swallow the failure and serve the file + # anyway, making every org's attachments world-readable by path. + auth_info = await get_current_user_or_widget( + request, db, authorization, x_conversation_token) + storage_key = authorized_attachment_key(file_path, auth_info.get("org_id")) + + if not settings.S3_FILE_STORAGE: + local_file_path = os.path.join("uploads", storage_key) + if not os.path.isfile(local_file_path): + raise _not_found() + return _inline_attachment(local_file_path, os.path.basename(storage_key)) + + # For AWS S3 - stream file from S3 + try: + s3_client = get_s3_client() + except Exception as client_err: + logger.error(f"Failed to create S3 client: {str(client_err)}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to create S3 client" + ) + + # S3 keys don't include the /uploads/ prefix + s3_key = storage_key + bucket = settings.S3_BUCKET + + # Check if the file exists + try: + s3_client.head_object(Bucket=bucket, Key=s3_key) + except s3_client.exceptions.NoSuchKey: + logger.warning(f"File not found in S3: {s3_key}") + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="File not found" + ) + except Exception as head_err: + logger.error(f"Error checking file in S3: {str(head_err)}", exc_info=True) + + # Get the object from S3 + try: + response = s3_client.get_object(Bucket=bucket, Key=s3_key) + except s3_client.exceptions.NoSuchKey: + logger.warning(f"File not found in S3: {s3_key}") + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="File not found" + ) + except Exception as s3_err: + logger.error(f"S3 error: {str(s3_err)}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to retrieve file from S3" + ) + + # Stream the file back to client + def generate(): + while True: + chunk = response['Body'].read(8192) + if not chunk: + break + yield chunk + + # Determine content type + content_type = response.get('ContentType', 'application/octet-stream') + + # Extract filename from the validated key + filename = storage_key.split('/')[-1] + + logger.info(f"Streaming file from S3: {filename}, content-type: {content_type}") + return StreamingResponse( + generate(), + media_type=content_type, + headers={ + **ATTACHMENT_HEADERS, + "Content-Disposition": f"inline; filename*=utf-8''{quote(filename)}", + } + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error downloading file {file_path}: {str(e)}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to download file" + ) + + + + + + + + diff --git a/backend/app/api/knowledge.py b/backend/app/api/knowledge.py new file mode 100644 index 0000000..89fca15 --- /dev/null +++ b/backend/app/api/knowledge.py @@ -0,0 +1,1628 @@ +""" +Copyright 2024-2026 ReplyPier + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from fastapi import APIRouter, HTTPException, UploadFile, File, Form, Depends, Body, Query, status +from typing import List, Optional +from app.models.user import User +from app.core.auth import get_current_user, require_any_permission, require_permissions +from app.core.logger import get_logger +import io +import os +import asyncio +from pydantic import BaseModel, field_validator +from sqlalchemy import text +from uuid import UUID +from app.database import get_db +from app.models.knowledge_to_agent import KnowledgeToAgent +from app.models.agent import Agent +from app.models.knowledge import Knowledge +from app.repositories.knowledge import KnowledgeRepository +from app.knowledge import page_editor +from app.knowledge.crawl_scope import CRAWL_SCOPES +from app.knowledge.failure_advice import FAILURE_ADVICE, FailureCause, plain_reason +from app.knowledge.pdf_download import MAX_PDF_SIZE +from app.knowledge.pdf_limits import PdfRefused, check_page_count +from app.repositories.knowledge_to_agent import KnowledgeToAgentRepository +from app.repositories.agent import AgentRepository +from app.models.knowledge_queue import KnowledgeQueue, QueueStatus +from app.repositories.knowledge_queue import KnowledgeQueueRepository +from app.core.config import settings +from app.utils.domain_check import bare_host, domain_resolves +from app.services import knowledge_vector_links +from sqlalchemy.orm import Session +from app.core.s3 import upload_file_to_s3, get_s3_signed_url +from app.core.file_validation import read_validated, safe_filename, PDF_MAGIC +from app.repositories.user import UserRepository +from app.models.notification import Notification, NotificationType +from app.services.user import send_fcm_notification +from app.workers.knowledge_processor import process_queue_item, run_processor + +# Try to import enterprise modules +try: + from app.enterprise.services.feature_access import require_accessible_subscription + HAS_ENTERPRISE = True +except ImportError: + HAS_ENTERPRISE = False + +router = APIRouter() + +# Shown in place of the database error when a source's page table can't be read. +# The real reason is logged; it must not travel to the caller. +PAGES_UNREADABLE = "Could not read this source's pages" +logger = get_logger(__name__) + +# Add this near the top of the file with other constants +TEMP_DIR = "temp" + +# PDF upload limits (see app/core/file_validation.py) +MAX_PDF_FILES = 20 # per request +ALLOWED_PDF_TYPES = ("application/pdf",) + +# Import processor status from shared module +from app.core.processor import PROCESSOR_STATUS +from app.core.params import DbIntId, PageNumber + + +def resolve_plan_max_links(subscription) -> int: + """Per-source sub-page crawl cap: the enterprise plan's ``max_sub_pages`` + when enterprise is active, otherwise the self-host configurable + ``KB_MAX_LINKS`` (so self-hosted installs actually control crawl scope).""" + if HAS_ENTERPRISE and subscription: + return subscription.plan.max_sub_pages + return settings.KB_MAX_LINKS + + +class UrlsRequest(BaseModel): + org_id: UUID + pdf_urls: List[str] = [] + websites: List[str] = [] + sitemaps: List[str] = [] + agent_id: Optional[str] = None + # Optional cap on pages per website source (e.g. 1 = "this page only"). Always + # clamped to the plan's max_sub_pages server-side; None uses the plan limit. + max_links: Optional[int] = None + # How far link-following may wander from the seed URL: 'path' | 'host' | + # 'domain'. None uses the default (the seed's own host). + crawl_scope: Optional[str] = None + + @field_validator('max_links') + @classmethod + def validate_max_links(cls, v): + if v is not None and v < 1: + raise ValueError('max_links must be at least 1') + return v + + @field_validator('crawl_scope') + @classmethod + def validate_crawl_scope(cls, v): + if v is not None and v not in CRAWL_SCOPES: + raise ValueError(f"crawl_scope must be one of {', '.join(CRAWL_SCOPES)}") + return v + + + @field_validator('websites', 'sitemaps') + @classmethod + def validate_website_url_format(cls, v): + """Validate that each website/sitemap URL is in https://domainname format""" + import re + validated_urls = [] + + for url in v: + if not url: + raise ValueError('URL cannot be empty') + + # Remove trailing slashes and whitespace + url = url.strip().rstrip('/') + + # Check if URL starts with https:// + if not url.startswith('https://'): + raise ValueError('URL must start with https://') + + # Extract domain part after https:// + domain_part = url[8:] # Remove 'https://' + + # Check if domain part is not empty + if not domain_part: + raise ValueError('URL must contain a domain name') + + # Allow alphanumeric, dots, hyphens, and forward slashes for paths + if not re.match(r'^[a-zA-Z0-9.-]+(/.*)?$', domain_part): + raise ValueError('Invalid URL format') + + # Ensure domain has at least one dot (for TLD) + domain_only = domain_part.split('/')[0] # Get just the domain part before any path + if '.' not in domain_only: + raise ValueError('URL must contain a valid domain with TLD') + + validated_urls.append(url) + + return validated_urls + + @field_validator('pdf_urls') + @classmethod + def validate_pdf_url_format(cls, v): + """Validate that each PDF URL is in https://domainname format""" + import re + validated_urls = [] + + for url in v: + if not url: + raise ValueError('URL cannot be empty') + + # Remove trailing slashes and whitespace + url = url.strip().rstrip('/') + + # Check if URL starts with https:// + if not url.startswith('https://'): + raise ValueError('URL must start with https://') + + # Extract domain part after https:// + domain_part = url[8:] # Remove 'https://' + + # Check if domain part is not empty + if not domain_part: + raise ValueError('URL must contain a domain name') + + # Allow alphanumeric, dots, hyphens, and forward slashes for paths + if not re.match(r'^[a-zA-Z0-9.-]+(/.*)?$', domain_part): + raise ValueError('Invalid URL format') + + # Ensure domain has at least one dot (for TLD) + domain_only = domain_part.split('/')[0] # Get just the domain part before any path + if '.' not in domain_only: + raise ValueError('URL must contain a valid domain with TLD') + + validated_urls.append(url) + + return validated_urls + + +@router.post("/upload/pdf") +async def upload_pdf_files( + files: List[UploadFile] = File(...), + org_id: str = Form(...), + agent_id: Optional[str] = Form(None), + current_user: User = Depends(require_permissions("manage_knowledge")), + db: Session = Depends(get_db) +): + """Upload PDF files to knowledge base""" + try: + # Convert org_id string to UUID for comparison + try: + org_uuid = UUID(org_id) + except ValueError: + raise HTTPException(status_code=400, detail="Invalid org_id") + if current_user.organization_id != org_uuid: + raise HTTPException(status_code=403, detail="Unauthorized access to organization") + + if len(files) > MAX_PDF_FILES: + raise HTTPException( + status_code=400, + detail=f"Too many files: at most {MAX_PDF_FILES} per request", + ) + + # Validate the agent (format + ownership) once, up front. + agent_uuid = None + if agent_id: + try: + agent_uuid = UUID(agent_id) + except ValueError: + raise HTTPException(status_code=400, detail="Invalid agent_id") + agent = AgentRepository(db).get_agent(agent_uuid) + if not agent or agent.organization_id != org_uuid: + # 404 (not 403) so we don't reveal another org's agent existence. + raise HTTPException(status_code=404, detail="Agent not found") + + # Bound for the non-enterprise path (resolve_plan_max_links below). + subscription = None + # Check enterprise subscription limits if enterprise module is available + if HAS_ENTERPRISE: + knowledge_repo = KnowledgeRepository(db) + + # Accessible = active/trial/past-due-in-period OR cancelled-but- + # still-in-paid-period; raises 403 when the org has no plan. + subscription = require_accessible_subscription(db, org_uuid) + + # Sources already spoken for: indexed, plus crawls still queued + current_count = knowledge_repo.count_sources_in_use(org_uuid) + + # Check if adding these files would exceed the limit + new_count = current_count + len(files) + if subscription.plan.max_knowledge_sources is not None and new_count > subscription.plan.max_knowledge_sources: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=f"Cannot add files: Maximum number of knowledge sources ({subscription.plan.max_knowledge_sources}) would be exceeded" + ) + + queue_repo = KnowledgeQueueRepository(db) + queued_items = [] + + # Process each file + for file in files: + # Validate content-type, size, and magic bytes; returns the bytes. + content = await read_validated( + file, + max_size=MAX_PDF_SIZE, + allowed_content_types=ALLOWED_PDF_TYPES, + magic_prefix=PDF_MAGIC, + label="PDF", + ) + try: + # pypdf on an untrusted upload: off the event loop + await asyncio.to_thread(check_page_count, io.BytesIO(content)) + except PdfRefused as e: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"{file.filename}: {e}") + # Sanitize the filename (strip path components, add a UUID prefix) + # to prevent path traversal and overwrites. + stored_name = safe_filename(file.filename) + source_type = "pdf_file" + + # Check if S3 storage is enabled + if settings.S3_FILE_STORAGE: + folder = f"knowledge/{org_uuid}" + file_url = await upload_file_to_s3(content, folder, stored_name, content_type="application/pdf") + logger.debug(f"Uploaded PDF to S3: {file_url}") + file_path = await get_s3_signed_url(file_url) + source_type = "pdf_url" + else: + # Save file locally + os.makedirs(TEMP_DIR, exist_ok=True) + file_path = os.path.join(TEMP_DIR, stored_name) + with open(file_path, "wb") as f: + f.write(content) + + # Create queue item with user_id + queue_item = KnowledgeQueue( + organization_id=org_uuid, + agent_id=agent_uuid, + user_id=current_user.id, + source_type=source_type, + source=file_path, + status=QueueStatus.PENDING, + queue_metadata={ + "max_links": resolve_plan_max_links(subscription) + } + ) + queued_items.append(queue_repo.create(queue_item)) + + return { + "message": "PDFs queued for processing,it will take a while to process, we will notify you when it is done", + "queue_items": [{"id": item.id, "status": item.status} for item in queued_items] + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error uploading PDFs: {str(e)}") + raise HTTPException(status_code=500, detail="Failed to upload files") + + +def _get_crawled_urls_info(crawled_urls): + """Get comprehensive information about crawled URLs""" + if not crawled_urls or not isinstance(crawled_urls, list): + return { + "latest_url": None, + "all_urls": [], + "count": 0 + } + + all_urls = [] + latest_url = None + + try: + for item in crawled_urls: + if isinstance(item, str): + # New format: just URL strings + all_urls.append(item) + latest_url = item # Keep updating to get the last one + elif isinstance(item, dict) and "url" in item: + # Legacy format: objects with url, timestamp, status + all_urls.append(item["url"]) + latest_url = item["url"] + except (KeyError, TypeError) as e: + logger.warning(f"Error processing crawled URLs: {str(e)}") + + return { + "latest_url": latest_url, + "all_urls": all_urls, + "count": len(all_urls) + } + + +@router.get("/explore/progress/{queue_id}") +async def get_explore_progress( + queue_id: DbIntId, + db: Session = Depends(get_db) +): + """Get progress status for knowledge base processing""" + try: + # Deliberately unauthenticated — it polls the job the equally public + # /explore/add-url just created. That makes scoping it to the shared + # explore org essential: queue ids are sequential, so an unscoped + # lookup let anyone walk every tenant's ingestion jobs and read their + # source URLs and crawled-page lists. + queue_item = db.query(KnowledgeQueue).filter( + KnowledgeQueue.id == queue_id, + KnowledgeQueue.organization_id == UUID(settings.EXPLORE_SOURCE_ORG_ID) + ).first() + + # Refresh the item to get the latest data from the database + if queue_item: + db.refresh(queue_item) + + if not queue_item: + raise HTTPException(status_code=404, detail="Queue item not found") + + # Get processing stage information + stage_info = { + "not_started": {"label": "Initializing", "step": 1, "total": 4}, + "NOT_STARTED": {"label": "Initializing", "step": 1, "total": 4}, + "crawling": {"label": "Crawling Website", "step": 2, "total": 4}, + "CRAWLING": {"label": "Crawling Website", "step": 2, "total": 4}, + "embedding": {"label": "Processing Content", "step": 3, "total": 4}, + "EMBEDDING": {"label": "Processing Content", "step": 3, "total": 4}, + "completed": {"label": "Completed", "step": 4, "total": 4}, + "COMPLETED": {"label": "Completed", "step": 4, "total": 4} + } + + # Safely get enum values + def get_enum_value(enum_field): + if enum_field is None: + return None + return enum_field.value if hasattr(enum_field, 'value') else str(enum_field) + + # Get processing stage as string + processing_stage_str = get_enum_value(queue_item.processing_stage) or "NOT_STARTED" + status_str = get_enum_value(queue_item.status) or "PENDING" + + # Override stage to "COMPLETED" if status is COMPLETED + if status_str.upper() == "COMPLETED": + processing_stage_str = "COMPLETED" + + logger.debug(f"Queue {queue_id}: status='{status_str}', stage='{processing_stage_str}', progress={queue_item.progress_percentage}, is_complete={status_str.upper() in ['COMPLETED', 'FAILED']}") + logger.debug(f"Queue {queue_id}: crawled_urls count={len(queue_item.crawled_urls) if queue_item.crawled_urls else 0}") + logger.debug(f"Queue {queue_id}: crawled_urls raw value: {queue_item.crawled_urls}") + + current_stage = stage_info.get(processing_stage_str, + {"label": "Processing", "step": 1, "total": 4}) + + # Calculate overall progress based on stage and percentage + if status_str.upper() == "COMPLETED": + overall_progress = 100.0 + else: + stage_weight = (current_stage["step"] - 1) / current_stage["total"] * 100 + stage_progress = (queue_item.progress_percentage or 0) / current_stage["total"] + overall_progress = min(100, stage_weight + stage_progress) + + # Get crawled URLs information + crawled_urls_info = _get_crawled_urls_info(queue_item.crawled_urls) + + return { + "queue_id": queue_item.id, + "status": status_str, + "processing_stage": processing_stage_str, + "progress_percentage": queue_item.progress_percentage or 0, + "overall_progress": round(overall_progress, 1), + "current_stage": current_stage, + "source": queue_item.source, + "total_items": queue_item.total_items or 0, + "processed_items": queue_item.processed_items or 0, + "created_at": queue_item.created_at.isoformat() if queue_item.created_at else None, + "updated_at": queue_item.updated_at.isoformat() if queue_item.updated_at else None, + "crawled_url": crawled_urls_info["latest_url"], + "crawled_urls": crawled_urls_info["all_urls"], + "crawled_count": crawled_urls_info["count"], + "is_complete": status_str.upper() in ["COMPLETED", "FAILED"], + "error_message": getattr(queue_item, 'error_message', None) + } + + except HTTPException: + # Otherwise the not-found above is reported as a 500 echoing its detail. + raise + except Exception as e: + logger.error(f"Error getting progress for queue {queue_id}: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) + + +async def _hosts_that_do_not_exist(urls: List[str]) -> set: + """Hosts DNS says have no such name. Lookups that time out or can't tell + are left to the crawler.""" + hosts = sorted({bare_host(url) for url in urls} - {""}) + answers = await asyncio.gather(*(domain_resolves(host) for host in hosts)) + return {host for host, answer in zip(hosts, answers) if answer is False} + + +@router.post("/add/urls") +async def add_urls( + request: UrlsRequest = Body(...), + current_user: User = Depends(require_permissions("manage_knowledge")), + db: Session = Depends(get_db) +): + """Add URLs to knowledge base""" + try: + # Verify organization access + if current_user.organization_id != request.org_id: + raise HTTPException(status_code=403, detail="Unauthorized access to organization") + + # Validate the agent (format + ownership) once, up front. + agent_uuid = None + if request.agent_id: + try: + agent_uuid = UUID(request.agent_id) + except (ValueError, TypeError): + raise HTTPException(status_code=400, detail="Invalid agent_id") + agent = AgentRepository(db).get_agent(agent_uuid) + if not agent or agent.organization_id != request.org_id: + # 404 (not 403) so we don't reveal another org's agent existence. + raise HTTPException(status_code=404, detail="Agent not found") + + # Bound for the non-enterprise path (resolve_plan_max_links below). + subscription = None + # Check enterprise subscription limits if enterprise module is available + if HAS_ENTERPRISE: + knowledge_repo = KnowledgeRepository(db) + + # Accessible = active/trial/past-due-in-period OR cancelled-but- + # still-in-paid-period; raises 403 when the org has no plan. + subscription = require_accessible_subscription(db, request.org_id) + + # Sources already spoken for: indexed, plus crawls still queued + current_count = knowledge_repo.count_sources_in_use(request.org_id) + + # Calculate total new URLs to be added (each sitemap is one source) + total_new_urls = len(request.pdf_urls) + len(request.websites) + len(request.sitemaps) + + # Check if adding these URLs would exceed the limit + new_count = current_count + total_new_urls + if subscription.plan.max_knowledge_sources is not None and new_count > subscription.plan.max_knowledge_sources: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=f"Cannot add URLs: Maximum number of knowledge sources ({subscription.plan.max_knowledge_sources}) would be exceeded" + ) + + queue_repo = KnowledgeQueueRepository(db) + knowledge_repo = KnowledgeRepository(db) + queued_items = [] + + # Per-source sub-page cap: the plan limit, optionally narrowed by the + # request (e.g. "this page only" -> max_links=1), never allowed to + # exceed the plan limit. + plan_sub_pages = resolve_plan_max_links(subscription) + website_max_links = ( + min(request.max_links, plan_sub_pages) if request.max_links else plan_sub_pages + ) + + # Check for duplicate URLs + all_urls = request.pdf_urls + request.websites + request.sitemaps + existing_sources = knowledge_repo.get_by_sources(request.org_id, all_urls) + + if existing_sources: + duplicate_urls = [source.source for source in existing_sources] + return { + "error": "Some URLs already exist in your knowledge base", + "duplicate_urls": duplicate_urls + } + + # A domain that doesn't exist fails now, not after its turn in the + # processor (up to a minute or more): onboarding is waiting to hear. + missing_hosts = await _hosts_that_do_not_exist(all_urls) + + def queue(source_type: str, url: str, metadata: dict) -> None: + dead = bare_host(url) in missing_hosts + queued_items.append(queue_repo.create(KnowledgeQueue( + organization_id=request.org_id, + agent_id=agent_uuid, + user_id=current_user.id, + source_type=source_type, + source=url, + status=QueueStatus.FAILED if dead else QueueStatus.PENDING, + error=FAILURE_ADVICE[FailureCause.DNS].format(url=url) if dead else None, + queue_metadata=metadata, + ))) + + for url in request.pdf_urls: + queue('pdf_url', url, {"max_links": plan_sub_pages}) + for url in request.websites: + queue('website', url, { + "max_links": website_max_links, + # Omitted (None) means the crawler's default: the seed's host. + "crawl_scope": request.crawl_scope, + }) + # Sitemaps — one source each; pages are discovered from the sitemap + # and capped at the plan's sub-page limit (no per-request crawl scope). + for url in request.sitemaps: + queue('sitemap', url, {"max_links": plan_sub_pages}) + + return { + "message": "URLs queued for processing, it will take a while to process, we will notify you when it is done", + "queue_items": [{"id": item.id, "status": item.status} for item in queued_items] + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error adding URLs: {str(e)}") + raise HTTPException(status_code=500, detail="Failed to add URLs") + + +class TextSourceRequest(BaseModel): + org_id: UUID + title: str + content: str + agent_id: Optional[str] = None + + @field_validator('title') + @classmethod + def validate_title(cls, v): + if not v or not v.strip(): + raise ValueError('Title cannot be empty') + return v.strip() + + @field_validator('content') + @classmethod + def validate_content(cls, v): + if not v or not v.strip(): + raise ValueError('Content cannot be empty') + return v + + +@router.post("/add/text") +async def add_text_source( + request: TextSourceRequest = Body(...), + current_user: User = Depends(require_permissions("manage_knowledge")), + db: Session = Depends(get_db) +): + """Create a knowledge source from pasted text and index it immediately.""" + try: + if current_user.organization_id != request.org_id: + raise HTTPException(status_code=403, detail="Unauthorized access to organization") + + # Validate the agent (format + ownership) up front. + agent_uuid = None + if request.agent_id: + try: + agent_uuid = UUID(request.agent_id) + except (ValueError, TypeError): + raise HTTPException(status_code=400, detail="Invalid agent_id") + agent = AgentRepository(db).get_agent(agent_uuid) + if not agent or agent.organization_id != request.org_id: + raise HTTPException(status_code=404, detail="Agent not found") + + knowledge_repo = KnowledgeRepository(db) + + # Enforce the source-count limit (enterprise only). + if HAS_ENTERPRISE: + subscription = require_accessible_subscription(db, request.org_id) + current_count = knowledge_repo.count_sources_in_use(request.org_id) + max_sources = subscription.plan.max_knowledge_sources + if max_sources is not None and current_count + 1 > max_sources: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=f"Cannot add source: Maximum number of knowledge sources ({max_sources}) would be exceeded" + ) + + # Reject a duplicate source title. + if knowledge_repo.get_by_sources(request.org_id, [request.title]): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="A knowledge source with this title already exists" + ) + + knowledge = page_editor.create_text_source( + db, request.org_id, request.title, request.content, agent_uuid + ) + return { + "message": "Text source added", + "knowledge": {"id": knowledge.id, "name": knowledge.source, "type": knowledge.source_type.value}, + } + except HTTPException: + raise + except Exception as e: + db.rollback() + logger.error(f"Error adding text source: {str(e)}") + raise HTTPException(status_code=500, detail="Failed to add text source") + + +@router.post("/link") +async def link_knowledge_to_agent( + knowledge_id: int, + agent_id: str, + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + """Link existing knowledge to an agent""" + try: + knowledge_repo = KnowledgeRepository(db) + link_repo = KnowledgeToAgentRepository(db) + + # Verify knowledge exists and belongs to user's org + knowledge = knowledge_repo.get_by_id(knowledge_id) + if not knowledge or knowledge.organization_id != current_user.organization_id: + raise HTTPException(status_code=404, detail="Knowledge source not found or unauthorized access") + + # Convert agent_id to UUID + try: + agent_uuid = UUID(agent_id) + except ValueError: + raise HTTPException(status_code=400, detail="Invalid agent ID format") + + # The agent must belong to the caller's org, not just the knowledge. + # 404 (not 403) so we don't reveal another org's agent existence. + agent = AgentRepository(db).get_agent(agent_uuid) + if not agent or agent.organization_id != current_user.organization_id: + raise HTTPException(status_code=404, detail="Agent not found") + + # Check if link already exists + existing_link = link_repo.get_by_ids(knowledge_id, agent_uuid) + if existing_link: + raise HTTPException(status_code=400, detail="Knowledge is already linked to this agent") + + # The link row and the vector-store filters must land together: retrieval + # matches on filters->'agent_id', so a committed link whose filter update + # failed shows as linked in the dashboard while the agent can retrieve + # nothing. commit=False defers to the single commit below. + link_repo.create( + KnowledgeToAgent(knowledge_id=knowledge_id, agent_id=agent_uuid), + commit=False, + ) + knowledge_vector_links.add_agent(db, knowledge, agent_uuid) + db.commit() + + return {"message": "Knowledge linked to agent successfully"} + + except HTTPException: + db.rollback() + raise + except Exception as e: + db.rollback() + logger.error(f"Error linking knowledge: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.delete("/unlink") +async def unlink_knowledge_from_agent( + knowledge_id: int, + agent_id: str, + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + """Unlink knowledge from an agent""" + try: + # Convert agent_id to UUID + try: + agent_uuid = UUID(agent_id) + except ValueError: + raise HTTPException(status_code=400, detail="Invalid agent ID format") + + knowledge_repo = KnowledgeRepository(db) + link_repo = KnowledgeToAgentRepository(db) + + # Verify knowledge exists and belongs to user's org + knowledge = knowledge_repo.get_by_id(knowledge_id) + if not knowledge or knowledge.organization_id != current_user.organization_id: + raise HTTPException(status_code=404, detail="Knowledge source not found or unauthorized access") + + # And the agent must be in the caller's org too. + agent = AgentRepository(db).get_agent(agent_uuid) + if not agent or agent.organization_id != current_user.organization_id: + raise HTTPException(status_code=404, detail="Agent not found") + + # Same transaction as the filter update below — an unlink that commits + # the row deletion but leaves the agent_id in the vector filters keeps + # the source retrievable by an agent the dashboard shows as unlinked. + success = link_repo.delete_by_ids(knowledge_id, agent_uuid, commit=False) + if not success: + raise HTTPException(status_code=404, detail="Link not found") + + knowledge_vector_links.remove_agent(db, knowledge, agent_uuid) + db.commit() + + return {"message": "Knowledge unlinked from agent successfully"} + + except HTTPException: + db.rollback() + raise + except Exception as e: + db.rollback() + logger.error(f"Error unlinking knowledge: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) + + +def _linked_agents_map(db: Session, knowledge_items) -> dict: + """Map knowledge_id -> [{id, name}] of the agents each source is linked to. + + Batched into a single query over the page's sources to avoid N+1 lookups. + """ + ids = [k.id for k in knowledge_items] + if not ids: + return {} + rows = ( + db.query(KnowledgeToAgent.knowledge_id, Agent.id, Agent.name, Agent.display_name) + .join(Agent, Agent.id == KnowledgeToAgent.agent_id) + .filter(KnowledgeToAgent.knowledge_id.in_(ids)) + .all() + ) + result: dict = {} + for knowledge_id, agent_id, name, display_name in rows: + result.setdefault(knowledge_id, []).append( + {"id": str(agent_id), "name": display_name or name} + ) + return result + + +@router.get("/agent/{agent_id}") +async def get_knowledge_by_agent( + agent_id: str, + page: PageNumber = 1, + page_size: int = Query(default=10, ge=1, le=100), + current_user: User = Depends( + require_any_permission("view_knowledge", "manage_knowledge")), + db: Session = Depends(get_db) +): + """Get knowledge sources and their data for an agent with pagination""" + try: + # Convert agent_id to UUID + try: + agent_uuid = UUID(agent_id) + except ValueError: + raise HTTPException(status_code=400, detail="Invalid agent ID format") + + # The agent must belong to the caller's organization. count_by_agent and + # get_by_agent below filter on agent_uuid alone, so without this an id + # from another tenant returns that tenant's knowledge inventory — the + # link/unlink routes have always checked this; this read never did. + agent = db.query(Agent).filter(Agent.id == agent_uuid).first() + if not agent or agent.organization_id != current_user.organization_id: + raise HTTPException(status_code=404, detail="Agent not found") + + logger.debug(f"Getting knowledge for agent {agent_uuid}") + knowledge_repo = KnowledgeRepository(db) + + # Get total count and paginated knowledge items + total_count = knowledge_repo.count_by_agent(agent_uuid) + logger.debug(f"Total count for agent {agent_uuid}: {total_count}") + + knowledge_items = knowledge_repo.get_by_agent( + agent_uuid, + skip=(page - 1) * page_size, + limit=page_size + ) + + + agents_map = _linked_agents_map(db, knowledge_items) + result = [] + for k in knowledge_items: + # Base knowledge data + knowledge_data = { + "id": k.id, + "name": k.source, + "type": k.source_type.value, + "agents": agents_map.get(k.id, []), + "pages": [] + } + + # Query the actual data if table_name is specified + if k.table_name and k.schema: + try: + # Create a safe query to get unique records with cleaned source + query = text(f""" + SELECT DISTINCT + {page_editor.PAGE_ID_EXPR} as subpage, + id, + created_at, + updated_at + FROM {k.schema}."{k.table_name}" + WHERE name = :source + """) + + # Execute query with parameters + rows = db.execute(query, {"source": k.source}) + + # Group pages by subpage + pages_dict = {} + for row in rows: + subpage = row.subpage + if subpage not in pages_dict: + pages_dict[subpage] = { + "subpage": subpage, + "created_at": row.created_at.isoformat() if row.created_at else None, + "updated_at": row.updated_at.isoformat() if row.updated_at else None + } + + knowledge_data["pages"] = list(pages_dict.values()) + + except Exception as e: + logger.error(f"Error querying table {k.table_name}: {str(e)}") + knowledge_data["error"] = PAGES_UNREADABLE + + result.append(knowledge_data) + + + return { + "knowledge": result, + "pagination": { + "total": total_count, + "page": page, + "page_size": page_size, + "total_pages": (total_count + page_size - 1) // page_size + } + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting knowledge by agent: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) + + +def _enum_value(enum_field): + """Safely unwrap an enum field to its string value (or None).""" + if enum_field is None: + return None + return enum_field.value if hasattr(enum_field, 'value') else str(enum_field) + + +def _serialize_queue_item(item: KnowledgeQueue) -> dict: + """Shape a KnowledgeQueue row for the queue-list API responses.""" + return { + "id": item.id, + "source": item.source, + "source_type": item.source_type, + "status": _enum_value(item.status), + "error": item.error, + # One plain sentence for people; `error` stays for the knowledge page + "reason": plain_reason(item.error, item.source) if _enum_value(item.status) == QueueStatus.FAILED.value else None, + "created_at": item.created_at.isoformat() if item.created_at else None, + "updated_at": item.updated_at.isoformat() if item.updated_at else None, + "processing_stage": _enum_value(item.processing_stage), + "progress_percentage": item.progress_percentage or 0, + } + + +@router.get("/queue/agent/{agent_id}") +async def get_agent_queue_items( + agent_id: str, + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + """Get all queue items (pending, processing, failed) for an agent""" + try: + # Convert agent_id to UUID + try: + agent_uuid = UUID(agent_id) + except ValueError: + raise HTTPException(status_code=400, detail="Invalid agent ID format") + + logger.debug(f"Getting queue items for agent {agent_uuid}") + + # Get queue items for this agent (excluding completed ones) + queue_items = db.query(KnowledgeQueue).filter( + KnowledgeQueue.agent_id == agent_uuid, + KnowledgeQueue.organization_id == current_user.organization_id, + KnowledgeQueue.status.in_([QueueStatus.PENDING, QueueStatus.PROCESSING, QueueStatus.FAILED]) + ).order_by(KnowledgeQueue.created_at.desc()).all() + + return {"queue_items": [_serialize_queue_item(item) for item in queue_items]} + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting queue items for agent: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.get("/queue/organization/{org_id}") +async def get_organization_queue_items( + org_id: UUID, + current_user: User = Depends( + require_any_permission("view_knowledge", "manage_knowledge")), + db: Session = Depends(get_db) +): + """Get all in-flight queue items (pending, processing, failed) for an org.""" + try: + if current_user.organization_id != org_id: + raise HTTPException(status_code=403, detail="Unauthorized access to organization") + + queue_items = db.query(KnowledgeQueue).filter( + KnowledgeQueue.organization_id == org_id, + KnowledgeQueue.status.in_([QueueStatus.PENDING, QueueStatus.PROCESSING, QueueStatus.FAILED]) + ).order_by(KnowledgeQueue.created_at.desc()).all() + + return {"queue_items": [_serialize_queue_item(item) for item in queue_items]} + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting queue items for org: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.delete("/queue/{queue_id}") +async def delete_queue_item( + queue_id: DbIntId, + current_user: User = Depends(require_permissions("manage_knowledge")), + db: Session = Depends(get_db) +): + """Delete a queue item (only if failed or pending)""" + try: + queue_repo = KnowledgeQueueRepository(db) + item = queue_repo.get_by_id(queue_id) + + if not item: + raise HTTPException(status_code=404, detail="Queue item not found") + + if item.organization_id != current_user.organization_id: + raise HTTPException(status_code=403, detail="Unauthorized") + + # Only allow deleting failed or pending items + # We might want to allow deleting processing items if they are stuck, but let's be safe for now + if item.status not in [QueueStatus.FAILED, QueueStatus.PENDING]: + raise HTTPException( + status_code=400, + detail="Only failed or pending items can be removed from queue" + ) + + db.delete(item) + db.commit() + + return {"message": "Queue item deleted successfully"} + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error deleting queue item: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.get("/organization/{org_id}") + +async def get_knowledge_by_organization( + org_id: str, + page: PageNumber = 1, + page_size: int = Query(default=10, ge=1, le=100), + current_user: User = Depends( + require_any_permission("view_knowledge", "manage_knowledge")), + db: Session = Depends(get_db) +): + """Get knowledge sources and their data for an organization with pagination""" + try: + # Convert org_id to UUID + try: + org_uuid = UUID(org_id) + except ValueError: + raise HTTPException(status_code=400, detail="Invalid organization ID format") + + logger.debug(f"Getting knowledge for organization {org_uuid}") + + # Verify organization access + if current_user.organization_id != org_uuid: + raise HTTPException(status_code=403, detail="Unauthorized access to organization") + + knowledge_repo = KnowledgeRepository(db) + + # Get total count and paginated knowledge items + total_count = knowledge_repo.count_by_organization(org_uuid) + logger.debug(f"Total count for organization {org_uuid}: {total_count}") + + knowledge_items = knowledge_repo.get_by_organization( + org_uuid, + skip=(page - 1) * page_size, + limit=page_size + ) + logger.debug(f"Knowledge items for organization {org_uuid}: {knowledge_items}") + + agents_map = _linked_agents_map(db, knowledge_items) + result = [] + for k in knowledge_items: + # Base knowledge data + knowledge_data = { + "id": k.id, + "name": k.source, + "type": k.source_type.value, + "agents": agents_map.get(k.id, []), + "pages": [] + } + + # Query the actual data if table_name is specified + if k.table_name and k.schema: + try: + # Create a safe query to get unique records with cleaned source + query = text(f""" + SELECT DISTINCT + {page_editor.PAGE_ID_EXPR} as subpage, + id, + created_at, + updated_at + FROM {k.schema}."{k.table_name}" + WHERE name = :source + """) + + # Execute query with parameters + rows = db.execute(query, {"source": k.source}) + + # Group pages by subpage + pages_dict = {} + for row in rows: + subpage = row.subpage + if subpage not in pages_dict: + pages_dict[subpage] = { + "subpage": subpage, + "created_at": row.created_at.isoformat() if row.created_at else None, + "updated_at": row.updated_at.isoformat() if row.updated_at else None + } + + knowledge_data["pages"] = list(pages_dict.values()) + + except Exception as e: + logger.error(f"Error querying table {k.table_name}: {str(e)}") + knowledge_data["error"] = PAGES_UNREADABLE + + result.append(knowledge_data) + + + return { + "knowledge": result, + "pagination": { + "total": total_count, + "page": page, + "page_size": page_size, + "total_pages": (total_count + page_size - 1) // page_size + } + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting knowledge by organization: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.get("/queue/{queue_id}") +async def get_queue_status( + queue_id: DbIntId, + current_user: User = Depends( + require_any_permission("view_knowledge", "manage_knowledge")), + db: Session = Depends(get_db) +): + """Get status of a queued knowledge item""" + try: + queue_repo = KnowledgeQueueRepository(db) + item = db.query(KnowledgeQueue).filter( + KnowledgeQueue.id == queue_id).first() + + if not item: + return {"error": "Queue item not found"} + + if item.organization_id != current_user.organization_id: + return {"error": "Unauthorized access to queue item"} + + return { + "id": item.id, + "status": item.status, + "error": item.error, + "created_at": item.created_at, + "updated_at": item.updated_at + } + + except Exception as e: + logger.error(f"Error getting queue status: {str(e)}") + return {"error": "Could not read the queue status"} + + +@router.get("/processor/status") +async def get_processor_status( + current_user: User = Depends( + require_any_permission("view_knowledge", "manage_knowledge")), + db: Session = Depends(get_db) +): + """Get status of the knowledge processor for user's organization""" + try: + # Get counts of items in different states for user's items + base_query = db.query(KnowledgeQueue)\ + .filter( + KnowledgeQueue.organization_id == current_user.organization_id, + KnowledgeQueue.user_id == current_user.id # Add user filter + ) + + pending_count = base_query.filter( + KnowledgeQueue.status == QueueStatus.PENDING + ).count() + + processing_count = base_query.filter( + KnowledgeQueue.status == QueueStatus.PROCESSING + ).count() + + completed_count = base_query.filter( + KnowledgeQueue.status == QueueStatus.COMPLETED + ).count() + + failed_count = base_query.filter( + KnowledgeQueue.status == QueueStatus.FAILED + ).count() + + return { + "last_run": PROCESSOR_STATUS["last_run"], + "is_running": PROCESSOR_STATUS["is_running"], + "error": PROCESSOR_STATUS["error"], + "queue_status": { + "pending": pending_count, + "processing": processing_count, + "completed": completed_count, + "failed": failed_count + } + } + + except Exception as e: + logger.error(f"Error getting processor status: {str(e)}") + return {"error": "Could not read the processor status"} + + +@router.delete("/{knowledge_id}") +async def delete_knowledge( + knowledge_id: DbIntId, + current_user: User = Depends(require_permissions("manage_knowledge")), + db: Session = Depends(get_db) +): + """Delete a knowledge source and its associated data""" + try: + knowledge_repo = KnowledgeRepository(db) + + # Verify knowledge exists and belongs to user's org + knowledge = knowledge_repo.get_by_id(knowledge_id) + if not knowledge: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Knowledge source not found" + ) + + if knowledge.organization_id != current_user.organization_id: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Unauthorized access to knowledge source" + ) + + # Delete knowledge and associated data + success = knowledge_repo.delete_with_data(knowledge_id) + + if success: + return {"message": "Knowledge source deleted successfully"} + + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to delete knowledge source" + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error deleting knowledge: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=str(e) + ) + + +@router.get("/{knowledge_id}/content") +async def get_knowledge_content( + knowledge_id: DbIntId, + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + """Get knowledge content chunks from vector database""" + try: + knowledge_repo = KnowledgeRepository(db) + + # Verify knowledge exists and belongs to user's org + knowledge = knowledge_repo.get_by_id(knowledge_id) + if not knowledge: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Knowledge source not found" + ) + + if knowledge.organization_id != current_user.organization_id: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Unauthorized access to knowledge source" + ) + + # Query the vector database for content chunks + if not knowledge.table_name or not knowledge.schema: + return { + "knowledge_id": knowledge_id, + "source": knowledge.source, + "chunks": [] + } + + try: + # Get all chunks for this knowledge source + query = text(f""" + SELECT + id, + content, + meta_data, + created_at + FROM {knowledge.schema}."{knowledge.table_name}" + WHERE name = :source + ORDER BY created_at ASC, {page_editor.NATURAL_ID_ORDER} + """) + + + rows = db.execute(query, {"source": knowledge.source}).fetchall() + + chunks = [] + for row in rows: + chunks.append({ + "id": row.id, + "content": row.content, + "metadata": row.meta_data, + "created_at": row.created_at.isoformat() if row.created_at else None + }) + + return { + "knowledge_id": knowledge_id, + "source": knowledge.source, + "source_type": knowledge.source_type.value, + "chunks": chunks + } + + except Exception as e: + logger.error(f"Error querying content for knowledge {knowledge_id}: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Error accessing content: {str(e)}" + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting knowledge content: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=str(e) + ) + + +def _require_editable_knowledge( + db: Session, knowledge_id: int, current_user: User +) -> Knowledge: + """Load a knowledge source, enforcing ownership and a usable vector table. + + Raises 404 if missing, 403 if it belongs to another org, 400 if it has no + vector table yet (e.g. a crawl that has not produced any rows). + """ + knowledge = KnowledgeRepository(db).get_by_id(knowledge_id) + if not knowledge: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Knowledge source not found" + ) + if knowledge.organization_id != current_user.organization_id: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Unauthorized access to knowledge source" + ) + if not knowledge.table_name or not knowledge.schema: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Knowledge source has no vector database table" + ) + return knowledge + + +@router.put("/{knowledge_id}/chunk/{chunk_id:path}") +async def update_chunk_content( + knowledge_id: DbIntId, + chunk_id: str, + content: str = Body(..., embed=True), + current_user: User = Depends(require_permissions("manage_knowledge")), + db: Session = Depends(get_db) +): + """Update a single chunk's content in vector database""" + try: + knowledge = _require_editable_knowledge(db, knowledge_id, current_user) + + try: + # Verify the chunk exists AND belongs to this source (the vector + # table is shared across the org's sources, so scope by name). + query = text(f""" + SELECT 1 + FROM {knowledge.schema}."{knowledge.table_name}" + WHERE id = :chunk_id AND name = :source + """) + row = db.execute( + query, {"chunk_id": chunk_id, "source": knowledge.source} + ).fetchone() + + if not row: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Chunk not found" + ) + + # Re-embed the updated content in place + page_editor.reembed_chunk(db, knowledge, chunk_id, content) + db.commit() + + logger.info(f"Updated chunk {chunk_id} for knowledge {knowledge_id}") + + return {"message": "Chunk updated successfully"} + + except HTTPException: + raise + except Exception as e: + db.rollback() + logger.error(f"Error updating chunk {chunk_id}: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Error updating chunk: {str(e)}" + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error updating chunk: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=str(e) + ) + + +@router.delete("/{knowledge_id}/chunk/{chunk_id:path}") +async def delete_chunk( + knowledge_id: DbIntId, + chunk_id: str, + current_user: User = Depends(require_permissions("manage_knowledge")), + db: Session = Depends(get_db) +): + """Delete a single chunk from vector database""" + try: + knowledge = _require_editable_knowledge(db, knowledge_id, current_user) + + try: + # Delete the chunk from the database (scoped to this source, since the + # vector table is shared across the org's sources). + delete_query = text(f""" + DELETE FROM {knowledge.schema}."{knowledge.table_name}" + WHERE id = :chunk_id AND name = :source + """) + + result = db.execute( + delete_query, {"chunk_id": chunk_id, "source": knowledge.source} + ) + db.commit() + + if result.rowcount == 0: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Chunk not found" + ) + + logger.info(f"Deleted chunk {chunk_id} from knowledge {knowledge_id}") + + return {"message": "Chunk deleted successfully"} + + except HTTPException: + raise + except Exception as e: + db.rollback() + logger.error(f"Error deleting chunk {chunk_id}: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Error deleting chunk: {str(e)}" + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error deleting chunk: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=str(e) + ) + + +@router.post("/{knowledge_id}/subpage") +async def add_subpage( + knowledge_id: DbIntId, + subpage_name: str = Body(..., embed=True), + content: str = Body(..., embed=True), + url: Optional[str] = Body(None, embed=True), + current_user: User = Depends(require_permissions("manage_knowledge")), + db: Session = Depends(get_db) +): + """Add a new subpage to existing knowledge""" + try: + knowledge_repo = KnowledgeRepository(db) + + # Verify knowledge exists and belongs to user's org + knowledge = knowledge_repo.get_by_id(knowledge_id) + if not knowledge: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Knowledge source not found" + ) + + if knowledge.organization_id != current_user.organization_id: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Unauthorized access to knowledge source" + ) + + if not knowledge.table_name or not knowledge.schema: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Knowledge source has no vector database table" + ) + + # Check subscription limits if enterprise module is available + if HAS_ENTERPRISE: + try: + from app.enterprise.repositories.subscription import SubscriptionRepository + + subscription_repo = SubscriptionRepository(db) + subscription = subscription_repo.get_active_subscription(str(knowledge.organization_id)) + + if subscription and subscription.plan: + # Count existing subpages for this knowledge source + current_count = page_editor.count_subpages(db, knowledge) + + # Check against max_sub_pages limit from the plan + subpages_limit = subscription.plan.max_sub_pages + + if current_count >= subpages_limit: + raise HTTPException( + status_code=status.HTTP_402_PAYMENT_REQUIRED, + detail=f"Subpage limit reached. Your plan allows {subpages_limit} subpages per knowledge source. Please upgrade your plan." + ) + except ImportError: + # Enterprise module not available, skip limit check + pass + + try: + # Check if subpage name already exists within this source. A page + # may be stored only as chunks, so compare the page id, not the row id. + check_query = text(f""" + SELECT id FROM {knowledge.schema}."{knowledge.table_name}" + WHERE {page_editor.PAGE_ID_EXPR} = :subpage_name AND name = :source + LIMIT 1 + """) + existing = db.execute( + check_query, {"subpage_name": subpage_name, "source": knowledge.source} + ).fetchone() + + if existing: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Subpage name already exists. Please use a unique name." + ) + + # A subpage of a URL-based source must be on the SAME registrable + # domain as that source — otherwise a single plan-limited source + # could accumulate content from arbitrary other domains. domain_of_url + # normalizes a scheme-less value (so 'evil.com/x' can't parse as a + # path with no host and slip past) and is ccTLD-aware (so a *.co.uk + # source can't accept another *.co.uk domain). + clean_url = (url or "").strip() or None + if clean_url: + from app.knowledge.domains import domain_of_url + + parent_domain = domain_of_url(knowledge.source) + if parent_domain and domain_of_url(clean_url) != parent_domain: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Subpage URL must be on the same domain as the source ({parent_domain}).", + ) + + # Embed and insert the new subpage into the vector database + page_editor.insert_subpage(knowledge, subpage_name, content, clean_url) + + logger.info(f"Added new subpage '{subpage_name}' to knowledge {knowledge_id}") + + return {"message": "Subpage added successfully", "subpage_id": subpage_name} + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error adding subpage: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Error adding subpage: {str(e)}" + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error adding subpage: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=str(e) + ) + + +@router.put("/{knowledge_id}/page/{page_id:path}") +async def update_page_content( + knowledge_id: DbIntId, + page_id: str, + content: str = Body(..., embed=True), + title: Optional[str] = Body(None, embed=True), + current_user: User = Depends(require_permissions("manage_knowledge")), + db: Session = Depends(get_db) +): + """Replace a sub-page's content and re-embed it. + + A page may span several ``_N`` chunks; this collapses it into one freshly + embedded chunk keyed by the page id, keeping the source's answers current. + """ + if not (content or "").strip(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Page content cannot be empty" + ) + + knowledge = _require_editable_knowledge(db, knowledge_id, current_user) + try: + replaced = page_editor.replace_page(db, knowledge, page_id, content, title) + if replaced == 0: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Page not found" + ) + logger.info(f"Updated page '{page_id}' for knowledge {knowledge_id}") + return {"message": "Page updated successfully", "page_id": page_id} + except HTTPException: + raise + except Exception as e: + db.rollback() + logger.error(f"Error updating page '{page_id}': {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Error updating page: {str(e)}" + ) + + +@router.delete("/{knowledge_id}/page/{page_id:path}") +async def delete_page( + knowledge_id: DbIntId, + page_id: str, + current_user: User = Depends(require_permissions("manage_knowledge")), + db: Session = Depends(get_db) +): + """Delete a whole sub-page (all of its chunks) from a knowledge source.""" + knowledge = _require_editable_knowledge(db, knowledge_id, current_user) + try: + removed = page_editor.delete_page_chunks(db, knowledge, page_id) + if removed == 0: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Page not found" + ) + db.commit() + logger.info( + f"Deleted page '{page_id}' ({removed} chunk(s)) from knowledge {knowledge_id}" + ) + return {"message": "Page deleted successfully", "removed_chunks": removed} + except HTTPException: + raise + except Exception as e: + db.rollback() + logger.error(f"Error deleting page '{page_id}': {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Error deleting page: {str(e)}" + ) + + diff --git a/backend/app/services/file_storage.py b/backend/app/services/file_storage.py new file mode 100644 index 0000000..126697d --- /dev/null +++ b/backend/app/services/file_storage.py @@ -0,0 +1,117 @@ +""" +Copyright 2024-2026 ReplyPier + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Storage backend shim shared by upload endpoints: S3 when configured, else the +local uploads/ directory. Only PUBLIC_UPLOAD_FOLDERS are static-mounted under +/api/v1/uploads; private files are reached through signed links. +""" + +import os +from typing import Optional + +import aiofiles + +from app.core.config import settings +from app.core.local_file_urls import sign_local_key +from app.core.s3 import ( + delete_file_from_s3, + download_file_from_s3, + get_s3_signed_url, + upload_file_to_s3, +) + + +async def store_upload(content: bytes, folder: str, file_name: str, content_type: str) -> str: + """Persist bytes and return the URL to store. S3 mode returns the raw S3 + URL, local mode the {API_V1_STR}/uploads/... path; either way, pass it + through resolve_public_url before handing it to a client.""" + if settings.S3_FILE_STORAGE: + return await upload_file_to_s3(content, folder, file_name, content_type=content_type) + upload_dir = os.path.join("uploads", folder) + os.makedirs(upload_dir, exist_ok=True) + async with aiofiles.open(os.path.join(upload_dir, file_name), "wb") as f: + await f.write(content) + return f"{settings.API_V1_STR}/uploads/{folder}/{file_name}" + + +# Local-storage folders anyone may read, static-mounted at {API_V1_STR}/uploads/: +# agent avatars and agents' profile pictures shown to anonymous widget visitors, +# and help-center logos and article images. Everything else under uploads/ (chat +# attachments, PDF imports awaiting a worker) is private. +PUBLIC_UPLOAD_FOLDERS = ("agents", "user", "help_center") +# Private folders handed to clients as short-lived signed links, the local +# counterpart of an S3 presigned URL. +SIGNED_UPLOAD_FOLDERS = ("chat_attachments",) + + +def upload_key(stored: str) -> str: + """Key relative to `uploads/` for a stored local upload URL. + + Tolerates every shape the app has ever written: the current + `{API_V1_STR}/uploads/...`, the legacy bare `/uploads/...` that + agent photos, profile pictures and chat attachments carried, and a plain + relative key with no prefix at all. + """ + relative = stored.lstrip("/") + api_prefix = f"{settings.API_V1_STR.strip('/')}/" + if relative.startswith(api_prefix): + relative = relative[len(api_prefix):] + if relative.startswith("uploads/"): + relative = relative[len("uploads/"):] + return relative + + +def local_upload_path(stored: str) -> str: + """Filesystem path under `uploads/` for a stored local upload URL.""" + return os.path.join("uploads", upload_key(stored)) + + +def signed_upload_key(stored: Optional[str]) -> Optional[str]: + """The key of a stored local upload that is only served through a signed + link, or None for anything else (S3 and other absolute URLs, public files).""" + if not stored or "://" in stored or stored.startswith(("data:", "blob:")): + return None + key = upload_key(stored) + return key if key.split("/", 1)[0] in SIGNED_UPLOAD_FOLDERS else None + + +async def load_upload(stored: str) -> bytes: + """Read back a previously stored upload by the URL/path store_upload + returned. Used by workers running in a different container than the API + (shared local uploads mount or S3).""" + if stored.startswith("http"): + return await download_file_from_s3(stored) + async with aiofiles.open(local_upload_path(stored), "rb") as f: + return await f.read() + + +async def delete_upload(stored: str) -> None: + """Best-effort removal of a stored upload (temp import files).""" + try: + if stored.startswith("http"): + await delete_file_from_s3(stored) + return + os.remove(local_upload_path(stored)) + except Exception: + pass + + +async def resolve_public_url(stored_url: str) -> str: + """Client-facing URL for a stored upload: presigned for S3 objects, a + signed link for private local files, verbatim otherwise.""" + if stored_url and settings.S3_FILE_STORAGE and stored_url.startswith("http"): + return await get_s3_signed_url(stored_url) + key = signed_upload_key(stored_url) + return sign_local_key(key) if key else stored_url diff --git a/backend/app/services/file_upload_service.py b/backend/app/services/file_upload_service.py new file mode 100644 index 0000000..b75a07d --- /dev/null +++ b/backend/app/services/file_upload_service.py @@ -0,0 +1,384 @@ +""" +Copyright 2024-2026 ReplyPier + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import base64 +import os +import uuid +from typing import Dict, Optional, List, Set +from app.core.config import settings +from app.core.s3 import upload_file_to_s3 +from app.services.file_storage import resolve_public_url +from app.core.logger import get_logger + +logger = get_logger(__name__) + +# Allowed file types and size limits +ALLOWED_IMAGE_TYPES = {'image/jpeg', 'image/jpg', 'image/png', 'image/gif', 'image/webp'} +ALLOWED_DOCUMENT_TYPES = {'application/pdf', 'application/msword', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'text/plain', 'text/csv', 'application/vnd.ms-excel', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'} +ALLOWED_FILE_TYPES = ALLOWED_IMAGE_TYPES | ALLOWED_DOCUMENT_TYPES + +# Security: SVG removed from default allowed types due to XSS risks +# Can be enabled per-agent if needed with proper sanitization + +MAX_FILE_SIZE = 10 * 1024 * 1024 # 10MB +MAX_IMAGE_SIZE = 5 * 1024 * 1024 # 5MB + +# Magic byte signatures for file type validation +# This prevents MIME type spoofing attacks +MAGIC_BYTES = { + # Images + 'image/jpeg': [ + (b'\xff\xd8\xff\xe0', 0), # JPEG JFIF + (b'\xff\xd8\xff\xe1', 0), # JPEG EXIF + (b'\xff\xd8\xff\xe2', 0), # JPEG ICC + (b'\xff\xd8\xff\xe8', 0), # JPEG SPIFF + (b'\xff\xd8\xff\xdb', 0), # JPEG DQT + (b'\xff\xd8\xff\xee', 0), # JPEG Adobe + ], + 'image/jpg': [ + (b'\xff\xd8\xff\xe0', 0), + (b'\xff\xd8\xff\xe1', 0), + (b'\xff\xd8\xff\xe2', 0), + (b'\xff\xd8\xff\xe8', 0), + (b'\xff\xd8\xff\xdb', 0), + (b'\xff\xd8\xff\xee', 0), + ], + 'image/png': [ + (b'\x89PNG\r\n\x1a\n', 0), + ], + 'image/gif': [ + (b'GIF87a', 0), + (b'GIF89a', 0), + ], + 'image/webp': [ + (b'RIFF', 0), # WebP starts with RIFF, then has WEBP at offset 8 + ], + # Documents + 'application/pdf': [ + (b'%PDF-', 0), + ], + 'application/msword': [ + (b'\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1', 0), # OLE Compound File (DOC) + ], + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': [ + (b'PK\x03\x04', 0), # ZIP-based (DOCX) + ], + 'application/vnd.ms-excel': [ + (b'\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1', 0), # OLE Compound File (XLS) + ], + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': [ + (b'PK\x03\x04', 0), # ZIP-based (XLSX) + ], + 'text/plain': [], # Text files don't have magic bytes, validated by content + 'text/csv': [], # CSV files don't have magic bytes, validated by content +} + +# File extension to MIME type mapping for validation +EXTENSION_TO_MIME = { + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.png': 'image/png', + '.gif': 'image/gif', + '.webp': 'image/webp', + '.pdf': 'application/pdf', + '.doc': 'application/msword', + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.txt': 'text/plain', + '.csv': 'text/csv', + '.xls': 'application/vnd.ms-excel', + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', +} + +# Category labels for display +FILE_TYPE_CATEGORIES = { + 'images': {'image/jpeg', 'image/jpg', 'image/png', 'image/gif', 'image/webp'}, + 'documents': {'application/pdf'}, + 'office': {'application/msword', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'application/vnd.ms-excel', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'}, + 'text': {'text/plain', 'text/csv'}, +} + + +class FileUploadService: + """Service for handling file uploads to S3 or local storage with enhanced security""" + + @staticmethod + def validate_magic_bytes(file_content: bytes, claimed_content_type: str) -> tuple[bool, Optional[str]]: + """ + Validate file content against magic byte signatures to prevent MIME type spoofing. + + Args: + file_content: The raw binary content of the file + claimed_content_type: The MIME type claimed by the client + + Returns: + (is_valid, error_message) + """ + if claimed_content_type not in MAGIC_BYTES: + logger.warning(f"Unknown content type for magic byte validation: {claimed_content_type}") + return False, f"File type '{claimed_content_type}' is not allowed" + + signatures = MAGIC_BYTES[claimed_content_type] + + # Text files don't have magic bytes - validate as text content + if not signatures: + if claimed_content_type in {'text/plain', 'text/csv'}: + # Check if content is valid text (not binary) + try: + # Try to decode as UTF-8, allowing for some common encodings + sample = file_content[:8192] # Check first 8KB + sample.decode('utf-8') + return True, None + except UnicodeDecodeError: + try: + sample.decode('latin-1') + return True, None + except UnicodeDecodeError: + return False, "File content does not appear to be valid text" + return True, None + + # Check for WebP special case (has WEBP at offset 8) + if claimed_content_type == 'image/webp': + if len(file_content) >= 12: + if file_content[:4] == b'RIFF' and file_content[8:12] == b'WEBP': + return True, None + return False, "File content does not match WebP format signature" + + # Check against all possible signatures for this MIME type + for signature, offset in signatures: + end_offset = offset + len(signature) + if len(file_content) >= end_offset: + if file_content[offset:end_offset] == signature: + return True, None + + return False, f"File content does not match the claimed type '{claimed_content_type}'. The file may be corrupted or mislabeled." + + @staticmethod + def validate_extension(filename: str, claimed_content_type: str) -> tuple[bool, Optional[str]]: + """ + Validate that file extension matches the claimed content type. + + Args: + filename: The name of the file + claimed_content_type: The MIME type claimed by the client + + Returns: + (is_valid, error_message) + """ + ext = os.path.splitext(filename.lower())[1] + + if not ext: + return False, "File must have an extension" + + if ext not in EXTENSION_TO_MIME: + return False, f"File extension '{ext}' is not allowed. Allowed extensions: {', '.join(sorted(EXTENSION_TO_MIME.keys()))}" + + expected_mime = EXTENSION_TO_MIME[ext] + + # Handle jpg/jpeg equivalence + if claimed_content_type in {'image/jpeg', 'image/jpg'} and expected_mime in {'image/jpeg', 'image/jpg'}: + return True, None + + if expected_mime != claimed_content_type: + return False, f"File extension '{ext}' does not match content type '{claimed_content_type}'" + + return True, None + + @staticmethod + def validate_file_type(content_type: str, allowed_types: Optional[Set[str]] = None) -> bool: + """ + Validate if the file type is allowed. + + Args: + content_type: The MIME type to validate + allowed_types: Optional set of allowed MIME types. If None, uses default ALLOWED_FILE_TYPES. + """ + types_to_check = allowed_types if allowed_types is not None else ALLOWED_FILE_TYPES + return content_type in types_to_check + + @staticmethod + def get_allowed_types_for_agent(agent_allowed_types: Optional[List[str]] = None) -> Set[str]: + """ + Get the set of allowed file types for an agent. + + Args: + agent_allowed_types: List of category names like ['images', 'documents', 'office', 'text'] + If None or empty, returns all default allowed types. + + Returns: + Set of MIME types that are allowed + """ + if not agent_allowed_types: + return ALLOWED_FILE_TYPES + + allowed = set() + for category in agent_allowed_types: + category_lower = category.lower() + if category_lower in FILE_TYPE_CATEGORIES: + allowed.update(FILE_TYPE_CATEGORIES[category_lower]) + + # If no valid categories found, return default + if not allowed: + return ALLOWED_FILE_TYPES + + return allowed + + @staticmethod + def validate_file_size(file_size: int, content_type: str) -> tuple[bool, Optional[str]]: + """ + Validate file size based on content type + Returns: (is_valid, error_message) + """ + if content_type in ALLOWED_IMAGE_TYPES and file_size > MAX_IMAGE_SIZE: + return False, f"Image file size exceeds maximum allowed size of {MAX_IMAGE_SIZE / (1024*1024):.1f}MB" + + if file_size > MAX_FILE_SIZE: + return False, f"File size exceeds maximum allowed size of {MAX_FILE_SIZE / (1024*1024):.1f}MB" + + return True, None + + @staticmethod + def get_friendly_allowed_types_message(allowed_types: Optional[Set[str]] = None) -> str: + """Generate a user-friendly message listing allowed file types.""" + types_to_use = allowed_types if allowed_types is not None else ALLOWED_FILE_TYPES + + extensions = [] + for ext, mime in EXTENSION_TO_MIME.items(): + if mime in types_to_use: + extensions.append(ext.upper().replace('.', '')) + + return ', '.join(sorted(set(extensions))) + + @staticmethod + async def upload_file( + file_data: dict, + org_id: str, + customer_id: Optional[str] = None, + user_id: Optional[str] = None, + allowed_types: Optional[Set[str]] = None + ) -> Dict[str, any]: + """ + Upload a file to S3 or local storage with enhanced security validation. + + Args: + file_data: Dict with 'content' (base64), 'filename', 'content_type', 'size' + org_id: Organization ID + customer_id: Optional customer ID (for widget users) + user_id: Optional user ID (for authenticated users) + allowed_types: Optional set of allowed MIME types for this upload + + Returns: + Dict with 'file_url', 'signed_url', 'filename', 'content_type', 'size' + + Raises: + ValueError: If validation fails or upload fails + """ + try: + filename = file_data.get('filename', '') + content_type = file_data.get('content_type', 'application/octet-stream') + + # Validate file data + if not file_data.get('content') or not filename: + raise ValueError("Missing required file data: content and filename") + + # Determine which types are allowed + types_allowed = allowed_types if allowed_types is not None else ALLOWED_FILE_TYPES + friendly_types = FileUploadService.get_friendly_allowed_types_message(types_allowed) + + # Step 1: Validate extension matches claimed content type + is_valid, error = FileUploadService.validate_extension(filename, content_type) + if not is_valid: + logger.warning(f"Extension validation failed for {filename}: {error}") + raise ValueError(error) + + # Step 2: Validate file type is allowed + if not FileUploadService.validate_file_type(content_type, types_allowed): + raise ValueError(f"File type '{content_type}' is not allowed. Allowed types: {friendly_types}") + + # Step 3: Decode base64 content + try: + file_content = base64.b64decode(file_data['content']) + except Exception as e: + logger.error(f"Error decoding base64 content: {str(e)}") + raise ValueError("Invalid file content encoding") + + file_size = len(file_content) + + # Step 4: Validate file size + is_valid, error_message = FileUploadService.validate_file_size(file_size, content_type) + if not is_valid: + raise ValueError(error_message) + + # Step 5: SECURITY - Validate magic bytes to prevent MIME spoofing + is_valid, magic_error = FileUploadService.validate_magic_bytes(file_content, content_type) + if not is_valid: + logger.warning(f"Magic byte validation failed for {filename}: {magic_error}") + raise ValueError(f"Security check failed: {magic_error}") + + logger.info(f"File validation passed for {filename}: type={content_type}, size={file_size}") + + # Generate unique filename with sanitized extension + file_extension = os.path.splitext(filename)[1].lower() + # Additional sanitization: only allow known extensions + if file_extension not in EXTENSION_TO_MIME: + raise ValueError(f"File extension '{file_extension}' is not allowed") + + unique_filename = f"{uuid.uuid4()}{file_extension}" + + # Determine folder based on auth type + folder = f"chat_attachments/{org_id}" + + # Upload to S3 if enabled, otherwise save locally + if settings.S3_FILE_STORAGE: + file_url = await upload_file_to_s3( + file_content=file_content, + folder=folder, + filename=unique_filename, + content_type=content_type + ) + else: + # Save locally + upload_dir = os.path.join("uploads", folder) + os.makedirs(upload_dir, exist_ok=True) + + file_path = os.path.join(upload_dir, unique_filename) + with open(file_path, "wb") as f: + f.write(file_content) + + file_url = f"/uploads/{folder}/{unique_filename}" + + signed_url = await resolve_public_url(file_url) + + logger.info(f"Successfully uploaded file: {filename} -> {file_url}") + + return { + 'file_url': file_url, + 'signed_url': signed_url, + 'filename': filename, + 'content_type': content_type, + 'size': file_size + } + + except ValueError: + # Re-raise validation errors + raise + except Exception as e: + logger.error(f"Error in FileUploadService.upload_file: {str(e)}", exc_info=True) + raise ValueError(f"Failed to upload file: {str(e)}") + diff --git a/backend/app/services/image_security.py b/backend/app/services/image_security.py new file mode 100644 index 0000000..ac205b2 --- /dev/null +++ b/backend/app/services/image_security.py @@ -0,0 +1,117 @@ +""" +Copyright 2024-2026 ReplyPier + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Hardening for user-uploaded images (help-center logo + article images). Every +uploaded raster is: + - size-capped (bytes), + - checked for a real, decodable image (not a polyglot / renamed file), + - guarded against decompression bombs (pixel-count cap read from the header + BEFORE the full image is decoded), + - re-encoded from decoded pixels — which strips EXIF/metadata, trailing bytes + and any appended payload, and normalises the stored format. + +SVG is intentionally NOT accepted here: it is active markup and would be stored +XSS on the public help center. Callers that previously took SVG rasterise on the +client (the cropper) or drop it. +""" + +import io + +from fastapi import HTTPException +from PIL import Image + +# Refuse absurd pixel counts before decoding — a few-KB file can claim to be +# tens of thousands of pixels per side and exhaust memory on decode. +_MAX_PIXELS = 40_000_000 # 40 MP +Image.MAX_IMAGE_PIXELS = _MAX_PIXELS + +# PIL format -> (content type, extension) for the formats we re-encode to. +_PNG = ("image/png", ".png") +_JPEG = ("image/jpeg", ".jpg") + +_FORMAT_CONTENT_TYPE = { + "PNG": "image/png", + "JPEG": "image/jpeg", + "WEBP": "image/webp", + "GIF": "image/gif", +} + + +def _open_validated(content: bytes, *, max_bytes: int, max_dim: int) -> Image.Image: + """Open bytes as an image, enforcing byte size, decodability and a pixel / + dimension cap (bomb guard) — all before the pixels are fully decoded.""" + if not content: + raise HTTPException(status_code=400, detail="Empty image file.") + if len(content) > max_bytes: + raise HTTPException(status_code=400, detail=f"Image must be {max_bytes // (1024 * 1024)} MB or smaller.") + try: + img = Image.open(io.BytesIO(content)) + width, height = img.size # header only; no full decode yet + except HTTPException: + raise + except Exception: + raise HTTPException(status_code=400, detail="Invalid or corrupt image file.") + if width <= 0 or height <= 0: + raise HTTPException(status_code=400, detail="Invalid image dimensions.") + if width > max_dim or height > max_dim or width * height > _MAX_PIXELS: + raise HTTPException( + status_code=400, + detail=f"Image is too large — keep it within {max_dim}×{max_dim} pixels.", + ) + try: + img.load() # dimensions are now known-safe, so decoding is bounded + except Exception: + raise HTTPException(status_code=400, detail="Invalid or corrupt image file.") + return img + + +def sanitize_image( + content: bytes, + *, + allowed_content_types: set[str], + max_bytes: int, + max_dim: int, + fit: int | None = None, +) -> tuple[bytes, str, str]: + """Validate and re-encode an uploaded raster image. + + Returns (safe_bytes, content_type, extension). Raises HTTPException(400) on + anything that is not a clean, in-policy raster image. `fit`, when given, + downscales the longest side to at most `fit` px (aspect preserved). + + JPEG re-encodes to JPEG (keeps photos small); everything else re-encodes to + PNG (keeps transparency). Either path strips metadata and any non-pixel + payload because the output is built from decoded pixels only. + """ + img = _open_validated(content, max_bytes=max_bytes, max_dim=max_dim) + + detected = _FORMAT_CONTENT_TYPE.get((img.format or "").upper()) + if detected is None or detected not in allowed_content_types: + raise HTTPException( + status_code=400, + detail="Unsupported image type. Use PNG, JPEG or WebP.", + ) + + if fit and max(img.size) > fit: + img.thumbnail((fit, fit), Image.LANCZOS) + + out = io.BytesIO() + if detected == "image/jpeg": + img.convert("RGB").save(out, format="JPEG", quality=85, optimize=True) + content_type, ext = _JPEG + else: + img.convert("RGBA").save(out, format="PNG", optimize=True) + content_type, ext = _PNG + return out.getvalue(), content_type, ext diff --git a/backend/tests/services/test_file_upload_service.py b/backend/tests/services/test_file_upload_service.py new file mode 100644 index 0000000..16c3652 --- /dev/null +++ b/backend/tests/services/test_file_upload_service.py @@ -0,0 +1,512 @@ +""" +Copyright 2024-2026 ReplyPier + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import pytest +import base64 +import os +import tempfile +from unittest.mock import Mock, patch, AsyncMock, MagicMock +from uuid import uuid4 + +from app.services.file_upload_service import ( + FileUploadService, + ALLOWED_IMAGE_TYPES, + ALLOWED_DOCUMENT_TYPES, + ALLOWED_FILE_TYPES, + MAX_FILE_SIZE, + MAX_IMAGE_SIZE +) + + +class TestFileUploadServiceValidation: + """Test file validation methods""" + + def test_validate_file_type_allowed_image(self): + """Test validation of allowed image types""" + for content_type in ALLOWED_IMAGE_TYPES: + assert FileUploadService.validate_file_type(content_type) is True + + def test_validate_file_type_allowed_document(self): + """Test validation of allowed document types""" + for content_type in ALLOWED_DOCUMENT_TYPES: + assert FileUploadService.validate_file_type(content_type) is True + + def test_validate_file_type_disallowed(self): + """Test validation of disallowed file types""" + disallowed_types = [ + 'video/mp4', + 'audio/mp3', + 'application/x-executable', + 'text/html', + 'application/javascript' + ] + for content_type in disallowed_types: + assert FileUploadService.validate_file_type(content_type) is False + + def test_validate_file_size_image_within_limit(self): + """Test image file size validation within limits""" + # Test image under image size limit + image_size = MAX_IMAGE_SIZE - 1000 # 1KB under limit + is_valid, error = FileUploadService.validate_file_size(image_size, 'image/jpeg') + assert is_valid is True + assert error is None + + def test_validate_file_size_image_exceeds_image_limit(self): + """Test image file size validation exceeding image limit""" + # Test image over image size limit + image_size = MAX_IMAGE_SIZE + 1000 # 1KB over image limit + is_valid, error = FileUploadService.validate_file_size(image_size, 'image/jpeg') + assert is_valid is False + assert "Image file size exceeds maximum" in error + assert "5.0MB" in error + + def test_validate_file_size_document_within_limit(self): + """Test document file size validation within limits""" + # Test document under general size limit + doc_size = MAX_FILE_SIZE - 1000 # 1KB under limit + is_valid, error = FileUploadService.validate_file_size(doc_size, 'application/pdf') + assert is_valid is True + assert error is None + + def test_validate_file_size_document_exceeds_limit(self): + """Test document file size validation exceeding limit""" + # Test document over general size limit + doc_size = MAX_FILE_SIZE + 1000 # 1KB over limit + is_valid, error = FileUploadService.validate_file_size(doc_size, 'application/pdf') + assert is_valid is False + assert "File size exceeds maximum" in error + assert "10.0MB" in error + + def test_validate_file_size_image_exceeds_general_limit(self): + """Test image file size validation exceeding general limit""" + # Test image over general size limit (but under image limit is irrelevant) + image_size = MAX_FILE_SIZE + 1000 # 1KB over general limit + is_valid, error = FileUploadService.validate_file_size(image_size, 'image/png') + assert is_valid is False + # The service checks image size limit first, so we get image-specific error + assert "Image file size exceeds maximum" in error + assert "5.0MB" in error + + +class TestFileUploadServiceUpload: + """Test file upload functionality""" + + @pytest.fixture + def sample_file_data(self): + """Create sample file data for testing""" + content = b"This is a test file content" + encoded_content = base64.b64encode(content).decode('utf-8') + return { + 'content': encoded_content, + 'filename': 'test_file.txt', + 'content_type': 'text/plain', + 'size': len(content) + } + + @pytest.fixture + def sample_image_data(self): + """Create sample image data for testing""" + # Simple fake image content + content = b"fake_image_data_" * 100 # Small image + encoded_content = base64.b64encode(content).decode('utf-8') + return { + 'content': encoded_content, + 'filename': 'test_image.jpg', + 'content_type': 'image/jpeg', + 'size': len(content) + } + + @pytest.fixture + def large_file_data(self): + """Create large file data that exceeds size limits""" + content = b"x" * (MAX_FILE_SIZE + 1000) # Exceeds max file size + encoded_content = base64.b64encode(content).decode('utf-8') + return { + 'content': encoded_content, + 'filename': 'large_file.pdf', + 'content_type': 'application/pdf', + 'size': len(content) + } + + @pytest.mark.asyncio + async def test_upload_file_s3_success(self, sample_file_data): + """Test successful file upload to S3""" + org_id = str(uuid4()) + user_id = str(uuid4()) + + with patch('app.services.file_upload_service.settings') as mock_settings, \ + patch('app.services.file_upload_service.upload_file_to_s3') as mock_s3_upload, \ + patch('app.services.file_upload_service.resolve_public_url') as mock_signed_url, \ + patch('uuid.uuid4') as mock_uuid: + + # Configure mocks + mock_settings.S3_FILE_STORAGE = True + mock_uuid.return_value = Mock() + mock_uuid.return_value.__str__ = Mock(return_value='test-uuid-123') + + mock_s3_upload.return_value = 'https://s3.amazonaws.com/bucket/chat_attachments/org123/test-uuid-123.txt' + mock_signed_url.return_value = 'https://s3.amazonaws.com/bucket/signed-url' + + result = await FileUploadService.upload_file( + file_data=sample_file_data, + org_id=org_id, + user_id=user_id + ) + + # Verify S3 upload was called + mock_s3_upload.assert_called_once() + call_args = mock_s3_upload.call_args + assert call_args[1]['folder'] == f'chat_attachments/{org_id}' + assert call_args[1]['filename'] == 'test-uuid-123.txt' + assert call_args[1]['content_type'] == 'text/plain' + + # Verify signed URL was generated + mock_signed_url.assert_called_once() + + # Verify result + assert result['file_url'] == 'https://s3.amazonaws.com/bucket/chat_attachments/org123/test-uuid-123.txt' + assert result['signed_url'] == 'https://s3.amazonaws.com/bucket/signed-url' + assert result['filename'] == 'test_file.txt' + assert result['content_type'] == 'text/plain' + assert result['size'] == len(base64.b64decode(sample_file_data['content'])) + + @pytest.mark.asyncio + async def test_upload_file_local_success(self, sample_file_data): + """Test successful file upload to local storage""" + org_id = str(uuid4()) + customer_id = str(uuid4()) + + with patch('app.services.file_upload_service.settings') as mock_settings, \ + patch('uuid.uuid4') as mock_uuid, \ + patch('os.makedirs') as mock_makedirs, \ + patch('builtins.open', create=True) as mock_open: + + # Configure mocks + mock_settings.S3_FILE_STORAGE = False + mock_uuid.return_value = Mock() + mock_uuid.return_value.__str__ = Mock(return_value='test-uuid-456') + + # Mock file operations + mock_file = MagicMock() + mock_open.return_value.__enter__.return_value = mock_file + + result = await FileUploadService.upload_file( + file_data=sample_file_data, + org_id=org_id, + customer_id=customer_id + ) + + # Verify directory creation + mock_makedirs.assert_called_once_with(f'uploads/chat_attachments/{org_id}', exist_ok=True) + + # Verify file write + mock_open.assert_called_once_with(f'uploads/chat_attachments/{org_id}/test-uuid-456.txt', 'wb') + mock_file.write.assert_called_once() + + # Verify result + assert result['file_url'] == f'/uploads/chat_attachments/{org_id}/test-uuid-456.txt' + # Local attachments are never static-served: clients get a signed link. + assert result['signed_url'].startswith( + f'/api/v1/files/local/chat_attachments/{org_id}/test-uuid-456.txt?exp=') + assert result['filename'] == 'test_file.txt' + assert result['content_type'] == 'text/plain' + + @pytest.mark.asyncio + async def test_upload_file_missing_content(self): + """Test upload with missing content""" + file_data = { + 'filename': 'test.txt', + 'content_type': 'text/plain' + # Missing 'content' + } + + with pytest.raises(ValueError) as exc_info: + await FileUploadService.upload_file( + file_data=file_data, + org_id=str(uuid4()) + ) + + assert "Missing required file data" in str(exc_info.value) + + @pytest.mark.asyncio + async def test_upload_file_missing_filename(self): + """Test upload with missing filename""" + file_data = { + 'content': base64.b64encode(b"test content").decode('utf-8'), + 'content_type': 'text/plain' + # Missing 'filename' + } + + with pytest.raises(ValueError) as exc_info: + await FileUploadService.upload_file( + file_data=file_data, + org_id=str(uuid4()) + ) + + assert "Missing required file data" in str(exc_info.value) + + @pytest.mark.asyncio + async def test_upload_file_invalid_type(self, sample_file_data): + """Test upload with invalid file type""" + # Modify to invalid type + sample_file_data['content_type'] = 'video/mp4' + sample_file_data['filename'] = 'test.mp4' + + with pytest.raises(ValueError) as exc_info: + await FileUploadService.upload_file( + file_data=sample_file_data, + org_id=str(uuid4()) + ) + + # Error should indicate the file type is not allowed + assert "not allowed" in str(exc_info.value) or "File type" in str(exc_info.value) + + @pytest.mark.asyncio + async def test_upload_file_invalid_base64(self): + """Test upload with invalid base64 content""" + file_data = { + 'content': 'Z', # Invalid base64 - single character that can't be decoded properly + 'filename': 'test.txt', + 'content_type': 'text/plain' + } + + with pytest.raises(ValueError) as exc_info: + await FileUploadService.upload_file( + file_data=file_data, + org_id=str(uuid4()) + ) + + assert "Invalid file content encoding" in str(exc_info.value) + + @pytest.mark.asyncio + async def test_upload_file_size_exceeds_limit(self, large_file_data): + """Test upload with file size exceeding limits""" + with pytest.raises(ValueError) as exc_info: + await FileUploadService.upload_file( + file_data=large_file_data, + org_id=str(uuid4()) + ) + + assert "File size exceeds maximum" in str(exc_info.value) + + @pytest.mark.asyncio + async def test_upload_file_image_size_exceeds_limit(self): + """Test upload with image size exceeding image limits""" + # Create large image that exceeds image size limit + content = b"x" * (MAX_IMAGE_SIZE + 1000) + encoded_content = base64.b64encode(content).decode('utf-8') + + file_data = { + 'content': encoded_content, + 'filename': 'large_image.jpg', + 'content_type': 'image/jpeg' + } + + with pytest.raises(ValueError) as exc_info: + await FileUploadService.upload_file( + file_data=file_data, + org_id=str(uuid4()) + ) + + assert "Image file size exceeds maximum" in str(exc_info.value) + + @pytest.mark.asyncio + async def test_upload_file_s3_upload_failure(self, sample_file_data): + """Test handling of S3 upload failure""" + with patch('app.services.file_upload_service.settings') as mock_settings, \ + patch('app.services.file_upload_service.upload_file_to_s3') as mock_s3_upload: + + mock_settings.S3_FILE_STORAGE = True + mock_s3_upload.side_effect = Exception("S3 upload failed") + + with pytest.raises(ValueError) as exc_info: + await FileUploadService.upload_file( + file_data=sample_file_data, + org_id=str(uuid4()) + ) + + assert "Failed to upload file" in str(exc_info.value) + + @pytest.mark.asyncio + async def test_upload_file_local_write_failure(self, sample_file_data): + """Test handling of local file write failure""" + with patch('app.services.file_upload_service.settings') as mock_settings, \ + patch('os.makedirs') as mock_makedirs, \ + patch('builtins.open', create=True) as mock_open: + + mock_settings.S3_FILE_STORAGE = False + mock_open.side_effect = OSError("Permission denied") + + with pytest.raises(ValueError) as exc_info: + await FileUploadService.upload_file( + file_data=sample_file_data, + org_id=str(uuid4()) + ) + + assert "Failed to upload file" in str(exc_info.value) + + @pytest.mark.asyncio + async def test_upload_file_default_content_type(self): + """Test upload with default content type when not provided""" + content = b"test content" + file_data = { + 'content': base64.b64encode(content).decode('utf-8'), + 'filename': 'test.txt' + # No content_type provided - will default to 'application/octet-stream' which is not allowed + } + + # Should raise error because default content type is not allowed + with pytest.raises(ValueError) as exc_info: + await FileUploadService.upload_file( + file_data=file_data, + org_id=str(uuid4()) + ) + + # Error could be extension mismatch or file type not allowed + error_str = str(exc_info.value).lower() + assert "not allowed" in error_str or "does not match" in error_str or "file type" in error_str + + @pytest.mark.asyncio + async def test_upload_file_folder_structure_user(self, sample_file_data): + """Test folder structure for user uploads""" + org_id = str(uuid4()) + user_id = str(uuid4()) + + with patch('app.services.file_upload_service.settings') as mock_settings, \ + patch('app.services.file_upload_service.upload_file_to_s3') as mock_s3_upload, \ + patch('app.services.file_upload_service.resolve_public_url') as mock_signed_url: + + mock_settings.S3_FILE_STORAGE = True + mock_s3_upload.return_value = 'https://s3.amazonaws.com/bucket/file.txt' + mock_signed_url.return_value = 'https://s3.amazonaws.com/bucket/signed' + + await FileUploadService.upload_file( + file_data=sample_file_data, + org_id=org_id, + user_id=user_id + ) + + # Verify folder structure + call_args = mock_s3_upload.call_args + assert call_args[1]['folder'] == f'chat_attachments/{org_id}' + + @pytest.mark.asyncio + async def test_upload_file_folder_structure_customer(self, sample_file_data): + """Test folder structure for customer uploads""" + org_id = str(uuid4()) + customer_id = str(uuid4()) + + with patch('app.services.file_upload_service.settings') as mock_settings, \ + patch('app.services.file_upload_service.upload_file_to_s3') as mock_s3_upload, \ + patch('app.services.file_upload_service.resolve_public_url') as mock_signed_url: + + mock_settings.S3_FILE_STORAGE = True + mock_s3_upload.return_value = 'https://s3.amazonaws.com/bucket/file.txt' + mock_signed_url.return_value = 'https://s3.amazonaws.com/bucket/signed' + + await FileUploadService.upload_file( + file_data=sample_file_data, + org_id=org_id, + customer_id=customer_id + ) + + # Verify folder structure (same as user for now) + call_args = mock_s3_upload.call_args + assert call_args[1]['folder'] == f'chat_attachments/{org_id}' + + @pytest.mark.asyncio + async def test_upload_file_unique_filename_generation(self, sample_file_data): + """Test that unique filenames are generated""" + org_id = str(uuid4()) + + with patch('app.services.file_upload_service.settings') as mock_settings, \ + patch('app.services.file_upload_service.upload_file_to_s3') as mock_s3_upload, \ + patch('app.services.file_upload_service.resolve_public_url') as mock_signed_url, \ + patch('uuid.uuid4') as mock_uuid: + + mock_settings.S3_FILE_STORAGE = True + mock_s3_upload.return_value = 'https://s3.amazonaws.com/bucket/file.txt' + mock_signed_url.return_value = 'https://s3.amazonaws.com/bucket/signed' + + # Mock UUID generation + mock_uuid.return_value = Mock() + mock_uuid.return_value.__str__ = Mock(return_value='unique-id-789') + + await FileUploadService.upload_file( + file_data=sample_file_data, + org_id=org_id + ) + + # Verify unique filename was generated + call_args = mock_s3_upload.call_args + assert call_args[1]['filename'] == 'unique-id-789.txt' + + @pytest.mark.asyncio + async def test_upload_file_no_extension(self): + """Test upload with filename that has no extension""" + content = b"test content" + file_data = { + 'content': base64.b64encode(content).decode('utf-8'), + 'filename': 'test_file_no_extension', + 'content_type': 'text/plain' + } + + # Should raise error because file has no extension + with pytest.raises(ValueError) as exc_info: + await FileUploadService.upload_file( + file_data=file_data, + org_id=str(uuid4()) + ) + + assert "must have an extension" in str(exc_info.value) or "extension" in str(exc_info.value).lower() + + +class TestFileUploadServiceConstants: + """Test service constants and configurations""" + + def test_allowed_file_types_combination(self): + """Test that ALLOWED_FILE_TYPES is correct combination""" + expected_types = ALLOWED_IMAGE_TYPES | ALLOWED_DOCUMENT_TYPES + assert ALLOWED_FILE_TYPES == expected_types + + def test_file_size_constants(self): + """Test file size constants are reasonable""" + assert MAX_FILE_SIZE == 10 * 1024 * 1024 # 10MB + assert MAX_IMAGE_SIZE == 5 * 1024 * 1024 # 5MB + assert MAX_IMAGE_SIZE < MAX_FILE_SIZE # Image limit should be smaller + + def test_allowed_image_types_content(self): + """Test allowed image types include common formats (SVG excluded for XSS safety)""" + expected_image_types = { + 'image/jpeg', 'image/jpg', 'image/png', + 'image/gif', 'image/webp' + } + assert ALLOWED_IMAGE_TYPES == expected_image_types + # Verify SVG is explicitly NOT included + assert 'image/svg+xml' not in ALLOWED_IMAGE_TYPES + + def test_allowed_document_types_content(self): + """Test allowed document types include common formats""" + expected_doc_types = { + 'application/pdf', + 'application/msword', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'text/plain', + 'text/csv', + 'application/vnd.ms-excel', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' + } + assert ALLOWED_DOCUMENT_TYPES == expected_doc_types diff --git a/backend/tests/services/test_image_security.py b/backend/tests/services/test_image_security.py new file mode 100644 index 0000000..f9d7087 --- /dev/null +++ b/backend/tests/services/test_image_security.py @@ -0,0 +1,94 @@ +""" +Copyright 2024-2026 ReplyPier + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import io + +import pytest +from fastapi import HTTPException +from PIL import Image + +from app.services.image_security import sanitize_image + +RASTER = {"image/png", "image/jpeg", "image/webp"} + + +def _img_bytes(fmt: str, size=(64, 48), color=(200, 30, 30)) -> bytes: + buf = io.BytesIO() + Image.new("RGB", size, color).save(buf, format=fmt) + return buf.getvalue() + + +def test_png_reencoded_to_png(): + out, ctype, ext = sanitize_image( + _img_bytes("PNG"), allowed_content_types=RASTER, max_bytes=2_000_000, max_dim=4000 + ) + assert ctype == "image/png" and ext == ".png" + assert Image.open(io.BytesIO(out)).format == "PNG" + + +def test_jpeg_stays_jpeg(): + out, ctype, ext = sanitize_image( + _img_bytes("JPEG"), allowed_content_types=RASTER, max_bytes=2_000_000, max_dim=4000 + ) + assert ctype == "image/jpeg" and ext == ".jpg" + assert Image.open(io.BytesIO(out)).format == "JPEG" + + +def test_fit_downscales_longest_side(): + out, _c, _e = sanitize_image( + _img_bytes("PNG", size=(1200, 600)), + allowed_content_types=RASTER, + max_bytes=2_000_000, + max_dim=4000, + fit=512, + ) + w, h = Image.open(io.BytesIO(out)).size + assert max(w, h) == 512 and (w, h) == (512, 256) + + +def test_rejects_non_image(): + with pytest.raises(HTTPException) as e: + sanitize_image(b"not an image", allowed_content_types=RASTER, max_bytes=2_000_000, max_dim=4000) + assert e.value.status_code == 400 + + +def test_rejects_oversized_bytes(): + with pytest.raises(HTTPException) as e: + sanitize_image(_img_bytes("PNG"), allowed_content_types=RASTER, max_bytes=10, max_dim=4000) + assert e.value.status_code == 400 + + +def test_rejects_dimensions_over_cap(): + with pytest.raises(HTTPException) as e: + sanitize_image( + _img_bytes("PNG", size=(300, 300)), + allowed_content_types=RASTER, + max_bytes=2_000_000, + max_dim=100, + ) + assert e.value.status_code == 400 + + +def test_rejects_disallowed_format(): + # GIF is a valid image but not in the allowlist here → rejected. + with pytest.raises(HTTPException) as e: + sanitize_image(_img_bytes("GIF"), allowed_content_types=RASTER, max_bytes=2_000_000, max_dim=4000) + assert e.value.status_code == 400 + + +def test_empty_rejected(): + with pytest.raises(HTTPException): + sanitize_image(b"", allowed_content_types=RASTER, max_bytes=2_000_000, max_dim=4000) diff --git a/backend/tests/services/test_upload_paths.py b/backend/tests/services/test_upload_paths.py new file mode 100644 index 0000000..a41aff3 --- /dev/null +++ b/backend/tests/services/test_upload_paths.py @@ -0,0 +1,88 @@ +""" +Copyright 2024-2026 ReplyPier + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import io +import os +from uuid import uuid4 + +import pytest +from fastapi import UploadFile +from PIL import Image + +from app.api.agent import save_file +from app.api.users import save_upload_file +from app.core.config import settings +from app.services.file_storage import local_upload_path, store_upload + +MOUNT = f"{settings.API_V1_STR}/uploads" + + +def _png_upload(name: str = "a.png") -> UploadFile: + buf = io.BytesIO() + Image.new("RGB", (2, 2), "red").save(buf, format="PNG") + buf.seek(0) + return UploadFile(filename=name, file=buf, headers={"content-type": "image/png"}) + + +class TestLocalUploadPath: + """The stored-URL → filesystem-path direction has to read every shape the + app has written, because rows predating the normalisation are still bare.""" + + def test_current_prefixed_shape(self): + assert local_upload_path(f"{MOUNT}/agents/org/a.png") == os.path.join( + "uploads", "agents/org/a.png") + + def test_legacy_bare_shape(self): + # What agent photos and profile pictures carried before the migration. + assert local_upload_path("/uploads/agents/org/a.png") == os.path.join( + "uploads", "agents/org/a.png") + + def test_relative_key(self): + assert local_upload_path("agents/org/a.png") == os.path.join( + "uploads", "agents/org/a.png") + + def test_never_doubles_the_uploads_segment(self): + for stored in (f"{MOUNT}/x.png", "/uploads/x.png", "uploads/x.png", "x.png"): + assert local_upload_path(stored) == os.path.join("uploads", "x.png"), stored + + +class TestWritersAgree: + """Every local writer must emit the same prefix, or the frontend resolves + the path against the API origin and 404s off the static mount.""" + + @pytest.mark.asyncio + async def test_store_upload_prefixes(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(settings, "S3_FILE_STORAGE", False) + stored = await store_upload(b"x", "help_center", "a.png", "image/png") + assert stored.startswith(f"{MOUNT}/") + + @pytest.mark.asyncio + async def test_agent_photo_prefixes_and_round_trips(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(settings, "S3_FILE_STORAGE", False) + stored = await save_file(_png_upload(), uuid4()) + assert stored.startswith(f"{MOUNT}/") + # The delete path reads it back through local_upload_path. + assert os.path.exists(local_upload_path(stored)) + + @pytest.mark.asyncio + async def test_profile_pic_prefixes_and_round_trips(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(settings, "S3_FILE_STORAGE", False) + stored = await save_upload_file(_png_upload(), str(uuid4()), str(uuid4())) + assert stored.startswith(f"{MOUNT}/") + assert os.path.exists(local_upload_path(stored)) diff --git a/frontend/src/__tests__/components/knowledge/KnowledgeAddSourceModal.scope.spec.ts b/frontend/src/__tests__/components/knowledge/KnowledgeAddSourceModal.scope.spec.ts new file mode 100644 index 0000000..eeae369 --- /dev/null +++ b/frontend/src/__tests__/components/knowledge/KnowledgeAddSourceModal.scope.spec.ts @@ -0,0 +1,71 @@ +/* +Copyright 2024-2026 ReplyPier + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { describe, it, expect } from 'vitest' +import { mount } from '@vue/test-utils' +import KnowledgeAddSourceModal from '../../../components/knowledge/KnowledgeAddSourceModal.vue' + +const mountModal = async (url: string) => { + const wrapper = mount(KnowledgeAddSourceModal) + await wrapper.find('#kb-add-url').setValue(url) + return wrapper +} + +const scopeTitles = (wrapper: ReturnType) => + wrapper.findAll('.scope__title').map((node) => node.text()) + +const scopeOption = (wrapper: ReturnType, title: string) => + wrapper.findAll('.scope__opt').find((node) => node.find('.scope__title').text() === title) + +describe('KnowledgeAddSourceModal crawl scope', () => { + it('offers no section option for a homepage URL', async () => { + // "Pages under / " is not a section — it is the whole site, which the next + // option already covers. + const wrapper = await mountModal('https://paywithatoa.co.uk') + expect(scopeTitles(wrapper)).toEqual(['仅此页面', '整个站点', '整个域名']) + expect(scopeOption(wrapper, '整个站点')!.text()).toContain('paywithatoa.co.uk 上的所有页面') + }) + + it('names the section when the URL has a path', async () => { + const wrapper = await mountModal('https://help.example.com/hc/en') + expect(scopeTitles(wrapper)).toContain('本栏目') + expect(scopeOption(wrapper, '本栏目')!.text()).toContain( + '/hc/en/ 下的页面', + ) + }) + + it('falls back to this-site when the section option disappears', async () => { + const wrapper = await mountModal('https://help.example.com/hc/en') + await scopeOption(wrapper, '本栏目')!.trigger('click') + + await wrapper.find('#kb-add-url').setValue('https://help.example.com') + await wrapper.find('button.btn--primary').trigger('click') + + expect(wrapper.emitted('submit')![0][0]).toMatchObject({ + type: 'website', + url: 'https://help.example.com', + scope: 'host', + }) + }) + + it('submits the selected scope', async () => { + const wrapper = await mountModal('https://help.example.com/hc/en') + await scopeOption(wrapper, '本栏目')!.trigger('click') + await wrapper.find('button.btn--primary').trigger('click') + + expect(wrapper.emitted('submit')![0][0]).toMatchObject({ scope: 'path' }) + }) +}) diff --git a/frontend/src/__tests__/composables/useKnowledgeManagement.url.spec.ts b/frontend/src/__tests__/composables/useKnowledgeManagement.url.spec.ts new file mode 100644 index 0000000..ece5ad0 --- /dev/null +++ b/frontend/src/__tests__/composables/useKnowledgeManagement.url.spec.ts @@ -0,0 +1,116 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { setActivePinia, createPinia } from 'pinia' + +vi.mock('@/services/knowledge', () => ({ + knowledgeService: { + getKnowledgeByAgent: vi.fn().mockResolvedValue({ knowledge: [], pagination: { total_pages: 0 } }), + getKnowledgeByOrganization: vi + .fn() + .mockResolvedValue({ knowledge: [], pagination: { total_pages: 0 } }), + getAgentQueueItems: vi.fn().mockResolvedValue({ queue_items: [] }) + } +})) + +import { useKnowledgeManagement } from '@/composables/useKnowledgeManagement' + +const setup = () => useKnowledgeManagement('agent-1', 'org-1') + +describe('useKnowledgeManagement - URL handling', () => { + beforeEach(() => { + setActivePinia(createPinia()) + vi.clearAllMocks() + }) + + describe('withScheme', () => { + const { withScheme } = setup() + + it.each([ + ['docs.company.com', 'https://docs.company.com'], + ['company.co.uk', 'https://company.co.uk'], + [' spaced.com ', 'https://spaced.com'], + ['//protocol-relative.com', 'https://protocol-relative.com'] + ])('defaults the scheme: %s -> %s', (input, expected) => { + expect(withScheme(input)).toBe(expected) + }) + + it.each([ + 'https://already.com', + 'http://plain.com', + 'HTTPS://shouty.com' + ])('leaves an existing scheme alone: %s', (input) => { + expect(withScheme(input)).toBe(input) + }) + + it('leaves an empty value empty', () => { + expect(withScheme(' ')).toBe('') + }) + }) + + describe('isValidUrl', () => { + const { isValidUrl } = setup() + + it.each(['https://docs.company.com', 'http://a.co', 'https://x.com/path?q=1'])( + 'accepts %s', + (url) => expect(isValidUrl(url)).toBe(true) + ) + + it.each(['mailto:someone@x.com', 'foo:bar', 'not a url', 'https://'])( + 'rejects %s', + (url) => expect(isValidUrl(url)).toBe(false) + ) + + it.each(['https://localhost', 'https://wiki', 'http://intranet:8080'])( + 'accepts the intranet hostnames a self-hosted install indexes: %s', + (url) => expect(isValidUrl(url)).toBe(true) + ) + }) + + describe('handleUrlAdd', () => { + it('stages a scheme-less URL instead of rejecting it', () => { + const { newUrl, urls, urlFormError, handleUrlAdd } = setup() + newUrl.value = 'docs.company.com' + + handleUrlAdd() + + expect(urls.value).toEqual(['https://docs.company.com']) + expect(urlFormError.value).toBeNull() + expect(newUrl.value).toBe('') + }) + + it('still rejects input that is not a URL at all', () => { + const { newUrl, urls, urlFormError, handleUrlAdd } = setup() + newUrl.value = 'not a url' + + handleUrlAdd() + + expect(urls.value).toEqual([]) + expect(urlFormError.value).toBe('请输入有效的 URL') + }) + + it('clears a previous error when a later attempt succeeds', () => { + const { newUrl, urls, urlFormError, handleUrlAdd } = setup() + + newUrl.value = 'not a url' + handleUrlAdd() + expect(urlFormError.value).toBe('请输入有效的 URL') + + newUrl.value = 'docs.company.com' + handleUrlAdd() + + expect(urlFormError.value).toBeNull() + expect(urls.value).toEqual(['https://docs.company.com']) + }) + + it('does not stage the same URL twice', () => { + const { newUrl, urls, urlFormError, handleUrlAdd } = setup() + + newUrl.value = 'docs.company.com' + handleUrlAdd() + newUrl.value = 'https://docs.company.com' + handleUrlAdd() + + expect(urls.value).toEqual(['https://docs.company.com']) + expect(urlFormError.value).toBe('该 URL 已添加到当前批次') + }) + }) +}) diff --git a/frontend/src/__tests__/composables/useStagedSourceCap.spec.ts b/frontend/src/__tests__/composables/useStagedSourceCap.spec.ts new file mode 100644 index 0000000..e884c99 --- /dev/null +++ b/frontend/src/__tests__/composables/useStagedSourceCap.spec.ts @@ -0,0 +1,55 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { ref } from 'vue' + +const planLimits = vi.fn() +vi.mock('@/utils/storage', () => ({ + subscriptionStorage: { getPlanLimits: () => planLimits() } +})) + +import { useStagedSourceCap } from '@/composables/useStagedSourceCap' + +const file = (name: string) => new File(['x'], name, { type: 'application/pdf' }) + +const setup = (cap: number, indexed: number, urls: string[], files: File[]) => { + planLimits.mockReturnValue({ maxKnowledgeSources: cap }) + const u = ref(urls) + const f = ref(files) + return { u, f, ...useStagedSourceCap(ref(indexed), u, f) } +} + +describe('useStagedSourceCap', () => { + beforeEach(() => planLimits.mockReset()) + + it('does nothing when the plan sets no cap', () => { + const { u, f, fitToCap } = setup(0, 3, ['a', 'b'], [file('x.pdf')]) + expect(fitToCap('urls')).toBeNull() + expect(u.value).toEqual(['a', 'b']) + expect(f.value).toHaveLength(1) + }) + + it('drops staged files before older URLs, oldest first', () => { + const { u, f, fitToCap } = setup(2, 0, ['a', 'b'], [file('x.pdf')]) + expect(fitToCap('urls')).toBeNull() + expect(f.value).toEqual([]) + expect(u.value).toEqual(['a', 'b']) + }) + + it('drops the oldest URL when only URLs are staged', () => { + const { u, fitToCap } = setup(2, 0, ['a', 'b', 'c'], []) + expect(fitToCap('urls')).toBeNull() + expect(u.value).toEqual(['b', 'c']) + }) + + it('unstages new files that cannot fit and says how many can', () => { + const { u, f, fitToCap } = setup(3, 1, ['a'], [file('x.pdf'), file('y.pdf'), file('z.pdf')]) + expect(fitToCap('files', 3)).toBe('你的套餐包含 3 个知识库来源，最多可选 2 个。') + expect(f.value).toEqual([]) + expect(u.value).toEqual(['a']) + }) + + it('reports a cap already filled by indexed sources', () => { + const { u, fitToCap } = setup(1, 1, ['a'], []) + expect(fitToCap('urls')).toBe('你的套餐包含 1 个知识库来源，且已在用。') + expect(u.value).toEqual([]) + }) +}) diff --git a/frontend/src/components/knowledge/KnowledgeAddSourceModal.vue b/frontend/src/components/knowledge/KnowledgeAddSourceModal.vue new file mode 100644 index 0000000..e5a465a --- /dev/null +++ b/frontend/src/components/knowledge/KnowledgeAddSourceModal.vue @@ -0,0 +1,597 @@ + + + + + + diff --git a/frontend/src/components/knowledge/KnowledgeExplorer.vue b/frontend/src/components/knowledge/KnowledgeExplorer.vue new file mode 100644 index 0000000..d2b757a --- /dev/null +++ b/frontend/src/components/knowledge/KnowledgeExplorer.vue @@ -0,0 +1,505 @@ + + + + + + diff --git a/frontend/src/components/knowledge/KnowledgeLinkModal.vue b/frontend/src/components/knowledge/KnowledgeLinkModal.vue new file mode 100644 index 0000000..d1b2185 --- /dev/null +++ b/frontend/src/components/knowledge/KnowledgeLinkModal.vue @@ -0,0 +1,308 @@ + + + + + + diff --git a/frontend/src/components/knowledge/KnowledgePageDetail.vue b/frontend/src/components/knowledge/KnowledgePageDetail.vue new file mode 100644 index 0000000..e65f4e4 --- /dev/null +++ b/frontend/src/components/knowledge/KnowledgePageDetail.vue @@ -0,0 +1,377 @@ + + + + + + diff --git a/frontend/src/components/knowledge/KnowledgePageEditor.vue b/frontend/src/components/knowledge/KnowledgePageEditor.vue new file mode 100644 index 0000000..31dc4c6 --- /dev/null +++ b/frontend/src/components/knowledge/KnowledgePageEditor.vue @@ -0,0 +1,216 @@ + + + + + + diff --git a/frontend/src/components/knowledge/KnowledgePlanMeters.vue b/frontend/src/components/knowledge/KnowledgePlanMeters.vue new file mode 100644 index 0000000..5d667ec --- /dev/null +++ b/frontend/src/components/knowledge/KnowledgePlanMeters.vue @@ -0,0 +1,171 @@ + + + + + + diff --git a/frontend/src/components/knowledge/KnowledgeSourceTree.vue b/frontend/src/components/knowledge/KnowledgeSourceTree.vue new file mode 100644 index 0000000..2f0fa05 --- /dev/null +++ b/frontend/src/components/knowledge/KnowledgeSourceTree.vue @@ -0,0 +1,501 @@ + + + + + + diff --git a/frontend/src/composables/useKnowledgeExplorer.ts b/frontend/src/composables/useKnowledgeExplorer.ts new file mode 100644 index 0000000..d750faa --- /dev/null +++ b/frontend/src/composables/useKnowledgeExplorer.ts @@ -0,0 +1,669 @@ +/* +Copyright 2024-2026 ReplyPier + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { computed, ref } from 'vue' +import { toast } from 'vue-sonner' +import { showPlanLimitError } from '@/utils/planLimitToast' +import type { + KnowledgeItem, + KnowledgeLinkedAgent, + KnowledgePage, + KnowledgeSubPage, + QueueItem, +} from '@/types/knowledge' +import { knowledgeService } from '@/services/knowledge' +import { basePageId, groupChunksIntoPages, titleFromId } from '@/utils/knowledgePages' + +export type ExplorerMode = 'agent' | 'org' +export type SourceStatus = 'queued' | 'crawling' | 'synced' | 'error' + +// How far a website crawl may follow links from the URL it starts at. +// 'page' indexes only that URL; the rest map to the backend's crawl scopes. +export type CrawlScope = 'page' | 'path' | 'host' | 'domain' + +// What the add-source modal submits; discriminated by `type`. +export type AddSourcePayload = + | { type: 'website'; url: string; scope: CrawlScope } + | { type: 'sitemap'; url: string } + | { type: 'pdf'; files: File[] } + | { type: 'text'; title: string; content: string } + +/** A knowledge source enriched with lazily-loaded, grouped sub-page content. */ +export interface ExplorerSource { + id: number + name: string + type: string + // Agents this source is linked to (for the "Used by" chips). + agents: KnowledgeLinkedAgent[] + pageStubs: KnowledgePage[] + expanded: boolean + loadingContent: boolean + contentError: string | null + pages: KnowledgeSubPage[] | null + // True for a placeholder built from an in-flight queue item that has not yet + // produced a real Knowledge source (id is the negated queue-item id). + queued?: boolean + // Status carried from the queue item for a placeholder (its display name may + // differ from the queue `source`, so it can't be re-derived by name lookup). + queuedStatus?: SourceStatus + // Failure reason for a failed queue item (e.g. blocked by bot protection). + queuedError?: string | null +} + +const POLL_INTERVAL_MS = 10000 + +/** + * State engine for the master-detail knowledge explorer, shared by the + * agent-editor tab (`mode: 'agent'`) and the dedicated knowledge page + * (`mode: 'org'`). Source list is fetched eagerly; each source's sub-page + * content is grouped from its chunks lazily on first expand/selection. + */ +export function useKnowledgeExplorer( + mode: ExplorerMode, + agentId: string | undefined, + organizationId: string, +) { + const sources = ref([]) + const queueItems = ref([]) + const isLoading = ref(false) + const error = ref(null) + const query = ref('') + + const selectedSourceId = ref(null) + const selectedPageId = ref(null) + + const editing = ref(false) + const isAddingPage = ref(false) + const draftTitle = ref('') + const draftContent = ref('') + const draftUrl = ref('') + const isSaving = ref(false) + const isDeleting = ref(false) + + let pollTimer: ReturnType | null = null + // Source names that were crawling on the previous poll tick, so we can force a + // final content refresh once a crawl finishes. + let crawlingNames = new Set() + // Queue-item ids seen on the previous tick, so we detect a crawl finishing + // (an item leaving the queue) even if it was never observed as active. + let lastQueueIds = new Set() + + const queueIds = () => new Set(queueItems.value.map((q) => q.id)) + const activeCrawlNames = () => + new Set( + queueItems.value + .filter((q) => q.status === 'pending' || q.status === 'processing') + .map((q) => q.source), + ) + const setsDiffer = (a: Set, b: Set) => + a.size !== b.size || [...a].some((id) => !b.has(id)) + + const findSource = (id: number) => sources.value.find((s) => s.id === id) + + const toExplorerSource = (item: KnowledgeItem): ExplorerSource => ({ + id: item.id, + name: item.name, + type: item.type, + agents: item.agents || [], + pageStubs: item.pages || [], + expanded: false, + loadingContent: false, + contentError: null, + pages: null, + }) + + // `background` refetches (from the crawl poll) skip the loading flag so any + // bound spinner doesn't flicker every tick. + const fetchSources = async (background = false) => { + if (!background) isLoading.value = true + error.value = null + try { + const response = + mode === 'agent' && agentId + ? await knowledgeService.getKnowledgeByAgent(agentId, 1, 100) + : await knowledgeService.getKnowledgeByOrganization(organizationId, 1, 100) + + const items: KnowledgeItem[] = response.knowledge || [] + // Preserve expand/loaded state across refreshes so a running crawl poll + // does not collapse the tree or drop already-fetched content. + const previous = new Map(sources.value.map((s) => [s.id, s])) + sources.value = items.map((item) => { + const prev = previous.get(item.id) + const next = toExplorerSource(item) + if (prev) { + next.expanded = prev.expanded + next.pages = prev.pages + } + return next + }) + } catch (err) { + console.error('Failed to load knowledge sources:', err) + error.value = '加载知识库来源失败' + } finally { + if (!background) isLoading.value = false + } + } + + const fetchQueue = async () => { + try { + const response = + mode === 'agent' && agentId + ? await knowledgeService.getAgentQueueItems(agentId) + : await knowledgeService.getOrgQueueItems(organizationId) + queueItems.value = response.queue_items || [] + } catch (err) { + console.error('Failed to load knowledge queue:', err) + } + } + + // A queue item's source_type ('website' | 'pdf_url' | 'pdf_file') mapped to the + // tree's source-kind glyph. + const queueKind = (sourceType: string): string => { + if (sourceType.includes('pdf')) return 'pdf' + if (sourceType.includes('site')) return 'sitemap' + if (sourceType.includes('web')) return 'web' + return 'custom' + } + + // A queue item's `source` is a URL for websites but an internal temp path or + // S3 signed URL for PDFs — show a readable filename for the placeholder. + const queueSourceName = (item: QueueItem): string => { + if (!item.source_type.includes('pdf')) return item.source + const path = item.source.split(/[?#]/)[0] + const base = decodeURIComponent(path.slice(path.lastIndexOf('/') + 1)) + .replace(/\.[^.]+$/, '') // drop extension + .replace(/^[0-9a-f-]{8,}[_-]/i, '') // drop a leading uuid_ prefix + return base || item.source + } + + const queueStatusToSource = (status: string): SourceStatus => + status === 'failed' ? 'error' : status === 'processing' ? 'crawling' : 'queued' + + // Placeholder source for an in-flight queue item (negative id = queue-item id). + const toSyntheticSource = (item: QueueItem): ExplorerSource => ({ + id: -item.id, + name: queueSourceName(item), + type: queueKind(item.source_type), + agents: [], + pageStubs: [], + expanded: false, + loadingContent: false, + contentError: null, + pages: [], + queued: true, + queuedStatus: queueStatusToSource(item.status), + queuedError: item.status === 'failed' ? item.error ?? null : null, + }) + + // Real sources plus a placeholder for every in-flight queue item that has not + // yet produced a real source — so a just-queued crawl shows immediately and is + // replaced by the real source once indexing completes and the poll refetches. + const allSources = computed(() => { + const realNames = new Set(sources.value.map((s) => s.name)) + const seen = new Set() + const placeholders: ExplorerSource[] = [] + for (const item of queueItems.value) { + if (realNames.has(item.source) || seen.has(item.source)) continue + seen.add(item.source) + placeholders.push(toSyntheticSource(item)) + } + return [...placeholders, ...sources.value] + }) + + // Resolve the source by id on every mutation. A concurrent (poll-driven) + // fetchSources() replaces the objects in `sources.value`, so writing to the + // originally-captured `source` would be lost — always re-find the live one. + const loadSourceContent = async (sourceId: number, force = false) => { + const source = findSource(sourceId) + if (!source || source.loadingContent) return + if (source.pages && !force) return + source.loadingContent = true + source.contentError = null + try { + const content = await knowledgeService.getKnowledgeContent(sourceId) + const pages = groupChunksIntoPages(content.chunks || []) + const live = findSource(sourceId) + if (live) live.pages = pages + } catch (err) { + console.error(`Failed to load content for source ${sourceId}:`, err) + const live = findSource(sourceId) + if (live) { + live.contentError = '加载页面失败' + live.pages = [] + } + } finally { + const live = findSource(sourceId) + if (live) live.loadingContent = false + } + } + + const toggleSource = async (source: ExplorerSource) => { + source.expanded = !source.expanded + if (source.expanded && source.pages === null) { + await loadSourceContent(source.id) + } + } + + const selectPage = async (source: ExplorerSource, pageId: string) => { + selectedSourceId.value = source.id + selectedPageId.value = pageId + editing.value = false + isAddingPage.value = false + if (source.pages === null) { + await loadSourceContent(source.id) + } + } + + const selectedSource = computed( + () => sources.value.find((s) => s.id === selectedSourceId.value) ?? null, + ) + + const selectedPage = computed(() => { + const source = selectedSource.value + if (!source || !source.pages || selectedPageId.value === null) return null + return source.pages.find((p) => p.page_id === selectedPageId.value) ?? null + }) + + const sourceStatus = (source: ExplorerSource): SourceStatus => { + // Placeholders carry their own status (their display name differs from the + // queue source, so a name lookup wouldn't find the item). + if (source.queued) return source.queuedStatus ?? 'queued' + const item = queueItems.value.find((q) => q.source === source.name) + if (item?.status === 'failed') return 'error' + if (item?.status === 'processing') return 'crawling' + if (item?.status === 'pending') return 'queued' + return 'synced' + } + + // Page rows for the tree: grouped pages once loaded, else lightweight stubs. + const pageRows = (source: ExplorerSource) => { + if (source.pages) { + return source.pages.map((p) => ({ + page_id: p.page_id, + title: p.title, + words: p.word_count, + })) + } + return source.pageStubs.map((stub: KnowledgePage) => ({ + page_id: stub.subpage, + title: titleFromId(stub.subpage), + words: null as number | null, + })) + } + + const normalizedQuery = computed(() => query.value.trim().toLowerCase()) + + const filteredSources = computed(() => { + const q = normalizedQuery.value + if (!q) return allSources.value + return allSources.value.filter((source) => { + if (source.name.toLowerCase().includes(q)) return true + const rows = pageRows(source) + return rows.some((r) => r.title.toLowerCase().includes(q) || r.page_id.toLowerCase().includes(q)) + }) + }) + + const startEdit = () => { + const page = selectedPage.value + if (!page) return + draftTitle.value = page.title + draftContent.value = page.content + isAddingPage.value = false + editing.value = true + } + + // Open the editor with empty fields to add a new sub-page to `source`. + const startAddPage = async (source: ExplorerSource) => { + if (source.queued) return + selectedSourceId.value = source.id + selectedPageId.value = null + draftTitle.value = '' + draftContent.value = '' + draftUrl.value = '' + isAddingPage.value = true + editing.value = true + source.expanded = true + if (source.pages === null) await loadSourceContent(source.id) + } + + const cancelEdit = () => { + editing.value = false + isAddingPage.value = false + } + + const addNewPage = async (source: ExplorerSource) => { + const title = draftTitle.value.trim() + const content = draftContent.value.trim() + if (!title) { + error.value = '请为子页面填写标题' + toast.error('请为子页面填写标题') + return + } + if (!content) { + error.value = '子页面内容不能为空' + return + } + isSaving.value = true + error.value = null + try { + await knowledgeService.addSubpage(source.id, title, content, draftUrl.value.trim() || undefined) + await loadSourceContent(source.id, true) + selectedPageId.value = basePageId(title) + isAddingPage.value = false + editing.value = false + toast.success('子页面已添加') + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : '添加子页面失败' + console.error('Failed to add sub-page:', err) + error.value = msg + void showPlanLimitError('无法添加子页面', msg) + } finally { + isSaving.value = false + } + } + + const savePage = async () => { + const source = selectedSource.value + if (!source) return + if (isAddingPage.value) return addNewPage(source) + + const page = selectedPage.value + if (!page) return + const content = draftContent.value.trim() + if (!content) { + error.value = '页面内容不能为空' + return + } + isSaving.value = true + error.value = null + try { + await knowledgeService.updatePage(source.id, page.page_id, content, draftTitle.value.trim()) + await loadSourceContent(source.id, true) + editing.value = false + } catch (err: unknown) { + console.error('Failed to save page:', err) + error.value = err instanceof Error ? err.message : '保存页面失败' + } finally { + isSaving.value = false + } + } + + const deletePage = async () => { + const source = selectedSource.value + const page = selectedPage.value + if (!source || !page) return + isDeleting.value = true + error.value = null + try { + await knowledgeService.deletePage(source.id, page.page_id) + selectedPageId.value = null + await loadSourceContent(source.id, true) + } catch (err: unknown) { + console.error('Failed to delete page:', err) + error.value = err instanceof Error ? err.message : '删除页面失败' + } finally { + isDeleting.value = false + } + } + + const deleteSource = async (source: ExplorerSource) => { + isDeleting.value = true + error.value = null + try { + // A queued placeholder isn't a real source yet — cancel its queue item. + if (source.queued) { + await knowledgeService.deleteQueueItem(-source.id) + } else { + await knowledgeService.deleteKnowledge(source.id) + } + if (selectedSourceId.value === source.id) { + selectedSourceId.value = null + selectedPageId.value = null + } + await Promise.all([fetchSources(), fetchQueue()]) + } catch (err: unknown) { + console.error('Failed to delete source:', err) + error.value = err instanceof Error ? err.message : '删除来源失败' + } finally { + isDeleting.value = false + } + } + + // ---- Link existing org-level knowledge to this agent (agent mode only) ---- + // Org sources are created with agent_id=NULL, so they never auto-link to an + // agent; this picker restores the ability to attach them from the agent tab. + const linkPickerOpen = ref(false) + const orgSources = ref([]) + const isLoadingOrgSources = ref(false) + const orgSourcesError = ref(null) + // Knowledge ids with a link/unlink request in flight (per-row busy state). + const linkingIds = ref>(new Set()) + + // Ids already linked to this agent — drives the Link/Unlink toggle per row. + const linkedSourceIds = computed(() => new Set(sources.value.map((s) => s.id))) + + const loadOrgSources = async () => { + isLoadingOrgSources.value = true + orgSourcesError.value = null + try { + const response = await knowledgeService.getKnowledgeByOrganization(organizationId, 1, 100) + orgSources.value = response.knowledge || [] + } catch (err) { + console.error('Failed to load organization knowledge:', err) + orgSourcesError.value = '加载组织知识库失败' + } finally { + isLoadingOrgSources.value = false + } + } + + const openLinkPicker = async () => { + if (mode !== 'agent' || !agentId) return + linkPickerOpen.value = true + await loadOrgSources() + } + + // Shared link/unlink runner: guards concurrent clicks per row, keeps the + // busy-state Set in sync, refetches the agent's linked sources, and toasts. + const runLinkOp = async ( + knowledgeId: number, + op: (id: number, agentId: string) => Promise, + { successMsg, failMsg, onSuccess }: { + successMsg: string + failMsg: string + onSuccess?: () => void + }, + ) => { + if (mode !== 'agent' || !agentId || linkingIds.value.has(knowledgeId)) return + linkingIds.value.add(knowledgeId) + try { + await op(knowledgeId, agentId) + onSuccess?.() + await fetchSources() + toast.success(successMsg) + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : failMsg + console.error(`${failMsg}:`, err) + toast.error(failMsg, { description: msg }) + } finally { + linkingIds.value.delete(knowledgeId) + } + } + + const linkSource = (knowledgeId: number) => + runLinkOp(knowledgeId, (id, aid) => knowledgeService.linkToAgent(id, aid), { + successMsg: '知识已关联到此客服', + failMsg: '无法关联知识', + }) + + const unlinkSource = (knowledgeId: number) => + runLinkOp(knowledgeId, (id, aid) => knowledgeService.unlinkFromAgent(id, aid), { + successMsg: '知识已从此客服移除', + failMsg: '无法移除知识', + // Clear the detail pane if the removed source was selected. + onSuccess: () => { + if (selectedSourceId.value === knowledgeId) { + selectedSourceId.value = null + selectedPageId.value = null + } + }, + }) + + const linkedAgentId = mode === 'agent' ? agentId : undefined + + // Handle add/urls responses that report duplicates instead of throwing. + const throwIfError = (res: { error?: string } | undefined) => { + if (res?.error) throw new Error(res.error) + } + + // Submit a new source from the add-source modal. Resolves true on success. + // Crawl sources (website/sitemap/pdf) are queued and indexed in the + // background; text is indexed immediately. Toasts announce the outcome. + const submitSource = async (payload: AddSourcePayload): Promise => { + error.value = null + try { + if (payload.type === 'website') { + throwIfError( + await knowledgeService.addUrls( + organizationId, + [payload.url], + linkedAgentId, + undefined, + // "This page only" is a one-page cap; the others are crawl scopes. + payload.scope === 'page' ? 1 : undefined, + payload.scope === 'page' ? undefined : payload.scope, + ), + ) + toast.success('已加入抓取队列', { + description: `${payload.url}，索引完成后我们会通知你。`, + }) + } else if (payload.type === 'sitemap') { + throwIfError(await knowledgeService.addSitemap(organizationId, payload.url, linkedAgentId)) + toast.success('已加入抓取队列', { + description: `${payload.url}，正在从站点地图中发现页面。`, + }) + } else if (payload.type === 'pdf') { + await knowledgeService.uploadPdfFiles(payload.files, organizationId, linkedAgentId) + toast.success('已加入抓取队列', { + description: '文档正在解析和索引中。', + }) + } else { + await knowledgeService.addText(organizationId, payload.title, payload.content, linkedAgentId) + toast.success('页面已添加', { + description: `${payload.title} 已加入知识库。`, + }) + } + await Promise.all([fetchSources(), fetchQueue()]) + return true + } catch (err: unknown) { + // Surface add-source failures via toast only (the inline error banner is + // reserved for the read/edit pane), so the user doesn't see it twice. + const msg = err instanceof Error ? err.message : '添加来源失败' + console.error('Failed to add source:', err) + // Plan caps get the offer toast when one applies; other failures the plain one + void showPlanLimitError('无法添加来源', msg) + return false + } + } + + const refresh = async () => { + await Promise.all([fetchSources(), fetchQueue()]) + // Baseline the queue snapshot so the first poll can detect a crawl that + // finishes before it is ever observed as active. + lastQueueIds = queueIds() + crawlingNames = activeCrawlNames() + } + + const startPolling = () => { + if (pollTimer) return + pollTimer = setInterval(async () => { + await fetchQueue() + const active = activeCrawlNames() + const currentIds = queueIds() + // Reconcile when a crawl is in flight OR the queue changed since the last + // tick (an item was added or finished/removed) — the latter catches a + // crawl that completed within a single poll window. + if (active.size || setsDiffer(currentIds, lastQueueIds)) { + await fetchSources(true) + // Force-refresh the content of any expanded source that is (or just was) + // crawling, so newly-extracted pages appear without a manual re-expand. + const touched = new Set([...active, ...crawlingNames]) + for (const source of sources.value) { + if (source.expanded && touched.has(source.name)) { + await loadSourceContent(source.id, true) + } + } + } + crawlingNames = active + lastQueueIds = currentIds + }, POLL_INTERVAL_MS) + } + + const stopPolling = () => { + if (pollTimer) { + clearInterval(pollTimer) + pollTimer = null + } + } + + return { + // state + sources, + queueItems, + isLoading, + error, + query, + selectedSourceId, + selectedPageId, + editing, + isAddingPage, + draftTitle, + draftContent, + draftUrl, + isSaving, + isDeleting, + linkPickerOpen, + orgSources, + isLoadingOrgSources, + orgSourcesError, + linkingIds, + // computed + selectedSource, + selectedPage, + filteredSources, + linkedSourceIds, + // helpers for the view + sourceStatus, + pageRows, + // actions + fetchSources, + fetchQueue, + refresh, + toggleSource, + selectPage, + startEdit, + startAddPage, + cancelEdit, + savePage, + deletePage, + deleteSource, + submitSource, + openLinkPicker, + linkSource, + unlinkSource, + startPolling, + stopPolling, + } +} diff --git a/frontend/src/composables/useKnowledgeManagement.ts b/frontend/src/composables/useKnowledgeManagement.ts new file mode 100644 index 0000000..c3c6d84 --- /dev/null +++ b/frontend/src/composables/useKnowledgeManagement.ts @@ -0,0 +1,506 @@ +/* +Copyright 2024-2026 ReplyPier + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { ref } from 'vue' +import type { KnowledgeItem, KnowledgePage, QueueItem, KnowledgeContent } from '@/types/knowledge' +import { knowledgeService } from '@/services/knowledge' + +export function useKnowledgeManagement(agentId: string, organizationId: string) { + // Knowledge list state + const knowledgeItems = ref([]) + const currentPage = ref(1) + const pageSize = ref(10) + const totalPages = ref(0) + const isLoading = ref(false) + const error = ref(null) + + // Queue state + const queueItems = ref([]) + const isLoadingQueue = ref(false) + + // Content viewing/editing state + const selectedKnowledge = ref(null) + const knowledgeContent = ref(null) + const isLoadingContent = ref(false) + const isEditingContent = ref(false) + const editedContent = ref('') + const isSavingContent = ref(false) + const showContentModal = ref(false) + + // Modal and upload state + const showKnowledgeModal = ref(false) + const activeTab = ref('pdf') + const files = ref([]) + const urls = ref([]) + const newUrl = ref('') + const isUploading = ref(false) + const uploadProgress = ref(0) + const successMessage = ref('') + const fileInput = ref(null) + + // Link state + const showLinkModal = ref(false) + const orgKnowledgeItems = ref([]) + const orgCurrentPage = ref(1) + const orgTotalPages = ref(0) + // Every source the org has indexed (all pages), for plan-cap checks. + const orgTotalCount = ref(0) + const isLoadingOrg = ref(false) + + // Delete state + const showDeleteConfirm = ref(false) + const knowledgeToDelete = ref(null) + + // Add new state for URL form errors + const urlFormError = ref(null) + + // Add upload error state + const uploadError = ref(null) + + + + // Fetch knowledge data + const fetchKnowledge = async () => { + try { + isLoading.value = true + isLoadingOrg.value = true + error.value = null + + // Fetch agent knowledge, org knowledge, and queue items in parallel + const [agentResponse, orgResponse, queueResponse] = await Promise.all([ + knowledgeService.getKnowledgeByAgent(agentId, currentPage.value, pageSize.value), + knowledgeService.getKnowledgeByOrganization( + organizationId, + orgCurrentPage.value, + pageSize.value, + ), + fetchQueueItems(), + ]) + + // Default to empty: a partial response used to leave these undefined, + // and every later .some()/.length on them threw. + knowledgeItems.value = agentResponse?.knowledge || [] + totalPages.value = agentResponse?.pagination?.total_pages || 0 + + // Update org knowledge + orgKnowledgeItems.value = orgResponse?.knowledge || [] + orgTotalPages.value = orgResponse?.pagination?.total_pages || 0 + orgTotalCount.value = orgResponse?.pagination?.total_count || 0 + } catch (err) { + error.value = '加载知识库来源失败' + console.error(err) + } finally { + isLoading.value = false + isLoadingOrg.value = false + } + } + + // Fetch queue items + const fetchQueueItems = async () => { + try { + isLoadingQueue.value = true + const response = await knowledgeService.getAgentQueueItems(agentId) + queueItems.value = response.queue_items || [] + return response + } catch (err) { + console.error('Failed to load queue items:', err) + queueItems.value = [] + return { queue_items: [] } + } finally { + isLoadingQueue.value = false + } + } + + const deleteQueueItem = async (queueId: number) => { + try { + await knowledgeService.deleteQueueItem(queueId) + await fetchQueueItems() // Refresh queue + } catch (err) { + console.error('Failed to delete queue item:', err) + error.value = '删除队列项失败' + } + } + + + + // Pagination handler + const handlePageChange = (page: number) => { + currentPage.value = page + fetchKnowledge() + } + + // Date formatting + const formatDate = (dateString: string | null): string => { + if (!dateString) return 'N/A' + return new Date(dateString).toLocaleDateString('en-US', { + year: 'numeric', + month: 'short', + day: 'numeric', + }) + } + + // Get earliest creation date + const getFirstCreated = (pages: KnowledgePage[]): string | null => { + return pages.reduce( + (earliest, page) => { + if (!page.created_at) return earliest + if (!earliest) return page.created_at + return page.created_at < earliest ? page.created_at : earliest + }, + null as string | null, + ) + } + + // People type "docs.company.com", not "https://docs.company.com". Without a + // scheme new URL() throws and they get "Please enter a valid URL" for an + // address that is perfectly fine, so default the scheme before validating. + const withScheme = (url: string): string => { + const trimmed = url.trim() + if (!trimmed) return trimmed + if (/^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed)) return trimmed + // Leading slashes would produce https:////host — drop them first. + return `https://${trimmed.replace(/^\/+/, '')}` + } + + // URL validation + const isValidUrl = (url: string): boolean => { + try { + const parsed = new URL(url) + // new URL() accepts things like "mailto:x" and "foo:bar"; a crawlable + // source has to be http(s). Deliberately not requiring a dot in the host: + // self-hosted installs index intranet names like https://wiki. + return /^https?:$/.test(parsed.protocol) && !!parsed.hostname + } catch { + return false + } + } + + // File handling methods + const triggerFileInput = () => { + fileInput.value?.click() + } + + const handleFileSelect = (event: Event) => { + const input = event.target as HTMLInputElement + if (input.files) { + files.value = Array.from(input.files) + } + } + + // Resolves true when the batch was accepted (or there was nothing to send), + // so a caller like the onboarding wizard can stop instead of moving on. + const handleFileUpload = async (): Promise => { + if (!files.value.length) return true + + try { + isUploading.value = true + uploadError.value = null // Clear previous errors + + const response = await knowledgeService.uploadPdfFiles( + files.value, + organizationId, + agentId, + (progress) => { + uploadProgress.value = progress + }, + ) + if (response.message) { + successMessage.value = response.message + await fetchKnowledge() // Refresh knowledge list + } + files.value = [] + if (fileInput.value) fileInput.value.value = '' + return true + } catch (error: any) { + console.error('Upload failed:', error) + uploadError.value = error.message || '上传文件失败' + return false + } finally { + isUploading.value = false + uploadProgress.value = 0 + } + } + + // URL handling methods + const handleUrlAdd = () => { + if (!newUrl.value) return + + // Start clean so the outcome of this attempt is the only thing on screen, + // and so callers can read urlFormError to tell whether it succeeded. + urlFormError.value = null + + // Clean the URL, defaulting the scheme when the user left it off + const cleanUrl = withScheme(newUrl.value) + + // Basic URL validation + if (!isValidUrl(cleanUrl)) { + urlFormError.value = '请输入有效的 URL' + return + } + + // Check if URL already exists in knowledgeItems or orgKnowledgeItems + const urlExistsInAgent = knowledgeItems.value.some((item) => item.name === cleanUrl) + const urlExistsInOrg = orgKnowledgeItems.value.some((item) => item.name === cleanUrl) + + if (urlExistsInAgent || urlExistsInOrg) { + urlFormError.value = '该 URL 已存在于你的知识库中' + newUrl.value = '' + return + } + + // Check if URL is already in the current batch + if (urls.value.includes(cleanUrl)) { + urlFormError.value = '该 URL 已添加到当前批次' + newUrl.value = '' + return + } + + // Add URL to batch + urls.value.push(cleanUrl) + newUrl.value = '' + urlFormError.value = null + } + + const removeUrl = (index: number) => { + urls.value.splice(index, 1) + } + + // Same contract as handleFileUpload: true when the batch was accepted. + const handleUrlUpload = async (): Promise => { + if (!urls.value.length) return true + + try { + isUploading.value = true + uploadError.value = null // Clear previous errors + + const response = await knowledgeService.addUrls( + organizationId, + urls.value, + agentId, + (progress) => { + uploadProgress.value = progress + }, + ) + // A duplicate comes back as a 200 with `error` and nothing queued; keep + // the batch staged so the user sees which URL to drop. + if (response.error) { + uploadError.value = response.error + return false + } + if (response.message) { + successMessage.value = response.message + await fetchKnowledge() // Refresh knowledge list + } + urls.value = [] + return true + } catch (error: any) { + console.error('URL upload failed:', error) + uploadError.value = error.message || '上传 URL 失败' + return false + } finally { + isUploading.value = false + uploadProgress.value = 0 + } + } + + const fetchOrgKnowledge = async () => { + try { + isLoadingOrg.value = true + const response = await knowledgeService.getKnowledgeByOrganization( + organizationId, + orgCurrentPage.value, + pageSize.value, + ) + orgKnowledgeItems.value = response.knowledge + orgTotalPages.value = response.pagination.total_pages + } catch (err) { + console.error(err) + } finally { + isLoadingOrg.value = false + } + } + + const handleOrgPageChange = (page: number) => { + orgCurrentPage.value = page + fetchOrgKnowledge() + } + + const linkKnowledge = async (knowledgeId: number) => { + try { + await knowledgeService.linkToAgent(knowledgeId, agentId) + await fetchKnowledge() // Refresh agent knowledge + } catch (error) { + console.error('Error linking knowledge:', error) + } + } + + const unlinkKnowledge = async (knowledgeId: number) => { + try { + await knowledgeService.unlinkFromAgent(knowledgeId, agentId) + await fetchKnowledge() // Refresh agent knowledge + } catch (error) { + console.error('Error unlinking knowledge:', error) + } + } + + // Delete methods + const confirmDelete = (knowledgeId: number) => { + knowledgeToDelete.value = knowledgeId + showDeleteConfirm.value = true + } + + const handleDelete = async () => { + if (!knowledgeToDelete.value) return + + try { + await knowledgeService.deleteKnowledge(knowledgeToDelete.value) + await fetchKnowledge() // Refresh the list + showDeleteConfirm.value = false + knowledgeToDelete.value = null + } catch (err) { + console.error('Error deleting knowledge:', err) + error.value = '删除知识来源失败' + } + } + + const cancelDelete = () => { + showDeleteConfirm.value = false + knowledgeToDelete.value = null + } + + // Content management methods + const viewKnowledgeContent = async (knowledgeId: number) => { + try { + selectedKnowledge.value = knowledgeId + isLoadingContent.value = true + showContentModal.value = true + + const response = await knowledgeService.getKnowledgeContent(knowledgeId) + knowledgeContent.value = response + + // Do not combine chunks - keep them separate for individual editing + editedContent.value = '' + } catch (err) { + console.error('Error loading knowledge content:', err) + error.value = '加载知识内容失败' + } finally { + isLoadingContent.value = false + } + } + + const enableContentEditing = () => { + isEditingContent.value = true + } + + const cancelContentEditing = () => { + isEditingContent.value = false + } + + const saveChunkContent = async (chunkId: string, content: string) => { + if (!selectedKnowledge.value) return + + try { + isSavingContent.value = true + await knowledgeService.updateChunkContent(selectedKnowledge.value, chunkId, content) + + successMessage.value = '内容块已更新' + + // Reload the content to show updated chunk + await viewKnowledgeContent(selectedKnowledge.value) + } catch (err: any) { + console.error('Error saving chunk content:', err) + error.value = err.message || '保存内容块失败' + } finally { + isSavingContent.value = false + } + } + + const closeContentModal = () => { + showContentModal.value = false + selectedKnowledge.value = null + knowledgeContent.value = null + isEditingContent.value = false + editedContent.value = '' + } + + return { + // State + knowledgeItems, + currentPage, + pageSize, + totalPages, + isLoading, + error, + showKnowledgeModal, + activeTab, + files, + urls, + newUrl, + isUploading, + uploadProgress, + successMessage, + fileInput, + showLinkModal, + orgKnowledgeItems, + orgCurrentPage, + orgTotalPages, + orgTotalCount, + isLoadingOrg, + showDeleteConfirm, + knowledgeToDelete, + urlFormError, + uploadError, + queueItems, + isLoadingQueue, + selectedKnowledge, + knowledgeContent, + isLoadingContent, + isEditingContent, + editedContent, + isSavingContent, + showContentModal, + + // Methods + fetchKnowledge, + fetchQueueItems, + deleteQueueItem, + handlePageChange, + formatDate, + + getFirstCreated, + isValidUrl, + withScheme, + triggerFileInput, + handleFileSelect, + handleFileUpload, + handleUrlAdd, + removeUrl, + handleUrlUpload, + fetchOrgKnowledge, + handleOrgPageChange, + linkKnowledge, + unlinkKnowledge, + confirmDelete, + handleDelete, + cancelDelete, + viewKnowledgeContent, + enableContentEditing, + cancelContentEditing, + saveChunkContent, + closeContentModal, + } +} + diff --git a/frontend/src/composables/useStagedSourceCap.ts b/frontend/src/composables/useStagedSourceCap.ts new file mode 100644 index 0000000..3c18d9a --- /dev/null +++ b/frontend/src/composables/useStagedSourceCap.ts @@ -0,0 +1,69 @@ +/* +Copyright 2024-2026 ReplyPier + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import type { Ref } from 'vue' +import { subscriptionStorage } from '@/utils/storage' + +/** + * Keeps a staged batch of knowledge sources within the plan's source cap. + * + * The server rejects a whole batch that would go over the cap, so staging one + * source too many loses all of them. On a one-source plan the onboarding wizard + * pre-stages the signup domain; a second add then swaps the staged source + * rather than queuing a batch that is bound to fail. + * + * `inUseCount` is every source already counting against the cap (indexed or + * still crawling), mirroring the server's count. + * + * A cap of 0 means the plan sets none we know of (self-hosted, or limits not + * loaded yet) and the server stays the only check. + */ +export function useStagedSourceCap( + inUseCount: Ref, + urls: Ref, + files: Ref, +) { + // Read on every call: the subscription can land in storage after mount. + const cap = () => subscriptionStorage.getPlanLimits().maxKnowledgeSources + const room = () => cap() - inUseCount.value + const capMessage = (detail: string) => + `你的套餐包含 ${cap()} 个知识库来源${detail}` + + /** + * Call right after staging `added` sources onto the end of `newest`. Older + * staged sources (the other list first, then the front of `newest`) are + * dropped until the batch fits the cap. If the new sources can't fit on + * their own they are unstaged instead and the message to show is returned; + * null means the batch fits. + */ + const fitToCap = (newest: 'urls' | 'files', added = 1): string | null => { + if (cap() <= 0) return null + const latest: Ref = newest === 'urls' ? urls : files + const other: Ref = newest === 'urls' ? files : urls + if (added > room()) { + latest.value = latest.value.slice(0, -added) + return room() > 0 ? capMessage(`，最多可选 ${room()} 个。`) : capMessage('，且已在用。') + } + let overflow = latest.value.length + other.value.length - room() + const fromOther = Math.min(Math.max(overflow, 0), other.value.length) + other.value = other.value.slice(fromOther) + overflow -= fromOther + if (overflow > 0) latest.value = latest.value.slice(overflow) + return null + } + + return { fitToCap } +} diff --git a/frontend/src/utils/knowledgePages.ts b/frontend/src/utils/knowledgePages.ts new file mode 100644 index 0000000..cb7291a --- /dev/null +++ b/frontend/src/utils/knowledgePages.ts @@ -0,0 +1,112 @@ +/* +Copyright 2024-2026 ReplyPier + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import type { KnowledgeContentChunk, KnowledgeSubPage } from '@/types/knowledge' + +/** `::` — chunk ids written by app/knowledge/chunking.py. */ +const CHUNK_ID_SUFFIX = /::\d+$/ +/** `_` — chunk ids agno's PDF reader used to write. */ +const LEGACY_CHUNK_ID_SUFFIX = /_\d+$/ + +/** + * Map a chunk id back to its page id by stripping the trailing chunk suffix: + * `::` (app/knowledge/chunking.py) or the legacy `_` that + * agno's PDF reader produced. + * + * Only a numeric suffix is stripped, so real underscores in a page name (e.g. + * `getting_started`) are preserved. Mirrors the backend `PAGE_ID_EXPR` + * (app/knowledge/page_editor.py) exactly, so the id derived here is the same + * `page_id` the backend expects for edit/delete and the `subpage` value it + * returns when listing sources — keep the two in sync. + */ +export function basePageId(id: string): string { + const chunked = id.replace(CHUNK_ID_SUFFIX, '') + if (chunked !== id) return chunked + return id.replace(LEGACY_CHUNK_ID_SUFFIX, '') +} + +/** + * What goes between a chunk and the one before it when a page is rebuilt. + * `::n` chunks (app/knowledge/chunking.py) are verbatim slices of the page, so + * nothing: inserting a break would change the text that gets saved back. + * Legacy `_n` chunks were cut on whitespace that was dropped, so a paragraph + * break is the closest reconstruction. + */ +function chunkJoiner(chunkId: string): string { + return CHUNK_ID_SUFFIX.test(chunkId) ? '' : '\n\n' +} + +function countWords(text: string): number { + const trimmed = text.trim() + return trimmed ? trimmed.split(/\s+/).length : 0 +} + +function laterIso(a: string | null, b: string | null): string | null { + if (!a) return b + if (!b) return a + return a > b ? a : b +} + +/** + * Group a source's content chunks into sub-pages. Chunks are concatenated in + * their existing order (the backend returns them ordered by `created_at`), and + * page-level url/title/updated_at are taken from the chunk metadata. + */ +export function groupChunksIntoPages(chunks: KnowledgeContentChunk[]): KnowledgeSubPage[] { + const byPage = new Map() + const order: string[] = [] + + for (const chunk of chunks) { + const pageId = basePageId(chunk.id) + const meta = chunk.metadata || {} + const content = chunk.content || '' + + let page = byPage.get(pageId) + if (!page) { + page = { + page_id: pageId, + url: typeof meta.url === 'string' && meta.url ? meta.url : pageId, + title: typeof meta.title === 'string' && meta.title ? meta.title : titleFromId(pageId), + content: '', + word_count: 0, + chunk_count: 0, + updated_at: chunk.created_at ?? null, + chunk_ids: [], + } + byPage.set(pageId, page) + order.push(pageId) + } + + page.content = page.content ? `${page.content}${chunkJoiner(chunk.id)}${content}` : content + page.chunk_count += 1 + page.chunk_ids.push(chunk.id) + page.updated_at = laterIso(page.updated_at, chunk.created_at ?? null) + } + + for (const pageId of order) { + const page = byPage.get(pageId)! + page.word_count = countWords(page.content) + } + + return order.map((pageId) => byPage.get(pageId)!) +} + +/** Human-ish title fallback: last non-empty URL/path segment of the page id. */ +export function titleFromId(pageId: string): string { + const cleaned = pageId.replace(/[#?].*$/, '').replace(/\/+$/, '') + const segment = cleaned.split('/').filter(Boolean).pop() + return segment || pageId +} diff --git a/frontend/src/views/KnowledgeView.vue b/frontend/src/views/KnowledgeView.vue new file mode 100644 index 0000000..7f58202 --- /dev/null +++ b/frontend/src/views/KnowledgeView.vue @@ -0,0 +1,51 @@ + + + + + +