1629 lines
63 KiB
Python
1629 lines
63 KiB
Python
"""
|
|
Copyright 2024-2026 ChatterMate
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
"""
|
|
|
|
from fastapi import APIRouter, HTTPException, UploadFile, File, Form, Depends, Body, Query, status
|
|
from typing import List, Optional
|
|
from app.models.user import User
|
|
from app.core.auth import get_current_user, require_any_permission, require_permissions
|
|
from app.core.logger import get_logger
|
|
import io
|
|
import os
|
|
import asyncio
|
|
from pydantic import BaseModel, field_validator
|
|
from sqlalchemy import text
|
|
from uuid import UUID
|
|
from app.database import get_db
|
|
from app.models.knowledge_to_agent import KnowledgeToAgent
|
|
from app.models.agent import Agent
|
|
from app.models.knowledge import Knowledge
|
|
from app.repositories.knowledge import KnowledgeRepository
|
|
from app.knowledge import page_editor
|
|
from app.knowledge.crawl_scope import CRAWL_SCOPES
|
|
from app.knowledge.failure_advice import FAILURE_ADVICE, FailureCause, plain_reason
|
|
from app.knowledge.pdf_download import MAX_PDF_SIZE
|
|
from app.knowledge.pdf_limits import PdfRefused, check_page_count
|
|
from app.repositories.knowledge_to_agent import KnowledgeToAgentRepository
|
|
from app.repositories.agent import AgentRepository
|
|
from app.models.knowledge_queue import KnowledgeQueue, QueueStatus
|
|
from app.repositories.knowledge_queue import KnowledgeQueueRepository
|
|
from app.core.config import settings
|
|
from app.utils.domain_check import bare_host, domain_resolves
|
|
from app.services import knowledge_vector_links
|
|
from sqlalchemy.orm import Session
|
|
from app.core.s3 import upload_file_to_s3, get_s3_signed_url
|
|
from app.core.file_validation import read_validated, safe_filename, PDF_MAGIC
|
|
from app.repositories.user import UserRepository
|
|
from app.models.notification import Notification, NotificationType
|
|
from app.services.user import send_fcm_notification
|
|
from app.workers.knowledge_processor import process_queue_item, run_processor
|
|
|
|
# Try to import enterprise modules
|
|
try:
|
|
from app.enterprise.services.feature_access import require_accessible_subscription
|
|
HAS_ENTERPRISE = True
|
|
except ImportError:
|
|
HAS_ENTERPRISE = False
|
|
|
|
router = APIRouter()
|
|
|
|
# Shown in place of the database error when a source's page table can't be read.
|
|
# The real reason is logged; it must not travel to the caller.
|
|
PAGES_UNREADABLE = "Could not read this source's pages"
|
|
logger = get_logger(__name__)
|
|
|
|
# Add this near the top of the file with other constants
|
|
TEMP_DIR = "temp"
|
|
|
|
# PDF upload limits (see app/core/file_validation.py)
|
|
MAX_PDF_FILES = 20 # per request
|
|
ALLOWED_PDF_TYPES = ("application/pdf",)
|
|
|
|
# Import processor status from shared module
|
|
from app.core.processor import PROCESSOR_STATUS
|
|
from app.core.params import DbIntId, PageNumber
|
|
|
|
|
|
def resolve_plan_max_links(subscription) -> int:
|
|
"""Per-source sub-page crawl cap: the enterprise plan's ``max_sub_pages``
|
|
when enterprise is active, otherwise the self-host configurable
|
|
``KB_MAX_LINKS`` (so self-hosted installs actually control crawl scope)."""
|
|
if HAS_ENTERPRISE and subscription:
|
|
return subscription.plan.max_sub_pages
|
|
return settings.KB_MAX_LINKS
|
|
|
|
|
|
class UrlsRequest(BaseModel):
|
|
org_id: UUID
|
|
pdf_urls: List[str] = []
|
|
websites: List[str] = []
|
|
sitemaps: List[str] = []
|
|
agent_id: Optional[str] = None
|
|
# Optional cap on pages per website source (e.g. 1 = "this page only"). Always
|
|
# clamped to the plan's max_sub_pages server-side; None uses the plan limit.
|
|
max_links: Optional[int] = None
|
|
# How far link-following may wander from the seed URL: 'path' | 'host' |
|
|
# 'domain'. None uses the default (the seed's own host).
|
|
crawl_scope: Optional[str] = None
|
|
|
|
@field_validator('max_links')
|
|
@classmethod
|
|
def validate_max_links(cls, v):
|
|
if v is not None and v < 1:
|
|
raise ValueError('max_links must be at least 1')
|
|
return v
|
|
|
|
@field_validator('crawl_scope')
|
|
@classmethod
|
|
def validate_crawl_scope(cls, v):
|
|
if v is not None and v not in CRAWL_SCOPES:
|
|
raise ValueError(f"crawl_scope must be one of {', '.join(CRAWL_SCOPES)}")
|
|
return v
|
|
|
|
|
|
@field_validator('websites', 'sitemaps')
|
|
@classmethod
|
|
def validate_website_url_format(cls, v):
|
|
"""Validate that each website/sitemap URL is in https://domainname format"""
|
|
import re
|
|
validated_urls = []
|
|
|
|
for url in v:
|
|
if not url:
|
|
raise ValueError('URL cannot be empty')
|
|
|
|
# Remove trailing slashes and whitespace
|
|
url = url.strip().rstrip('/')
|
|
|
|
# Check if URL starts with https://
|
|
if not url.startswith('https://'):
|
|
raise ValueError('URL must start with https://')
|
|
|
|
# Extract domain part after https://
|
|
domain_part = url[8:] # Remove 'https://'
|
|
|
|
# Check if domain part is not empty
|
|
if not domain_part:
|
|
raise ValueError('URL must contain a domain name')
|
|
|
|
# Allow alphanumeric, dots, hyphens, and forward slashes for paths
|
|
if not re.match(r'^[a-zA-Z0-9.-]+(/.*)?$', domain_part):
|
|
raise ValueError('Invalid URL format')
|
|
|
|
# Ensure domain has at least one dot (for TLD)
|
|
domain_only = domain_part.split('/')[0] # Get just the domain part before any path
|
|
if '.' not in domain_only:
|
|
raise ValueError('URL must contain a valid domain with TLD')
|
|
|
|
validated_urls.append(url)
|
|
|
|
return validated_urls
|
|
|
|
@field_validator('pdf_urls')
|
|
@classmethod
|
|
def validate_pdf_url_format(cls, v):
|
|
"""Validate that each PDF URL is in https://domainname format"""
|
|
import re
|
|
validated_urls = []
|
|
|
|
for url in v:
|
|
if not url:
|
|
raise ValueError('URL cannot be empty')
|
|
|
|
# Remove trailing slashes and whitespace
|
|
url = url.strip().rstrip('/')
|
|
|
|
# Check if URL starts with https://
|
|
if not url.startswith('https://'):
|
|
raise ValueError('URL must start with https://')
|
|
|
|
# Extract domain part after https://
|
|
domain_part = url[8:] # Remove 'https://'
|
|
|
|
# Check if domain part is not empty
|
|
if not domain_part:
|
|
raise ValueError('URL must contain a domain name')
|
|
|
|
# Allow alphanumeric, dots, hyphens, and forward slashes for paths
|
|
if not re.match(r'^[a-zA-Z0-9.-]+(/.*)?$', domain_part):
|
|
raise ValueError('Invalid URL format')
|
|
|
|
# Ensure domain has at least one dot (for TLD)
|
|
domain_only = domain_part.split('/')[0] # Get just the domain part before any path
|
|
if '.' not in domain_only:
|
|
raise ValueError('URL must contain a valid domain with TLD')
|
|
|
|
validated_urls.append(url)
|
|
|
|
return validated_urls
|
|
|
|
|
|
@router.post("/upload/pdf")
|
|
async def upload_pdf_files(
|
|
files: List[UploadFile] = File(...),
|
|
org_id: str = Form(...),
|
|
agent_id: Optional[str] = Form(None),
|
|
current_user: User = Depends(require_permissions("manage_knowledge")),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Upload PDF files to knowledge base"""
|
|
try:
|
|
# Convert org_id string to UUID for comparison
|
|
try:
|
|
org_uuid = UUID(org_id)
|
|
except ValueError:
|
|
raise HTTPException(status_code=400, detail="Invalid org_id")
|
|
if current_user.organization_id != org_uuid:
|
|
raise HTTPException(status_code=403, detail="Unauthorized access to organization")
|
|
|
|
if len(files) > MAX_PDF_FILES:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail=f"Too many files: at most {MAX_PDF_FILES} per request",
|
|
)
|
|
|
|
# Validate the agent (format + ownership) once, up front.
|
|
agent_uuid = None
|
|
if agent_id:
|
|
try:
|
|
agent_uuid = UUID(agent_id)
|
|
except ValueError:
|
|
raise HTTPException(status_code=400, detail="Invalid agent_id")
|
|
agent = AgentRepository(db).get_agent(agent_uuid)
|
|
if not agent or agent.organization_id != org_uuid:
|
|
# 404 (not 403) so we don't reveal another org's agent existence.
|
|
raise HTTPException(status_code=404, detail="Agent not found")
|
|
|
|
# Bound for the non-enterprise path (resolve_plan_max_links below).
|
|
subscription = None
|
|
# Check enterprise subscription limits if enterprise module is available
|
|
if HAS_ENTERPRISE:
|
|
knowledge_repo = KnowledgeRepository(db)
|
|
|
|
# Accessible = active/trial/past-due-in-period OR cancelled-but-
|
|
# still-in-paid-period; raises 403 when the org has no plan.
|
|
subscription = require_accessible_subscription(db, org_uuid)
|
|
|
|
# Sources already spoken for: indexed, plus crawls still queued
|
|
current_count = knowledge_repo.count_sources_in_use(org_uuid)
|
|
|
|
# Check if adding these files would exceed the limit
|
|
new_count = current_count + len(files)
|
|
if subscription.plan.max_knowledge_sources is not None and new_count > subscription.plan.max_knowledge_sources:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_403_FORBIDDEN,
|
|
detail=f"Cannot add files: Maximum number of knowledge sources ({subscription.plan.max_knowledge_sources}) would be exceeded"
|
|
)
|
|
|
|
queue_repo = KnowledgeQueueRepository(db)
|
|
queued_items = []
|
|
|
|
# Process each file
|
|
for file in files:
|
|
# Validate content-type, size, and magic bytes; returns the bytes.
|
|
content = await read_validated(
|
|
file,
|
|
max_size=MAX_PDF_SIZE,
|
|
allowed_content_types=ALLOWED_PDF_TYPES,
|
|
magic_prefix=PDF_MAGIC,
|
|
label="PDF",
|
|
)
|
|
try:
|
|
# pypdf on an untrusted upload: off the event loop
|
|
await asyncio.to_thread(check_page_count, io.BytesIO(content))
|
|
except PdfRefused as e:
|
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"{file.filename}: {e}")
|
|
# Sanitize the filename (strip path components, add a UUID prefix)
|
|
# to prevent path traversal and overwrites.
|
|
stored_name = safe_filename(file.filename)
|
|
source_type = "pdf_file"
|
|
|
|
# Check if S3 storage is enabled
|
|
if settings.S3_FILE_STORAGE:
|
|
folder = f"knowledge/{org_uuid}"
|
|
file_url = await upload_file_to_s3(content, folder, stored_name, content_type="application/pdf")
|
|
logger.debug(f"Uploaded PDF to S3: {file_url}")
|
|
file_path = await get_s3_signed_url(file_url)
|
|
source_type = "pdf_url"
|
|
else:
|
|
# Save file locally
|
|
os.makedirs(TEMP_DIR, exist_ok=True)
|
|
file_path = os.path.join(TEMP_DIR, stored_name)
|
|
with open(file_path, "wb") as f:
|
|
f.write(content)
|
|
|
|
# Create queue item with user_id
|
|
queue_item = KnowledgeQueue(
|
|
organization_id=org_uuid,
|
|
agent_id=agent_uuid,
|
|
user_id=current_user.id,
|
|
source_type=source_type,
|
|
source=file_path,
|
|
status=QueueStatus.PENDING,
|
|
queue_metadata={
|
|
"max_links": resolve_plan_max_links(subscription)
|
|
}
|
|
)
|
|
queued_items.append(queue_repo.create(queue_item))
|
|
|
|
return {
|
|
"message": "PDFs queued for processing,it will take a while to process, we will notify you when it is done",
|
|
"queue_items": [{"id": item.id, "status": item.status} for item in queued_items]
|
|
}
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Error uploading PDFs: {str(e)}")
|
|
raise HTTPException(status_code=500, detail="Failed to upload files")
|
|
|
|
|
|
def _get_crawled_urls_info(crawled_urls):
|
|
"""Get comprehensive information about crawled URLs"""
|
|
if not crawled_urls or not isinstance(crawled_urls, list):
|
|
return {
|
|
"latest_url": None,
|
|
"all_urls": [],
|
|
"count": 0
|
|
}
|
|
|
|
all_urls = []
|
|
latest_url = None
|
|
|
|
try:
|
|
for item in crawled_urls:
|
|
if isinstance(item, str):
|
|
# New format: just URL strings
|
|
all_urls.append(item)
|
|
latest_url = item # Keep updating to get the last one
|
|
elif isinstance(item, dict) and "url" in item:
|
|
# Legacy format: objects with url, timestamp, status
|
|
all_urls.append(item["url"])
|
|
latest_url = item["url"]
|
|
except (KeyError, TypeError) as e:
|
|
logger.warning(f"Error processing crawled URLs: {str(e)}")
|
|
|
|
return {
|
|
"latest_url": latest_url,
|
|
"all_urls": all_urls,
|
|
"count": len(all_urls)
|
|
}
|
|
|
|
|
|
@router.get("/explore/progress/{queue_id}")
|
|
async def get_explore_progress(
|
|
queue_id: DbIntId,
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Get progress status for knowledge base processing"""
|
|
try:
|
|
# Deliberately unauthenticated — it polls the job the equally public
|
|
# /explore/add-url just created. That makes scoping it to the shared
|
|
# explore org essential: queue ids are sequential, so an unscoped
|
|
# lookup let anyone walk every tenant's ingestion jobs and read their
|
|
# source URLs and crawled-page lists.
|
|
queue_item = db.query(KnowledgeQueue).filter(
|
|
KnowledgeQueue.id == queue_id,
|
|
KnowledgeQueue.organization_id == UUID(settings.EXPLORE_SOURCE_ORG_ID)
|
|
).first()
|
|
|
|
# Refresh the item to get the latest data from the database
|
|
if queue_item:
|
|
db.refresh(queue_item)
|
|
|
|
if not queue_item:
|
|
raise HTTPException(status_code=404, detail="Queue item not found")
|
|
|
|
# Get processing stage information
|
|
stage_info = {
|
|
"not_started": {"label": "Initializing", "step": 1, "total": 4},
|
|
"NOT_STARTED": {"label": "Initializing", "step": 1, "total": 4},
|
|
"crawling": {"label": "Crawling Website", "step": 2, "total": 4},
|
|
"CRAWLING": {"label": "Crawling Website", "step": 2, "total": 4},
|
|
"embedding": {"label": "Processing Content", "step": 3, "total": 4},
|
|
"EMBEDDING": {"label": "Processing Content", "step": 3, "total": 4},
|
|
"completed": {"label": "Completed", "step": 4, "total": 4},
|
|
"COMPLETED": {"label": "Completed", "step": 4, "total": 4}
|
|
}
|
|
|
|
# Safely get enum values
|
|
def get_enum_value(enum_field):
|
|
if enum_field is None:
|
|
return None
|
|
return enum_field.value if hasattr(enum_field, 'value') else str(enum_field)
|
|
|
|
# Get processing stage as string
|
|
processing_stage_str = get_enum_value(queue_item.processing_stage) or "NOT_STARTED"
|
|
status_str = get_enum_value(queue_item.status) or "PENDING"
|
|
|
|
# Override stage to "COMPLETED" if status is COMPLETED
|
|
if status_str.upper() == "COMPLETED":
|
|
processing_stage_str = "COMPLETED"
|
|
|
|
logger.debug(f"Queue {queue_id}: status='{status_str}', stage='{processing_stage_str}', progress={queue_item.progress_percentage}, is_complete={status_str.upper() in ['COMPLETED', 'FAILED']}")
|
|
logger.debug(f"Queue {queue_id}: crawled_urls count={len(queue_item.crawled_urls) if queue_item.crawled_urls else 0}")
|
|
logger.debug(f"Queue {queue_id}: crawled_urls raw value: {queue_item.crawled_urls}")
|
|
|
|
current_stage = stage_info.get(processing_stage_str,
|
|
{"label": "Processing", "step": 1, "total": 4})
|
|
|
|
# Calculate overall progress based on stage and percentage
|
|
if status_str.upper() == "COMPLETED":
|
|
overall_progress = 100.0
|
|
else:
|
|
stage_weight = (current_stage["step"] - 1) / current_stage["total"] * 100
|
|
stage_progress = (queue_item.progress_percentage or 0) / current_stage["total"]
|
|
overall_progress = min(100, stage_weight + stage_progress)
|
|
|
|
# Get crawled URLs information
|
|
crawled_urls_info = _get_crawled_urls_info(queue_item.crawled_urls)
|
|
|
|
return {
|
|
"queue_id": queue_item.id,
|
|
"status": status_str,
|
|
"processing_stage": processing_stage_str,
|
|
"progress_percentage": queue_item.progress_percentage or 0,
|
|
"overall_progress": round(overall_progress, 1),
|
|
"current_stage": current_stage,
|
|
"source": queue_item.source,
|
|
"total_items": queue_item.total_items or 0,
|
|
"processed_items": queue_item.processed_items or 0,
|
|
"created_at": queue_item.created_at.isoformat() if queue_item.created_at else None,
|
|
"updated_at": queue_item.updated_at.isoformat() if queue_item.updated_at else None,
|
|
"crawled_url": crawled_urls_info["latest_url"],
|
|
"crawled_urls": crawled_urls_info["all_urls"],
|
|
"crawled_count": crawled_urls_info["count"],
|
|
"is_complete": status_str.upper() in ["COMPLETED", "FAILED"],
|
|
"error_message": getattr(queue_item, 'error_message', None)
|
|
}
|
|
|
|
except HTTPException:
|
|
# Otherwise the not-found above is reported as a 500 echoing its detail.
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Error getting progress for queue {queue_id}: {str(e)}")
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
|
|
async def _hosts_that_do_not_exist(urls: List[str]) -> set:
|
|
"""Hosts DNS says have no such name. Lookups that time out or can't tell
|
|
are left to the crawler."""
|
|
hosts = sorted({bare_host(url) for url in urls} - {""})
|
|
answers = await asyncio.gather(*(domain_resolves(host) for host in hosts))
|
|
return {host for host, answer in zip(hosts, answers) if answer is False}
|
|
|
|
|
|
@router.post("/add/urls")
|
|
async def add_urls(
|
|
request: UrlsRequest = Body(...),
|
|
current_user: User = Depends(require_permissions("manage_knowledge")),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Add URLs to knowledge base"""
|
|
try:
|
|
# Verify organization access
|
|
if current_user.organization_id != request.org_id:
|
|
raise HTTPException(status_code=403, detail="Unauthorized access to organization")
|
|
|
|
# Validate the agent (format + ownership) once, up front.
|
|
agent_uuid = None
|
|
if request.agent_id:
|
|
try:
|
|
agent_uuid = UUID(request.agent_id)
|
|
except (ValueError, TypeError):
|
|
raise HTTPException(status_code=400, detail="Invalid agent_id")
|
|
agent = AgentRepository(db).get_agent(agent_uuid)
|
|
if not agent or agent.organization_id != request.org_id:
|
|
# 404 (not 403) so we don't reveal another org's agent existence.
|
|
raise HTTPException(status_code=404, detail="Agent not found")
|
|
|
|
# Bound for the non-enterprise path (resolve_plan_max_links below).
|
|
subscription = None
|
|
# Check enterprise subscription limits if enterprise module is available
|
|
if HAS_ENTERPRISE:
|
|
knowledge_repo = KnowledgeRepository(db)
|
|
|
|
# Accessible = active/trial/past-due-in-period OR cancelled-but-
|
|
# still-in-paid-period; raises 403 when the org has no plan.
|
|
subscription = require_accessible_subscription(db, request.org_id)
|
|
|
|
# Sources already spoken for: indexed, plus crawls still queued
|
|
current_count = knowledge_repo.count_sources_in_use(request.org_id)
|
|
|
|
# Calculate total new URLs to be added (each sitemap is one source)
|
|
total_new_urls = len(request.pdf_urls) + len(request.websites) + len(request.sitemaps)
|
|
|
|
# Check if adding these URLs would exceed the limit
|
|
new_count = current_count + total_new_urls
|
|
if subscription.plan.max_knowledge_sources is not None and new_count > subscription.plan.max_knowledge_sources:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_403_FORBIDDEN,
|
|
detail=f"Cannot add URLs: Maximum number of knowledge sources ({subscription.plan.max_knowledge_sources}) would be exceeded"
|
|
)
|
|
|
|
queue_repo = KnowledgeQueueRepository(db)
|
|
knowledge_repo = KnowledgeRepository(db)
|
|
queued_items = []
|
|
|
|
# Per-source sub-page cap: the plan limit, optionally narrowed by the
|
|
# request (e.g. "this page only" -> max_links=1), never allowed to
|
|
# exceed the plan limit.
|
|
plan_sub_pages = resolve_plan_max_links(subscription)
|
|
website_max_links = (
|
|
min(request.max_links, plan_sub_pages) if request.max_links else plan_sub_pages
|
|
)
|
|
|
|
# Check for duplicate URLs
|
|
all_urls = request.pdf_urls + request.websites + request.sitemaps
|
|
existing_sources = knowledge_repo.get_by_sources(request.org_id, all_urls)
|
|
|
|
if existing_sources:
|
|
duplicate_urls = [source.source for source in existing_sources]
|
|
return {
|
|
"error": "Some URLs already exist in your knowledge base",
|
|
"duplicate_urls": duplicate_urls
|
|
}
|
|
|
|
# A domain that doesn't exist fails now, not after its turn in the
|
|
# processor (up to a minute or more): onboarding is waiting to hear.
|
|
missing_hosts = await _hosts_that_do_not_exist(all_urls)
|
|
|
|
def queue(source_type: str, url: str, metadata: dict) -> None:
|
|
dead = bare_host(url) in missing_hosts
|
|
queued_items.append(queue_repo.create(KnowledgeQueue(
|
|
organization_id=request.org_id,
|
|
agent_id=agent_uuid,
|
|
user_id=current_user.id,
|
|
source_type=source_type,
|
|
source=url,
|
|
status=QueueStatus.FAILED if dead else QueueStatus.PENDING,
|
|
error=FAILURE_ADVICE[FailureCause.DNS].format(url=url) if dead else None,
|
|
queue_metadata=metadata,
|
|
)))
|
|
|
|
for url in request.pdf_urls:
|
|
queue('pdf_url', url, {"max_links": plan_sub_pages})
|
|
for url in request.websites:
|
|
queue('website', url, {
|
|
"max_links": website_max_links,
|
|
# Omitted (None) means the crawler's default: the seed's host.
|
|
"crawl_scope": request.crawl_scope,
|
|
})
|
|
# Sitemaps — one source each; pages are discovered from the sitemap
|
|
# and capped at the plan's sub-page limit (no per-request crawl scope).
|
|
for url in request.sitemaps:
|
|
queue('sitemap', url, {"max_links": plan_sub_pages})
|
|
|
|
return {
|
|
"message": "URLs queued for processing, it will take a while to process, we will notify you when it is done",
|
|
"queue_items": [{"id": item.id, "status": item.status} for item in queued_items]
|
|
}
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Error adding URLs: {str(e)}")
|
|
raise HTTPException(status_code=500, detail="Failed to add URLs")
|
|
|
|
|
|
class TextSourceRequest(BaseModel):
|
|
org_id: UUID
|
|
title: str
|
|
content: str
|
|
agent_id: Optional[str] = None
|
|
|
|
@field_validator('title')
|
|
@classmethod
|
|
def validate_title(cls, v):
|
|
if not v or not v.strip():
|
|
raise ValueError('Title cannot be empty')
|
|
return v.strip()
|
|
|
|
@field_validator('content')
|
|
@classmethod
|
|
def validate_content(cls, v):
|
|
if not v or not v.strip():
|
|
raise ValueError('Content cannot be empty')
|
|
return v
|
|
|
|
|
|
@router.post("/add/text")
|
|
async def add_text_source(
|
|
request: TextSourceRequest = Body(...),
|
|
current_user: User = Depends(require_permissions("manage_knowledge")),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Create a knowledge source from pasted text and index it immediately."""
|
|
try:
|
|
if current_user.organization_id != request.org_id:
|
|
raise HTTPException(status_code=403, detail="Unauthorized access to organization")
|
|
|
|
# Validate the agent (format + ownership) up front.
|
|
agent_uuid = None
|
|
if request.agent_id:
|
|
try:
|
|
agent_uuid = UUID(request.agent_id)
|
|
except (ValueError, TypeError):
|
|
raise HTTPException(status_code=400, detail="Invalid agent_id")
|
|
agent = AgentRepository(db).get_agent(agent_uuid)
|
|
if not agent or agent.organization_id != request.org_id:
|
|
raise HTTPException(status_code=404, detail="Agent not found")
|
|
|
|
knowledge_repo = KnowledgeRepository(db)
|
|
|
|
# Enforce the source-count limit (enterprise only).
|
|
if HAS_ENTERPRISE:
|
|
subscription = require_accessible_subscription(db, request.org_id)
|
|
current_count = knowledge_repo.count_sources_in_use(request.org_id)
|
|
max_sources = subscription.plan.max_knowledge_sources
|
|
if max_sources is not None and current_count + 1 > max_sources:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_403_FORBIDDEN,
|
|
detail=f"Cannot add source: Maximum number of knowledge sources ({max_sources}) would be exceeded"
|
|
)
|
|
|
|
# Reject a duplicate source title.
|
|
if knowledge_repo.get_by_sources(request.org_id, [request.title]):
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail="A knowledge source with this title already exists"
|
|
)
|
|
|
|
knowledge = page_editor.create_text_source(
|
|
db, request.org_id, request.title, request.content, agent_uuid
|
|
)
|
|
return {
|
|
"message": "Text source added",
|
|
"knowledge": {"id": knowledge.id, "name": knowledge.source, "type": knowledge.source_type.value},
|
|
}
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
db.rollback()
|
|
logger.error(f"Error adding text source: {str(e)}")
|
|
raise HTTPException(status_code=500, detail="Failed to add text source")
|
|
|
|
|
|
@router.post("/link")
|
|
async def link_knowledge_to_agent(
|
|
knowledge_id: int,
|
|
agent_id: str,
|
|
current_user: User = Depends(get_current_user),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Link existing knowledge to an agent"""
|
|
try:
|
|
knowledge_repo = KnowledgeRepository(db)
|
|
link_repo = KnowledgeToAgentRepository(db)
|
|
|
|
# Verify knowledge exists and belongs to user's org
|
|
knowledge = knowledge_repo.get_by_id(knowledge_id)
|
|
if not knowledge or knowledge.organization_id != current_user.organization_id:
|
|
raise HTTPException(status_code=404, detail="Knowledge source not found or unauthorized access")
|
|
|
|
# Convert agent_id to UUID
|
|
try:
|
|
agent_uuid = UUID(agent_id)
|
|
except ValueError:
|
|
raise HTTPException(status_code=400, detail="Invalid agent ID format")
|
|
|
|
# The agent must belong to the caller's org, not just the knowledge.
|
|
# 404 (not 403) so we don't reveal another org's agent existence.
|
|
agent = AgentRepository(db).get_agent(agent_uuid)
|
|
if not agent or agent.organization_id != current_user.organization_id:
|
|
raise HTTPException(status_code=404, detail="Agent not found")
|
|
|
|
# Check if link already exists
|
|
existing_link = link_repo.get_by_ids(knowledge_id, agent_uuid)
|
|
if existing_link:
|
|
raise HTTPException(status_code=400, detail="Knowledge is already linked to this agent")
|
|
|
|
# The link row and the vector-store filters must land together: retrieval
|
|
# matches on filters->'agent_id', so a committed link whose filter update
|
|
# failed shows as linked in the dashboard while the agent can retrieve
|
|
# nothing. commit=False defers to the single commit below.
|
|
link_repo.create(
|
|
KnowledgeToAgent(knowledge_id=knowledge_id, agent_id=agent_uuid),
|
|
commit=False,
|
|
)
|
|
knowledge_vector_links.add_agent(db, knowledge, agent_uuid)
|
|
db.commit()
|
|
|
|
return {"message": "Knowledge linked to agent successfully"}
|
|
|
|
except HTTPException:
|
|
db.rollback()
|
|
raise
|
|
except Exception as e:
|
|
db.rollback()
|
|
logger.error(f"Error linking knowledge: {str(e)}")
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
|
|
@router.delete("/unlink")
|
|
async def unlink_knowledge_from_agent(
|
|
knowledge_id: int,
|
|
agent_id: str,
|
|
current_user: User = Depends(get_current_user),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Unlink knowledge from an agent"""
|
|
try:
|
|
# Convert agent_id to UUID
|
|
try:
|
|
agent_uuid = UUID(agent_id)
|
|
except ValueError:
|
|
raise HTTPException(status_code=400, detail="Invalid agent ID format")
|
|
|
|
knowledge_repo = KnowledgeRepository(db)
|
|
link_repo = KnowledgeToAgentRepository(db)
|
|
|
|
# Verify knowledge exists and belongs to user's org
|
|
knowledge = knowledge_repo.get_by_id(knowledge_id)
|
|
if not knowledge or knowledge.organization_id != current_user.organization_id:
|
|
raise HTTPException(status_code=404, detail="Knowledge source not found or unauthorized access")
|
|
|
|
# And the agent must be in the caller's org too.
|
|
agent = AgentRepository(db).get_agent(agent_uuid)
|
|
if not agent or agent.organization_id != current_user.organization_id:
|
|
raise HTTPException(status_code=404, detail="Agent not found")
|
|
|
|
# Same transaction as the filter update below — an unlink that commits
|
|
# the row deletion but leaves the agent_id in the vector filters keeps
|
|
# the source retrievable by an agent the dashboard shows as unlinked.
|
|
success = link_repo.delete_by_ids(knowledge_id, agent_uuid, commit=False)
|
|
if not success:
|
|
raise HTTPException(status_code=404, detail="Link not found")
|
|
|
|
knowledge_vector_links.remove_agent(db, knowledge, agent_uuid)
|
|
db.commit()
|
|
|
|
return {"message": "Knowledge unlinked from agent successfully"}
|
|
|
|
except HTTPException:
|
|
db.rollback()
|
|
raise
|
|
except Exception as e:
|
|
db.rollback()
|
|
logger.error(f"Error unlinking knowledge: {str(e)}")
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
|
|
def _linked_agents_map(db: Session, knowledge_items) -> dict:
|
|
"""Map knowledge_id -> [{id, name}] of the agents each source is linked to.
|
|
|
|
Batched into a single query over the page's sources to avoid N+1 lookups.
|
|
"""
|
|
ids = [k.id for k in knowledge_items]
|
|
if not ids:
|
|
return {}
|
|
rows = (
|
|
db.query(KnowledgeToAgent.knowledge_id, Agent.id, Agent.name, Agent.display_name)
|
|
.join(Agent, Agent.id == KnowledgeToAgent.agent_id)
|
|
.filter(KnowledgeToAgent.knowledge_id.in_(ids))
|
|
.all()
|
|
)
|
|
result: dict = {}
|
|
for knowledge_id, agent_id, name, display_name in rows:
|
|
result.setdefault(knowledge_id, []).append(
|
|
{"id": str(agent_id), "name": display_name or name}
|
|
)
|
|
return result
|
|
|
|
|
|
@router.get("/agent/{agent_id}")
|
|
async def get_knowledge_by_agent(
|
|
agent_id: str,
|
|
page: PageNumber = 1,
|
|
page_size: int = Query(default=10, ge=1, le=100),
|
|
current_user: User = Depends(
|
|
require_any_permission("view_knowledge", "manage_knowledge")),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Get knowledge sources and their data for an agent with pagination"""
|
|
try:
|
|
# Convert agent_id to UUID
|
|
try:
|
|
agent_uuid = UUID(agent_id)
|
|
except ValueError:
|
|
raise HTTPException(status_code=400, detail="Invalid agent ID format")
|
|
|
|
# The agent must belong to the caller's organization. count_by_agent and
|
|
# get_by_agent below filter on agent_uuid alone, so without this an id
|
|
# from another tenant returns that tenant's knowledge inventory — the
|
|
# link/unlink routes have always checked this; this read never did.
|
|
agent = db.query(Agent).filter(Agent.id == agent_uuid).first()
|
|
if not agent or agent.organization_id != current_user.organization_id:
|
|
raise HTTPException(status_code=404, detail="Agent not found")
|
|
|
|
logger.debug(f"Getting knowledge for agent {agent_uuid}")
|
|
knowledge_repo = KnowledgeRepository(db)
|
|
|
|
# Get total count and paginated knowledge items
|
|
total_count = knowledge_repo.count_by_agent(agent_uuid)
|
|
logger.debug(f"Total count for agent {agent_uuid}: {total_count}")
|
|
|
|
knowledge_items = knowledge_repo.get_by_agent(
|
|
agent_uuid,
|
|
skip=(page - 1) * page_size,
|
|
limit=page_size
|
|
)
|
|
|
|
|
|
agents_map = _linked_agents_map(db, knowledge_items)
|
|
result = []
|
|
for k in knowledge_items:
|
|
# Base knowledge data
|
|
knowledge_data = {
|
|
"id": k.id,
|
|
"name": k.source,
|
|
"type": k.source_type.value,
|
|
"agents": agents_map.get(k.id, []),
|
|
"pages": []
|
|
}
|
|
|
|
# Query the actual data if table_name is specified
|
|
if k.table_name and k.schema:
|
|
try:
|
|
# Create a safe query to get unique records with cleaned source
|
|
query = text(f"""
|
|
SELECT DISTINCT
|
|
{page_editor.PAGE_ID_EXPR} as subpage,
|
|
id,
|
|
created_at,
|
|
updated_at
|
|
FROM {k.schema}."{k.table_name}"
|
|
WHERE name = :source
|
|
""")
|
|
|
|
# Execute query with parameters
|
|
rows = db.execute(query, {"source": k.source})
|
|
|
|
# Group pages by subpage
|
|
pages_dict = {}
|
|
for row in rows:
|
|
subpage = row.subpage
|
|
if subpage not in pages_dict:
|
|
pages_dict[subpage] = {
|
|
"subpage": subpage,
|
|
"created_at": row.created_at.isoformat() if row.created_at else None,
|
|
"updated_at": row.updated_at.isoformat() if row.updated_at else None
|
|
}
|
|
|
|
knowledge_data["pages"] = list(pages_dict.values())
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error querying table {k.table_name}: {str(e)}")
|
|
knowledge_data["error"] = PAGES_UNREADABLE
|
|
|
|
result.append(knowledge_data)
|
|
|
|
|
|
return {
|
|
"knowledge": result,
|
|
"pagination": {
|
|
"total": total_count,
|
|
"page": page,
|
|
"page_size": page_size,
|
|
"total_pages": (total_count + page_size - 1) // page_size
|
|
}
|
|
}
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Error getting knowledge by agent: {str(e)}")
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
|
|
def _enum_value(enum_field):
|
|
"""Safely unwrap an enum field to its string value (or None)."""
|
|
if enum_field is None:
|
|
return None
|
|
return enum_field.value if hasattr(enum_field, 'value') else str(enum_field)
|
|
|
|
|
|
def _serialize_queue_item(item: KnowledgeQueue) -> dict:
|
|
"""Shape a KnowledgeQueue row for the queue-list API responses."""
|
|
return {
|
|
"id": item.id,
|
|
"source": item.source,
|
|
"source_type": item.source_type,
|
|
"status": _enum_value(item.status),
|
|
"error": item.error,
|
|
# One plain sentence for people; `error` stays for the knowledge page
|
|
"reason": plain_reason(item.error, item.source) if _enum_value(item.status) == QueueStatus.FAILED.value else None,
|
|
"created_at": item.created_at.isoformat() if item.created_at else None,
|
|
"updated_at": item.updated_at.isoformat() if item.updated_at else None,
|
|
"processing_stage": _enum_value(item.processing_stage),
|
|
"progress_percentage": item.progress_percentage or 0,
|
|
}
|
|
|
|
|
|
@router.get("/queue/agent/{agent_id}")
|
|
async def get_agent_queue_items(
|
|
agent_id: str,
|
|
current_user: User = Depends(get_current_user),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Get all queue items (pending, processing, failed) for an agent"""
|
|
try:
|
|
# Convert agent_id to UUID
|
|
try:
|
|
agent_uuid = UUID(agent_id)
|
|
except ValueError:
|
|
raise HTTPException(status_code=400, detail="Invalid agent ID format")
|
|
|
|
logger.debug(f"Getting queue items for agent {agent_uuid}")
|
|
|
|
# Get queue items for this agent (excluding completed ones)
|
|
queue_items = db.query(KnowledgeQueue).filter(
|
|
KnowledgeQueue.agent_id == agent_uuid,
|
|
KnowledgeQueue.organization_id == current_user.organization_id,
|
|
KnowledgeQueue.status.in_([QueueStatus.PENDING, QueueStatus.PROCESSING, QueueStatus.FAILED])
|
|
).order_by(KnowledgeQueue.created_at.desc()).all()
|
|
|
|
return {"queue_items": [_serialize_queue_item(item) for item in queue_items]}
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Error getting queue items for agent: {str(e)}")
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
|
|
@router.get("/queue/organization/{org_id}")
|
|
async def get_organization_queue_items(
|
|
org_id: UUID,
|
|
current_user: User = Depends(
|
|
require_any_permission("view_knowledge", "manage_knowledge")),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Get all in-flight queue items (pending, processing, failed) for an org."""
|
|
try:
|
|
if current_user.organization_id != org_id:
|
|
raise HTTPException(status_code=403, detail="Unauthorized access to organization")
|
|
|
|
queue_items = db.query(KnowledgeQueue).filter(
|
|
KnowledgeQueue.organization_id == org_id,
|
|
KnowledgeQueue.status.in_([QueueStatus.PENDING, QueueStatus.PROCESSING, QueueStatus.FAILED])
|
|
).order_by(KnowledgeQueue.created_at.desc()).all()
|
|
|
|
return {"queue_items": [_serialize_queue_item(item) for item in queue_items]}
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Error getting queue items for org: {str(e)}")
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
|
|
@router.delete("/queue/{queue_id}")
|
|
async def delete_queue_item(
|
|
queue_id: DbIntId,
|
|
current_user: User = Depends(require_permissions("manage_knowledge")),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Delete a queue item (only if failed or pending)"""
|
|
try:
|
|
queue_repo = KnowledgeQueueRepository(db)
|
|
item = queue_repo.get_by_id(queue_id)
|
|
|
|
if not item:
|
|
raise HTTPException(status_code=404, detail="Queue item not found")
|
|
|
|
if item.organization_id != current_user.organization_id:
|
|
raise HTTPException(status_code=403, detail="Unauthorized")
|
|
|
|
# Only allow deleting failed or pending items
|
|
# We might want to allow deleting processing items if they are stuck, but let's be safe for now
|
|
if item.status not in [QueueStatus.FAILED, QueueStatus.PENDING]:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail="Only failed or pending items can be removed from queue"
|
|
)
|
|
|
|
db.delete(item)
|
|
db.commit()
|
|
|
|
return {"message": "Queue item deleted successfully"}
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Error deleting queue item: {str(e)}")
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
|
|
@router.get("/organization/{org_id}")
|
|
|
|
async def get_knowledge_by_organization(
|
|
org_id: str,
|
|
page: PageNumber = 1,
|
|
page_size: int = Query(default=10, ge=1, le=100),
|
|
current_user: User = Depends(
|
|
require_any_permission("view_knowledge", "manage_knowledge")),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Get knowledge sources and their data for an organization with pagination"""
|
|
try:
|
|
# Convert org_id to UUID
|
|
try:
|
|
org_uuid = UUID(org_id)
|
|
except ValueError:
|
|
raise HTTPException(status_code=400, detail="Invalid organization ID format")
|
|
|
|
logger.debug(f"Getting knowledge for organization {org_uuid}")
|
|
|
|
# Verify organization access
|
|
if current_user.organization_id != org_uuid:
|
|
raise HTTPException(status_code=403, detail="Unauthorized access to organization")
|
|
|
|
knowledge_repo = KnowledgeRepository(db)
|
|
|
|
# Get total count and paginated knowledge items
|
|
total_count = knowledge_repo.count_by_organization(org_uuid)
|
|
logger.debug(f"Total count for organization {org_uuid}: {total_count}")
|
|
|
|
knowledge_items = knowledge_repo.get_by_organization(
|
|
org_uuid,
|
|
skip=(page - 1) * page_size,
|
|
limit=page_size
|
|
)
|
|
logger.debug(f"Knowledge items for organization {org_uuid}: {knowledge_items}")
|
|
|
|
agents_map = _linked_agents_map(db, knowledge_items)
|
|
result = []
|
|
for k in knowledge_items:
|
|
# Base knowledge data
|
|
knowledge_data = {
|
|
"id": k.id,
|
|
"name": k.source,
|
|
"type": k.source_type.value,
|
|
"agents": agents_map.get(k.id, []),
|
|
"pages": []
|
|
}
|
|
|
|
# Query the actual data if table_name is specified
|
|
if k.table_name and k.schema:
|
|
try:
|
|
# Create a safe query to get unique records with cleaned source
|
|
query = text(f"""
|
|
SELECT DISTINCT
|
|
{page_editor.PAGE_ID_EXPR} as subpage,
|
|
id,
|
|
created_at,
|
|
updated_at
|
|
FROM {k.schema}."{k.table_name}"
|
|
WHERE name = :source
|
|
""")
|
|
|
|
# Execute query with parameters
|
|
rows = db.execute(query, {"source": k.source})
|
|
|
|
# Group pages by subpage
|
|
pages_dict = {}
|
|
for row in rows:
|
|
subpage = row.subpage
|
|
if subpage not in pages_dict:
|
|
pages_dict[subpage] = {
|
|
"subpage": subpage,
|
|
"created_at": row.created_at.isoformat() if row.created_at else None,
|
|
"updated_at": row.updated_at.isoformat() if row.updated_at else None
|
|
}
|
|
|
|
knowledge_data["pages"] = list(pages_dict.values())
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error querying table {k.table_name}: {str(e)}")
|
|
knowledge_data["error"] = PAGES_UNREADABLE
|
|
|
|
result.append(knowledge_data)
|
|
|
|
|
|
return {
|
|
"knowledge": result,
|
|
"pagination": {
|
|
"total": total_count,
|
|
"page": page,
|
|
"page_size": page_size,
|
|
"total_pages": (total_count + page_size - 1) // page_size
|
|
}
|
|
}
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Error getting knowledge by organization: {str(e)}")
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
|
|
@router.get("/queue/{queue_id}")
|
|
async def get_queue_status(
|
|
queue_id: DbIntId,
|
|
current_user: User = Depends(
|
|
require_any_permission("view_knowledge", "manage_knowledge")),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Get status of a queued knowledge item"""
|
|
try:
|
|
queue_repo = KnowledgeQueueRepository(db)
|
|
item = db.query(KnowledgeQueue).filter(
|
|
KnowledgeQueue.id == queue_id).first()
|
|
|
|
if not item:
|
|
return {"error": "Queue item not found"}
|
|
|
|
if item.organization_id != current_user.organization_id:
|
|
return {"error": "Unauthorized access to queue item"}
|
|
|
|
return {
|
|
"id": item.id,
|
|
"status": item.status,
|
|
"error": item.error,
|
|
"created_at": item.created_at,
|
|
"updated_at": item.updated_at
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error getting queue status: {str(e)}")
|
|
return {"error": "Could not read the queue status"}
|
|
|
|
|
|
@router.get("/processor/status")
|
|
async def get_processor_status(
|
|
current_user: User = Depends(
|
|
require_any_permission("view_knowledge", "manage_knowledge")),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Get status of the knowledge processor for user's organization"""
|
|
try:
|
|
# Get counts of items in different states for user's items
|
|
base_query = db.query(KnowledgeQueue)\
|
|
.filter(
|
|
KnowledgeQueue.organization_id == current_user.organization_id,
|
|
KnowledgeQueue.user_id == current_user.id # Add user filter
|
|
)
|
|
|
|
pending_count = base_query.filter(
|
|
KnowledgeQueue.status == QueueStatus.PENDING
|
|
).count()
|
|
|
|
processing_count = base_query.filter(
|
|
KnowledgeQueue.status == QueueStatus.PROCESSING
|
|
).count()
|
|
|
|
completed_count = base_query.filter(
|
|
KnowledgeQueue.status == QueueStatus.COMPLETED
|
|
).count()
|
|
|
|
failed_count = base_query.filter(
|
|
KnowledgeQueue.status == QueueStatus.FAILED
|
|
).count()
|
|
|
|
return {
|
|
"last_run": PROCESSOR_STATUS["last_run"],
|
|
"is_running": PROCESSOR_STATUS["is_running"],
|
|
"error": PROCESSOR_STATUS["error"],
|
|
"queue_status": {
|
|
"pending": pending_count,
|
|
"processing": processing_count,
|
|
"completed": completed_count,
|
|
"failed": failed_count
|
|
}
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error getting processor status: {str(e)}")
|
|
return {"error": "Could not read the processor status"}
|
|
|
|
|
|
@router.delete("/{knowledge_id}")
|
|
async def delete_knowledge(
|
|
knowledge_id: DbIntId,
|
|
current_user: User = Depends(require_permissions("manage_knowledge")),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Delete a knowledge source and its associated data"""
|
|
try:
|
|
knowledge_repo = KnowledgeRepository(db)
|
|
|
|
# Verify knowledge exists and belongs to user's org
|
|
knowledge = knowledge_repo.get_by_id(knowledge_id)
|
|
if not knowledge:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail="Knowledge source not found"
|
|
)
|
|
|
|
if knowledge.organization_id != current_user.organization_id:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_403_FORBIDDEN,
|
|
detail="Unauthorized access to knowledge source"
|
|
)
|
|
|
|
# Delete knowledge and associated data
|
|
success = knowledge_repo.delete_with_data(knowledge_id)
|
|
|
|
if success:
|
|
return {"message": "Knowledge source deleted successfully"}
|
|
|
|
raise HTTPException(
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
detail="Failed to delete knowledge source"
|
|
)
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Error deleting knowledge: {str(e)}")
|
|
raise HTTPException(
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
detail=str(e)
|
|
)
|
|
|
|
|
|
@router.get("/{knowledge_id}/content")
|
|
async def get_knowledge_content(
|
|
knowledge_id: DbIntId,
|
|
current_user: User = Depends(get_current_user),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Get knowledge content chunks from vector database"""
|
|
try:
|
|
knowledge_repo = KnowledgeRepository(db)
|
|
|
|
# Verify knowledge exists and belongs to user's org
|
|
knowledge = knowledge_repo.get_by_id(knowledge_id)
|
|
if not knowledge:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail="Knowledge source not found"
|
|
)
|
|
|
|
if knowledge.organization_id != current_user.organization_id:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_403_FORBIDDEN,
|
|
detail="Unauthorized access to knowledge source"
|
|
)
|
|
|
|
# Query the vector database for content chunks
|
|
if not knowledge.table_name or not knowledge.schema:
|
|
return {
|
|
"knowledge_id": knowledge_id,
|
|
"source": knowledge.source,
|
|
"chunks": []
|
|
}
|
|
|
|
try:
|
|
# Get all chunks for this knowledge source
|
|
query = text(f"""
|
|
SELECT
|
|
id,
|
|
content,
|
|
meta_data,
|
|
created_at
|
|
FROM {knowledge.schema}."{knowledge.table_name}"
|
|
WHERE name = :source
|
|
ORDER BY created_at ASC, {page_editor.NATURAL_ID_ORDER}
|
|
""")
|
|
|
|
|
|
rows = db.execute(query, {"source": knowledge.source}).fetchall()
|
|
|
|
chunks = []
|
|
for row in rows:
|
|
chunks.append({
|
|
"id": row.id,
|
|
"content": row.content,
|
|
"metadata": row.meta_data,
|
|
"created_at": row.created_at.isoformat() if row.created_at else None
|
|
})
|
|
|
|
return {
|
|
"knowledge_id": knowledge_id,
|
|
"source": knowledge.source,
|
|
"source_type": knowledge.source_type.value,
|
|
"chunks": chunks
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error querying content for knowledge {knowledge_id}: {str(e)}")
|
|
raise HTTPException(
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
detail=f"Error accessing content: {str(e)}"
|
|
)
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Error getting knowledge content: {str(e)}")
|
|
raise HTTPException(
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
detail=str(e)
|
|
)
|
|
|
|
|
|
def _require_editable_knowledge(
|
|
db: Session, knowledge_id: int, current_user: User
|
|
) -> Knowledge:
|
|
"""Load a knowledge source, enforcing ownership and a usable vector table.
|
|
|
|
Raises 404 if missing, 403 if it belongs to another org, 400 if it has no
|
|
vector table yet (e.g. a crawl that has not produced any rows).
|
|
"""
|
|
knowledge = KnowledgeRepository(db).get_by_id(knowledge_id)
|
|
if not knowledge:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail="Knowledge source not found"
|
|
)
|
|
if knowledge.organization_id != current_user.organization_id:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_403_FORBIDDEN,
|
|
detail="Unauthorized access to knowledge source"
|
|
)
|
|
if not knowledge.table_name or not knowledge.schema:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail="Knowledge source has no vector database table"
|
|
)
|
|
return knowledge
|
|
|
|
|
|
@router.put("/{knowledge_id}/chunk/{chunk_id:path}")
|
|
async def update_chunk_content(
|
|
knowledge_id: DbIntId,
|
|
chunk_id: str,
|
|
content: str = Body(..., embed=True),
|
|
current_user: User = Depends(require_permissions("manage_knowledge")),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Update a single chunk's content in vector database"""
|
|
try:
|
|
knowledge = _require_editable_knowledge(db, knowledge_id, current_user)
|
|
|
|
try:
|
|
# Verify the chunk exists AND belongs to this source (the vector
|
|
# table is shared across the org's sources, so scope by name).
|
|
query = text(f"""
|
|
SELECT 1
|
|
FROM {knowledge.schema}."{knowledge.table_name}"
|
|
WHERE id = :chunk_id AND name = :source
|
|
""")
|
|
row = db.execute(
|
|
query, {"chunk_id": chunk_id, "source": knowledge.source}
|
|
).fetchone()
|
|
|
|
if not row:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail="Chunk not found"
|
|
)
|
|
|
|
# Re-embed the updated content in place
|
|
page_editor.reembed_chunk(db, knowledge, chunk_id, content)
|
|
db.commit()
|
|
|
|
logger.info(f"Updated chunk {chunk_id} for knowledge {knowledge_id}")
|
|
|
|
return {"message": "Chunk updated successfully"}
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
db.rollback()
|
|
logger.error(f"Error updating chunk {chunk_id}: {str(e)}")
|
|
raise HTTPException(
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
detail=f"Error updating chunk: {str(e)}"
|
|
)
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Error updating chunk: {str(e)}")
|
|
raise HTTPException(
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
detail=str(e)
|
|
)
|
|
|
|
|
|
@router.delete("/{knowledge_id}/chunk/{chunk_id:path}")
|
|
async def delete_chunk(
|
|
knowledge_id: DbIntId,
|
|
chunk_id: str,
|
|
current_user: User = Depends(require_permissions("manage_knowledge")),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Delete a single chunk from vector database"""
|
|
try:
|
|
knowledge = _require_editable_knowledge(db, knowledge_id, current_user)
|
|
|
|
try:
|
|
# Delete the chunk from the database (scoped to this source, since the
|
|
# vector table is shared across the org's sources).
|
|
delete_query = text(f"""
|
|
DELETE FROM {knowledge.schema}."{knowledge.table_name}"
|
|
WHERE id = :chunk_id AND name = :source
|
|
""")
|
|
|
|
result = db.execute(
|
|
delete_query, {"chunk_id": chunk_id, "source": knowledge.source}
|
|
)
|
|
db.commit()
|
|
|
|
if result.rowcount == 0:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail="Chunk not found"
|
|
)
|
|
|
|
logger.info(f"Deleted chunk {chunk_id} from knowledge {knowledge_id}")
|
|
|
|
return {"message": "Chunk deleted successfully"}
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
db.rollback()
|
|
logger.error(f"Error deleting chunk {chunk_id}: {str(e)}")
|
|
raise HTTPException(
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
detail=f"Error deleting chunk: {str(e)}"
|
|
)
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Error deleting chunk: {str(e)}")
|
|
raise HTTPException(
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
detail=str(e)
|
|
)
|
|
|
|
|
|
@router.post("/{knowledge_id}/subpage")
|
|
async def add_subpage(
|
|
knowledge_id: DbIntId,
|
|
subpage_name: str = Body(..., embed=True),
|
|
content: str = Body(..., embed=True),
|
|
url: Optional[str] = Body(None, embed=True),
|
|
current_user: User = Depends(require_permissions("manage_knowledge")),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Add a new subpage to existing knowledge"""
|
|
try:
|
|
knowledge_repo = KnowledgeRepository(db)
|
|
|
|
# Verify knowledge exists and belongs to user's org
|
|
knowledge = knowledge_repo.get_by_id(knowledge_id)
|
|
if not knowledge:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail="Knowledge source not found"
|
|
)
|
|
|
|
if knowledge.organization_id != current_user.organization_id:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_403_FORBIDDEN,
|
|
detail="Unauthorized access to knowledge source"
|
|
)
|
|
|
|
if not knowledge.table_name or not knowledge.schema:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail="Knowledge source has no vector database table"
|
|
)
|
|
|
|
# Check subscription limits if enterprise module is available
|
|
if HAS_ENTERPRISE:
|
|
try:
|
|
from app.enterprise.repositories.subscription import SubscriptionRepository
|
|
|
|
subscription_repo = SubscriptionRepository(db)
|
|
subscription = subscription_repo.get_active_subscription(str(knowledge.organization_id))
|
|
|
|
if subscription and subscription.plan:
|
|
# Count existing subpages for this knowledge source
|
|
current_count = page_editor.count_subpages(db, knowledge)
|
|
|
|
# Check against max_sub_pages limit from the plan
|
|
subpages_limit = subscription.plan.max_sub_pages
|
|
|
|
if current_count >= subpages_limit:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_402_PAYMENT_REQUIRED,
|
|
detail=f"Subpage limit reached. Your plan allows {subpages_limit} subpages per knowledge source. Please upgrade your plan."
|
|
)
|
|
except ImportError:
|
|
# Enterprise module not available, skip limit check
|
|
pass
|
|
|
|
try:
|
|
# Check if subpage name already exists within this source. A page
|
|
# may be stored only as chunks, so compare the page id, not the row id.
|
|
check_query = text(f"""
|
|
SELECT id FROM {knowledge.schema}."{knowledge.table_name}"
|
|
WHERE {page_editor.PAGE_ID_EXPR} = :subpage_name AND name = :source
|
|
LIMIT 1
|
|
""")
|
|
existing = db.execute(
|
|
check_query, {"subpage_name": subpage_name, "source": knowledge.source}
|
|
).fetchone()
|
|
|
|
if existing:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail="Subpage name already exists. Please use a unique name."
|
|
)
|
|
|
|
# A subpage of a URL-based source must be on the SAME registrable
|
|
# domain as that source — otherwise a single plan-limited source
|
|
# could accumulate content from arbitrary other domains. domain_of_url
|
|
# normalizes a scheme-less value (so 'evil.com/x' can't parse as a
|
|
# path with no host and slip past) and is ccTLD-aware (so a *.co.uk
|
|
# source can't accept another *.co.uk domain).
|
|
clean_url = (url or "").strip() or None
|
|
if clean_url:
|
|
from app.knowledge.domains import domain_of_url
|
|
|
|
parent_domain = domain_of_url(knowledge.source)
|
|
if parent_domain and domain_of_url(clean_url) != parent_domain:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail=f"Subpage URL must be on the same domain as the source ({parent_domain}).",
|
|
)
|
|
|
|
# Embed and insert the new subpage into the vector database
|
|
page_editor.insert_subpage(knowledge, subpage_name, content, clean_url)
|
|
|
|
logger.info(f"Added new subpage '{subpage_name}' to knowledge {knowledge_id}")
|
|
|
|
return {"message": "Subpage added successfully", "subpage_id": subpage_name}
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Error adding subpage: {str(e)}")
|
|
raise HTTPException(
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
detail=f"Error adding subpage: {str(e)}"
|
|
)
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Error adding subpage: {str(e)}")
|
|
raise HTTPException(
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
detail=str(e)
|
|
)
|
|
|
|
|
|
@router.put("/{knowledge_id}/page/{page_id:path}")
|
|
async def update_page_content(
|
|
knowledge_id: DbIntId,
|
|
page_id: str,
|
|
content: str = Body(..., embed=True),
|
|
title: Optional[str] = Body(None, embed=True),
|
|
current_user: User = Depends(require_permissions("manage_knowledge")),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Replace a sub-page's content and re-embed it.
|
|
|
|
A page may span several ``_N`` chunks; this collapses it into one freshly
|
|
embedded chunk keyed by the page id, keeping the source's answers current.
|
|
"""
|
|
if not (content or "").strip():
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail="Page content cannot be empty"
|
|
)
|
|
|
|
knowledge = _require_editable_knowledge(db, knowledge_id, current_user)
|
|
try:
|
|
replaced = page_editor.replace_page(db, knowledge, page_id, content, title)
|
|
if replaced == 0:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail="Page not found"
|
|
)
|
|
logger.info(f"Updated page '{page_id}' for knowledge {knowledge_id}")
|
|
return {"message": "Page updated successfully", "page_id": page_id}
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
db.rollback()
|
|
logger.error(f"Error updating page '{page_id}': {str(e)}")
|
|
raise HTTPException(
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
detail=f"Error updating page: {str(e)}"
|
|
)
|
|
|
|
|
|
@router.delete("/{knowledge_id}/page/{page_id:path}")
|
|
async def delete_page(
|
|
knowledge_id: DbIntId,
|
|
page_id: str,
|
|
current_user: User = Depends(require_permissions("manage_knowledge")),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Delete a whole sub-page (all of its chunks) from a knowledge source."""
|
|
knowledge = _require_editable_knowledge(db, knowledge_id, current_user)
|
|
try:
|
|
removed = page_editor.delete_page_chunks(db, knowledge, page_id)
|
|
if removed == 0:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail="Page not found"
|
|
)
|
|
db.commit()
|
|
logger.info(
|
|
f"Deleted page '{page_id}' ({removed} chunk(s)) from knowledge {knowledge_id}"
|
|
)
|
|
return {"message": "Page deleted successfully", "removed_chunks": removed}
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
db.rollback()
|
|
logger.error(f"Error deleting page '{page_id}': {str(e)}")
|
|
raise HTTPException(
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
detail=f"Error deleting page: {str(e)}"
|
|
)
|
|
|
|
|