replypier/backend/tests/api/test_knowledge.py

"""
Copyright 2024-2026 ReplyPier

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import pytest
from fastapi.testclient import TestClient
from sqlalchemy.orm import Session
from app.database import Base, get_db
from fastapi import FastAPI
from app.models.user import User
from app.models.role import Role
from app.models.permission import Permission, role_permissions
from app.models.knowledge import Knowledge, SourceType
from app.models.knowledge_queue import KnowledgeQueue, QueueStatus
from app.models.knowledge_to_agent import KnowledgeToAgent
from app.models.agent import Agent, AgentType
from uuid import UUID, uuid4
from app.api import knowledge as knowledge_router
from app.core.auth import get_current_user, require_permissions
from app.main import app
from app.core.config import settings
from tests.conftest import engine, TestingSessionLocal, create_tables
from datetime import datetime, timezone
import io
import os
from unittest.mock import PropertyMock, patch
from unittest.mock import MagicMock

# Set enterprise flag to False
knowledge_router.HAS_ENTERPRISE = False

# Create a test FastAPI app
app = FastAPI()
app.include_router(
    knowledge_router.router,
    prefix=f"{settings.API_V1_STR}/knowledge",
    tags=["knowledge"]
)

@pytest.fixture(scope="function")
def db():
    """Create a fresh database for each test."""
    Base.metadata.create_all(bind=engine)
    db = TestingSessionLocal()
    try:
        yield db
    finally:
        db.close()
        Base.metadata.drop_all(bind=engine)

@pytest.fixture
def test_permissions(db) -> list[Permission]:
    """Create test permissions"""
    permissions = []
    for name in ["manage_knowledge"]:
        perm = Permission(
            name=name,
            description=f"Can {name}"
        )
        db.add(perm)
        permissions.append(perm)
    db.commit()
    for p in permissions:
        db.refresh(p)
    return permissions

@pytest.fixture
def test_role(db, test_organization, test_permissions) -> Role:
    """Create a test role with required permissions"""
    role = Role(
        id=1,
        name="Test Role",
        organization_id=test_organization.id
    )
    db.add(role)
    db.commit()

    # Associate permissions with role
    for perm in test_permissions:
        db.execute(
            role_permissions.insert().values(
                role_id=role.id,
                permission_id=perm.id
            )
        )
    db.commit()
    db.refresh(role)
    return role

@pytest.fixture
def test_user(db: Session, test_organization, test_role: Role) -> User:
    """Create a test user with required permissions"""
    user = User(
        id=uuid4(),
        email="test@test.com",
        hashed_password="hashed_password",
        organization_id=test_organization.id,
        role_id=test_role.id,
        is_active=True,
        full_name="Test User"
    )
    db.add(user)
    db.commit()
    db.refresh(user)
    return user

@pytest.fixture
def test_agent(db: Session, test_organization) -> Agent:
    """Create a test agent"""
    agent = Agent(
        id=uuid4(),
        name="Test Agent",
        agent_type=AgentType.CUSTOMER_SUPPORT,
        instructions=["Test instruction"],
        organization_id=test_organization.id,
        is_active=True
    )
    db.add(agent)
    db.commit()
    db.refresh(agent)
    return agent

@pytest.fixture
def test_knowledge(db: Session, test_organization) -> Knowledge:
    """Create a test knowledge entry"""
    knowledge = Knowledge(
        source="test.pdf",
        source_type=SourceType.FILE,
        organization_id=test_organization.id,
        created_at=datetime.now(timezone.utc)
    )
    db.add(knowledge)
    db.commit()
    db.refresh(knowledge)
    return knowledge

@pytest.fixture
def test_knowledge_queue(db: Session, test_organization, test_user) -> KnowledgeQueue:
    """Create a test knowledge queue entry"""
    queue = KnowledgeQueue(
        organization_id=test_organization.id,
        user_id=test_user.id,
        source_type="pdf_file",
        source="test.pdf",
        status=QueueStatus.PENDING,
        created_at=datetime.now(timezone.utc)
    )
    db.add(queue)
    db.commit()
    db.refresh(queue)
    return queue

@pytest.fixture
def client(test_user: User) -> TestClient:
    """Create test client with mocked dependencies"""
    async def override_get_current_user():
        return test_user

    async def override_require_permissions(*args, **kwargs):
        return test_user

    def override_get_db():
        try:
            db = TestingSessionLocal()
            yield db
        finally:
            db.close()

    app.dependency_overrides[get_current_user] = override_get_current_user
    app.dependency_overrides[require_permissions] = lambda x: override_require_permissions
    app.dependency_overrides[get_db] = override_get_db
    
    return TestClient(app)

def test_upload_pdf(client: TestClient, test_organization):
    """Test uploading PDF files"""
    # Create a test PDF file
    pdf_content = b"%PDF-1.4 test pdf content"
    files = [
        ("files", ("test.pdf", io.BytesIO(pdf_content), "application/pdf"))
    ]

    # Create temp directory if it doesn't exist
    os.makedirs("temp", exist_ok=True)

    # Mock S3 configuration and upload
    mock_s3_client = MagicMock()
    mock_s3_client.put_object.return_value = {'ResponseMetadata': {'HTTPStatusCode': 200}}
    mock_s3_client.generate_presigned_url.return_value = "https://test-bucket.s3.amazonaws.com/test.pdf"
    
    with patch('app.core.config.settings.S3_FILE_STORAGE', True), \
         patch('app.core.s3.upload_file_to_s3') as mock_upload, \
         patch('app.core.s3.get_s3_signed_url') as mock_signed_url, \
         patch('app.core.s3.get_s3_client', return_value=mock_s3_client):
        # Configure mocks
        mock_upload.return_value = "s3://test-bucket/test.pdf"
        mock_signed_url.return_value = "https://test-bucket.s3.amazonaws.com/test.pdf"

        response = client.post(
            "/api/v1/knowledge/upload/pdf",
            files=files,
            data={
                "org_id": str(test_organization.id)
            }
        )

        assert response.status_code == 200
        assert "queue_items" in response.json()
        assert len(response.json()["queue_items"]) == 1

@pytest.fixture(autouse=True)
def dns(monkeypatch):
    """No real lookups from tests: DNS "can't tell" unless a test says so."""
    answers = {}

    async def resolves(host):
        return answers.get(host)
    monkeypatch.setattr(knowledge_router, "domain_resolves", resolves)
    return answers


def test_add_urls_fails_a_domain_that_does_not_exist_at_once(client: TestClient, test_organization, dns, db):
    dns["teeckbox.com"] = False
    dns["example.com"] = True
    response = client.post("/api/v1/knowledge/add/urls", json={
        "org_id": str(test_organization.id),
        "websites": ["https://teeckbox.com", "https://example.com"],
    })

    assert response.status_code == 200
    assert [item["status"] for item in response.json()["queue_items"]] == ["failed", "pending"]
    failed = db.query(KnowledgeQueue).filter(KnowledgeQueue.source == "https://teeckbox.com").one()
    # The same words the crawler would have stored, so the reason reads the same
    assert knowledge_router._serialize_queue_item(failed)["reason"].startswith("This site could not be found")


def test_add_urls(client: TestClient, test_organization):
    """Test adding URLs for processing"""
    urls_data = {
        "org_id": str(test_organization.id),
        "pdf_urls": ["https://example.com/test.pdf"],
        "websites": ["https://example.com"]
    }
    
    response = client.post("/api/v1/knowledge/add/urls", json=urls_data)
    
    assert response.status_code == 200
    data = response.json()
    assert "queue_items" in data
    assert len(data["queue_items"]) == 2  # One for PDF, one for website
    assert all(item["status"] == "pending" for item in data["queue_items"])

def test_add_urls_counts_queued_crawls_against_the_plan(
    client: TestClient, test_organization, test_user, db
):
    """A crawl holds its plan slot from the moment it is queued. Before this,
    every request read a count of zero until the queue drained, so a one-source
    plan could be filled several times over in one sitting."""
    db.add(KnowledgeQueue(
        organization_id=test_organization.id,
        user_id=test_user.id,
        source_type="website",
        source="https://already-queued.example",
        status=QueueStatus.PENDING,
    ))
    db.commit()

    plan = MagicMock()
    plan.max_knowledge_sources = 1
    plan.max_sub_pages = 10
    subscription = MagicMock()
    subscription.plan = plan

    with patch.object(knowledge_router, "HAS_ENTERPRISE", True), \
            patch.object(knowledge_router, "require_accessible_subscription",
                         return_value=subscription, create=True):
        response = client.post("/api/v1/knowledge/add/urls", json={
            "org_id": str(test_organization.id),
            "pdf_urls": [],
            "websites": ["https://second.example"],
        })

    assert response.status_code == 403
    assert "Maximum number of knowledge sources (1)" in response.json()["detail"]
    # and nothing new was queued
    assert db.query(KnowledgeQueue).filter(
        KnowledgeQueue.source == "https://second.example"
    ).count() == 0


def test_add_sitemap(client: TestClient, test_organization, db):
    """A sitemaps request enqueues one source_type='sitemap' queue item."""
    response = client.post("/api/v1/knowledge/add/urls", json={
        "org_id": str(test_organization.id),
        "sitemaps": ["https://example.com/sitemap.xml"],
    })
    assert response.status_code == 200
    assert len(response.json()["queue_items"]) == 1
    item = db.query(KnowledgeQueue).filter(
        KnowledgeQueue.source == "https://example.com/sitemap.xml"
    ).first()
    assert item is not None
    assert item.source_type == "sitemap"
    # Non-enterprise honors the configurable KB_MAX_LINKS (not a hardcoded default).
    assert item.queue_metadata.get("max_links") == settings.KB_MAX_LINKS


def test_add_website_honors_kb_max_links(client: TestClient, test_organization, db):
    """A self-hosted (non-enterprise) website crawl records KB_MAX_LINKS, not a
    hardcoded 10, so the setting actually controls crawl scope (issue #237)."""
    with patch("app.core.config.settings.KB_MAX_LINKS", 200):
        response = client.post("/api/v1/knowledge/add/urls", json={
            "org_id": str(test_organization.id),
            "websites": ["https://example.com/docs"],
        })
        assert response.status_code == 200
        item = db.query(KnowledgeQueue).filter(
            KnowledgeQueue.source == "https://example.com/docs"
        ).first()
        assert item is not None
        assert item.queue_metadata.get("max_links") == 200


def test_add_website_records_crawl_scope(client: TestClient, test_organization, db):
    """The chosen crawl scope reaches the crawler via the queue item; omitting it
    leaves the crawler's default (the seed's own host) in charge."""
    response = client.post("/api/v1/knowledge/add/urls", json={
        "org_id": str(test_organization.id),
        "websites": ["https://help.example.com/hc/en"],
        "crawl_scope": "path",
    })
    assert response.status_code == 200
    item = db.query(KnowledgeQueue).filter(
        KnowledgeQueue.source == "https://help.example.com/hc/en"
    ).first()
    assert item.queue_metadata.get("crawl_scope") == "path"

    response = client.post("/api/v1/knowledge/add/urls", json={
        "org_id": str(test_organization.id),
        "websites": ["https://help.example.com/other"],
    })
    assert response.status_code == 200
    item = db.query(KnowledgeQueue).filter(
        KnowledgeQueue.source == "https://help.example.com/other"
    ).first()
    assert item.queue_metadata.get("crawl_scope") is None


def test_add_website_rejects_unknown_crawl_scope(client: TestClient, test_organization):
    response = client.post("/api/v1/knowledge/add/urls", json={
        "org_id": str(test_organization.id),
        "websites": ["https://example.com"],
        "crawl_scope": "everything",
    })
    assert response.status_code == 422


def test_link_knowledge_to_agent(client: TestClient, test_knowledge, test_agent):
    """Test linking knowledge to an agent"""
    response = client.post(
        "/api/v1/knowledge/link",
        params={
            "knowledge_id": test_knowledge.id,
            "agent_id": str(test_agent.id)
        }
    )
    
    assert response.status_code == 200
    data = response.json()
    assert data["message"] == "Knowledge linked to agent successfully"

def _vector_backed(knowledge, db):
    """Give a knowledge row the schema/table_name that triggers a vector sync."""
    knowledge.schema = "ai"
    knowledge.table_name = f"d_{uuid4()}"
    db.commit()


def test_link_syncs_the_vector_store(client: TestClient, test_knowledge, test_agent, db):
    """A link is only real once the agent id reaches the vector filters.

    Retrieval matches on filters->'agent_id'; a link row alone is invisible to
    the agent, so the sync is part of the contract, not a side effect.
    """
    _vector_backed(test_knowledge, db)

    with patch("app.api.knowledge.knowledge_vector_links.add_agent") as add_agent:
        response = client.post(
            "/api/v1/knowledge/link",
            params={"knowledge_id": test_knowledge.id, "agent_id": str(test_agent.id)},
        )

    assert response.status_code == 200
    add_agent.assert_called_once()
    assert add_agent.call_args.args[2] == test_agent.id


def test_link_rolls_back_when_the_vector_sync_fails(
    client: TestClient, test_knowledge, test_agent, db
):
    """Regression (prod, 2026-07-30): the link row was committed before the
    vector update ran, so a failing update left the source shown as linked in
    the dashboard while the agent could not retrieve any of it. The 500 the user
    dismissed was the only hint. Both must land together or not at all.
    """
    _vector_backed(test_knowledge, db)

    with patch(
        "app.api.knowledge.knowledge_vector_links.add_agent",
        side_effect=Exception("vector store unavailable"),
    ):
        response = client.post(
            "/api/v1/knowledge/link",
            params={"knowledge_id": test_knowledge.id, "agent_id": str(test_agent.id)},
        )

    assert response.status_code == 500
    orphan = (
        db.query(KnowledgeToAgent)
        .filter(
            KnowledgeToAgent.knowledge_id == test_knowledge.id,
            KnowledgeToAgent.agent_id == test_agent.id,
        )
        .first()
    )
    assert orphan is None, "link row survived a failed vector sync"


def test_unlink_syncs_the_vector_store(client: TestClient, test_knowledge, test_agent, db):
    _vector_backed(test_knowledge, db)
    db.add(KnowledgeToAgent(knowledge_id=test_knowledge.id, agent_id=test_agent.id))
    db.commit()

    with patch("app.api.knowledge.knowledge_vector_links.remove_agent") as remove_agent:
        response = client.delete(
            "/api/v1/knowledge/unlink",
            params={"knowledge_id": test_knowledge.id, "agent_id": str(test_agent.id)},
        )

    assert response.status_code == 200
    remove_agent.assert_called_once()
    assert remove_agent.call_args.args[2] == test_agent.id


def test_unlink_rolls_back_when_the_vector_sync_fails(
    client: TestClient, test_knowledge, test_agent, db
):
    """The mirror risk: deleting the link row but leaving the agent id in the
    vector filters keeps the source retrievable by an agent the dashboard shows
    as unlinked — knowledge the user believes they revoked.
    """
    _vector_backed(test_knowledge, db)
    db.add(KnowledgeToAgent(knowledge_id=test_knowledge.id, agent_id=test_agent.id))
    db.commit()

    with patch(
        "app.api.knowledge.knowledge_vector_links.remove_agent",
        side_effect=Exception("vector store unavailable"),
    ):
        response = client.delete(
            "/api/v1/knowledge/unlink",
            params={"knowledge_id": test_knowledge.id, "agent_id": str(test_agent.id)},
        )

    assert response.status_code == 500
    db.expire_all()
    still_linked = (
        db.query(KnowledgeToAgent)
        .filter(
            KnowledgeToAgent.knowledge_id == test_knowledge.id,
            KnowledgeToAgent.agent_id == test_agent.id,
        )
        .first()
    )
    assert still_linked is not None, "link row was deleted despite a failed vector sync"


def test_vector_store_updates_are_delegated_not_hand_rolled():
    """Neither handler may build vector-store SQL itself.

    The previous version of this test grepped the handler source for the string
    ':new_filters' — which was the *broken* construct: ':new_filters::jsonb' is
    not parsed as a bind parameter at all, so the statement failed at the driver
    with a syntax error while this test stayed green. Assert the structural
    property instead; the binding and injection behaviour is covered for real in
    tests/services/test_knowledge_vector_links.py.
    """
    import inspect
    from app.api.knowledge import link_knowledge_to_agent, unlink_knowledge_from_agent

    for handler in (link_knowledge_to_agent, unlink_knowledge_from_agent):
        source = inspect.getsource(handler)
        assert "text(" not in source, \
            f"{handler.__name__} should delegate to knowledge_vector_links, not build SQL"
        assert "db.execute(" not in source, \
            f"{handler.__name__} should not run raw statements against the vector table"
        assert "knowledge_vector_links." in source, \
            f"{handler.__name__} should sync the vector store via knowledge_vector_links"


def test_unlink_knowledge_from_agent(client: TestClient, test_knowledge, test_agent, db):
    """Test unlinking knowledge from an agent"""
    # First link the knowledge
    link = KnowledgeToAgent(
        knowledge_id=test_knowledge.id,
        agent_id=test_agent.id
    )
    db.add(link)
    db.commit()
    
    response = client.delete(
        "/api/v1/knowledge/unlink",
        params={
            "knowledge_id": test_knowledge.id,
            "agent_id": str(test_agent.id)
        }
    )
    
    assert response.status_code == 200
    data = response.json()
    assert data["message"] == "Knowledge unlinked from agent successfully"

def test_get_knowledge_by_agent(client: TestClient, test_knowledge, test_agent, db):
    """Test getting knowledge entries by agent"""
    # Link knowledge to agent
    link = KnowledgeToAgent(
        knowledge_id=test_knowledge.id,
        agent_id=test_agent.id
    )
    db.add(link)
    db.commit()
    
    response = client.get(
        f"/api/v1/knowledge/agent/{test_agent.id}",
        params={"page": 1, "page_size": 10}
    )
    
    assert response.status_code == 200
    data = response.json()
    assert "knowledge" in data
    assert len(data["knowledge"]) == 1
    assert data["knowledge"][0]["id"] == test_knowledge.id
    # The source reports which agents it is linked to.
    agents = data["knowledge"][0]["agents"]
    assert [a["id"] for a in agents] == [str(test_agent.id)]
    assert agents[0]["name"] == (test_agent.display_name or test_agent.name)
    assert "pagination" in data
    assert data["pagination"]["page"] == 1
    assert data["pagination"]["page_size"] == 10

def test_get_knowledge_by_organization(client: TestClient, test_knowledge, test_organization):
    """Test getting knowledge entries by organization"""
    response = client.get(
        f"/api/v1/knowledge/organization/{test_organization.id}",
        params={"page": 1, "page_size": 10}
    )
    
    assert response.status_code == 200
    data = response.json()
    assert "knowledge" in data
    assert len(data["knowledge"]) == 1
    assert data["knowledge"][0]["id"] == test_knowledge.id
    assert "pagination" in data
    assert data["pagination"]["page"] == 1
    assert data["pagination"]["page_size"] == 10

def test_get_queue_status(client: TestClient, test_knowledge_queue):
    """Test getting queue status"""
    response = client.get(f"/api/v1/knowledge/queue/{test_knowledge_queue.id}")
    
    assert response.status_code == 200
    data = response.json()
    assert data["id"] == test_knowledge_queue.id
    assert data["status"] == "pending"

def test_get_organization_queue_items(client: TestClient, test_organization, test_knowledge_queue):
    """The org queue endpoint returns in-flight items for the org."""
    response = client.get(f"/api/v1/knowledge/queue/organization/{test_organization.id}")
    assert response.status_code == 200
    sources = [item["source"] for item in response.json()["queue_items"]]
    assert test_knowledge_queue.source in sources


def test_failed_queue_items_carry_a_plain_reason(client: TestClient, db: Session, test_organization, test_user):
    """Raw errors stay in `error`; people get one sentence in `reason`."""
    db.add(KnowledgeQueue(organization_id=test_organization.id, user_id=test_user.id, source_type="website",
                          source="https://nope.invalid", status=QueueStatus.FAILED,
                          error="Traceback: KeyError 'x'", created_at=datetime.now(timezone.utc)))
    db.commit()
    response = client.get(f"/api/v1/knowledge/queue/organization/{test_organization.id}")
    (item,) = [i for i in response.json()["queue_items"] if i["source"] == "https://nope.invalid"]
    assert item["reason"].startswith("We couldn't read any pages there")
    assert "Traceback" not in item["reason"]


def test_get_organization_queue_items_wrong_org(client: TestClient):
    """The org queue endpoint rejects a different organization."""
    response = client.get(f"/api/v1/knowledge/queue/organization/{uuid4()}")
    assert response.status_code == 403


def test_get_processor_status(client: TestClient):
    """Test getting processor status"""
    response = client.get("/api/v1/knowledge/processor/status")
    
    assert response.status_code == 200
    data = response.json()
    assert "queue_status" in data
    assert "is_running" in data
    assert "last_run" in data
    assert "error" in data


def test_processor_status_failure_does_not_return_the_exception(client: TestClient):
    """A failure here used to answer 200 with {"error": str(e)}.

    These routes report on infrastructure, so the exception text is a database
    or Redis error — connection strings and SQL, handed to whoever asked. The
    real reason belongs in the log only.
    """
    leak = "could not connect to postgresql://admin:hunter2@10.0.0.4/app"
    with patch("app.api.knowledge.PROCESSOR_STATUS", new_callable=PropertyMock) as status:
        status.side_effect = RuntimeError(leak)
        response = client.get("/api/v1/knowledge/processor/status")

    assert response.status_code == 200
    body = response.text
    assert "hunter2" not in body
    assert "postgresql://" not in body


def test_queue_status_failure_does_not_return_the_exception(client: TestClient, test_knowledge_queue):
    leak = "relation \"knowledge_queue\" does not exist"
    with patch("app.api.knowledge.KnowledgeQueueRepository") as repo:
        repo.side_effect = RuntimeError(leak)
        response = client.get(f"/api/v1/knowledge/queue/{test_knowledge_queue.id}")

    assert "does not exist" not in response.text

def test_delete_knowledge(client: TestClient, test_knowledge):
    """Test deleting knowledge"""
    response = client.delete(f"/api/v1/knowledge/{test_knowledge.id}")
    
    assert response.status_code == 200
    data = response.json()
    assert data["message"] == "Knowledge source deleted successfully"

# Negative test cases

def test_upload_invalid_file(client: TestClient, test_organization):
    """Test uploading invalid file type is rejected"""
    # A text file must be rejected by content-type validation
    text_content = b"This is not a PDF file"
    files = [
        ("files", ("test.txt", io.BytesIO(text_content), "text/plain"))
    ]

    response = client.post(
        "/api/v1/knowledge/upload/pdf",
        files=files,
        data={
            "org_id": str(test_organization.id)
        }
    )

    assert response.status_code == 400
    assert "unsupported content type" in response.json()["detail"]

def test_a_pdf_over_the_page_limit_is_refused_at_upload(client: TestClient, test_organization, db):
    from tests.knowledge.test_pdf_limits import pdf_with_pages
    files = [("files", ("catalogue.pdf", io.BytesIO(pdf_with_pages(4)), "application/pdf"))]

    with patch('app.core.config.settings.KB_PDF_MAX_PAGES', 3):
        response = client.post("/api/v1/knowledge/upload/pdf", files=files,
                               data={"org_id": str(test_organization.id)})

    assert response.status_code == 400
    assert response.json()["detail"] == (
        "catalogue.pdf: This PDF has 4 pages; the limit is 3. Upload the part you need, "
        "or split it into smaller files.")
    assert db.query(KnowledgeQueue).filter(KnowledgeQueue.organization_id == test_organization.id).count() == 0


def test_a_pdf_within_the_page_limit_is_queued(client: TestClient, test_organization, tmp_path):
    from tests.knowledge.test_pdf_limits import pdf_with_pages
    files = [("files", ("catalogue.pdf", io.BytesIO(pdf_with_pages(3)), "application/pdf"))]

    with patch('app.core.config.settings.KB_PDF_MAX_PAGES', 3), \
         patch('app.core.config.settings.S3_FILE_STORAGE', False), \
         patch.object(knowledge_router, 'TEMP_DIR', str(tmp_path)):
        response = client.post("/api/v1/knowledge/upload/pdf", files=files,
                               data={"org_id": str(test_organization.id)})

    assert response.status_code == 200
    assert len(response.json()["queue_items"]) == 1


def test_upload_fake_pdf_rejected(client: TestClient, test_organization):
    """Test uploading a file with a PDF content type but non-PDF bytes is rejected"""
    # Correct content type, but the magic-byte check must reject it
    files = [
        ("files", ("fake.pdf", io.BytesIO(b"This is not a PDF file"), "application/pdf"))
    ]

    response = client.post(
        "/api/v1/knowledge/upload/pdf",
        files=files,
        data={
            "org_id": str(test_organization.id)
        }
    )

    assert response.status_code == 400
    assert "not a valid PDF" in response.json()["detail"]

def test_add_invalid_urls(client: TestClient, test_organization):
    """Test adding invalid URLs"""
    urls_data = {
        "org_id": str(test_organization.id),
        "pdf_urls": ["not_a_url"],
        "websites": ["not_a_url"]
    }
    
    response = client.post("/api/v1/knowledge/add/urls", json=urls_data)
    
    # The API now validates URL formats strictly and returns 422 for invalid inputs
    assert response.status_code == 422
    data = response.json()
    assert "detail" in data
    # Ensure validation errors point to either pdf_urls or websites fields
    errors = data.get("detail", [])
    assert any(
        any(part in ("websites", "pdf_urls") for part in err.get("loc", []))
        for err in errors
    )

def test_link_nonexistent_knowledge(client: TestClient, test_agent):
    """Test linking nonexistent knowledge"""
    response = client.post(
        "/api/v1/knowledge/link",
        params={
            "knowledge_id": 999999,  # Nonexistent ID
            "agent_id": str(test_agent.id)
        }
    )
    
    assert response.status_code == 404
    assert "Knowledge source not found or unauthorized access" in response.json()["detail"]

def test_get_nonexistent_queue(client: TestClient):
    """Test getting nonexistent queue status"""
    response = client.get("/api/v1/knowledge/queue/999999")

    assert response.status_code == 200  # API returns 200 with error message
    data = response.json()
    assert "error" in data
    assert data["error"] == "Queue item not found"


# Test cases for add_subpage endpoint
def test_add_subpage_knowledge_not_found(client: TestClient):
    """Test adding subpage to non-existent knowledge source"""
    response = client.post(
        "/api/v1/knowledge/999999/subpage",
        json={
            "subpage_name": "test_subpage",
            "content": "Test content for subpage"
        }
    )

    assert response.status_code == 404
    assert "Knowledge source not found" in response.json()["detail"]


def test_add_subpage_unauthorized(client: TestClient, test_knowledge):
    """Test adding subpage to knowledge from different organization"""
    # Create a knowledge source with different org_id
    from uuid import uuid4
    from app.models.organization import Organization
    from tests.conftest import TestingSessionLocal

    db = TestingSessionLocal()
    try:
        # Create a different organization
        other_org = Organization(
            id=uuid4(),
            name="Other Org",
            domain="other.com",
            timezone="UTC"
        )
        db.add(other_org)

        # Create knowledge for different org
        other_knowledge = Knowledge(
            source="other.pdf",
            source_type=SourceType.FILE,
            organization_id=other_org.id,
            created_at=datetime.now(timezone.utc)
        )
        db.add(other_knowledge)
        db.commit()
        db.refresh(other_knowledge)

        response = client.post(
            f"/api/v1/knowledge/{other_knowledge.id}/subpage",
            json={
                "subpage_name": "test_subpage",
                "content": "Test content"
            }
        )

        assert response.status_code == 403
        assert "Unauthorized access to knowledge source" in response.json()["detail"]
    finally:
        db.close()


def test_add_subpage_no_table_name(client: TestClient, test_knowledge):
    """Test adding subpage when knowledge has no vector database table"""
    # test_knowledge by default has no table_name set
    response = client.post(
        f"/api/v1/knowledge/{test_knowledge.id}/subpage",
        json={
            "subpage_name": "test_subpage",
            "content": "Test content for subpage"
        }
    )

    assert response.status_code == 400
    assert "Knowledge source has no vector database table" in response.json()["detail"]


def test_add_subpage_integration_logic():
    """Test that add_subpage endpoint has correct integration points"""
    # This test verifies the endpoint exists and has the expected logic flow
    # Actual database operations with pgvector are tested in integration tests

    # Verify the endpoint is registered
    from app.api.knowledge import router
    route_found = False
    for route in router.routes:
        if hasattr(route, 'path') and '/{knowledge_id}/subpage' in route.path:
            route_found = True
            break

    assert route_found, "add_subpage endpoint should be registered"


def test_add_subpage_with_enterprise_limits_mocked():
    """Test that enterprise limits are checked in add_subpage endpoint"""
    import pytest
    from unittest.mock import patch, MagicMock
    from app.models.user import User
    from app.models.knowledge import Knowledge, SourceType
    from uuid import uuid4
    import app.api.knowledge as knowledge_module

    # Skip this test if enterprise module is not available
    if not knowledge_module.HAS_ENTERPRISE:
        pytest.skip("Enterprise module not available")

    from app.api.knowledge import add_subpage
    from fastapi import HTTPException

    # Create mock objects
    mock_db = MagicMock()
    mock_user = User(
        id=uuid4(),
        email="test@test.com",
        hashed_password="hashed",
        organization_id=uuid4(),
        role_id=1,
        is_active=True,
        full_name="Test User"
    )

    # Create mock knowledge with table_name and schema set
    mock_knowledge = Knowledge(
        id=1,
        source="test.pdf",
        source_type=SourceType.FILE,
        organization_id=mock_user.organization_id,
        table_name="test_table",
        schema="public"
    )
    mock_knowledge.agent_links = []

    # Mock repository
    with patch('app.api.knowledge.KnowledgeRepository') as mock_repo_class:
        mock_repo = MagicMock()
        mock_repo.get_by_id.return_value = mock_knowledge
        mock_repo_class.return_value = mock_repo

        # Mock subscription check with limit reached
        with patch('app.enterprise.repositories.subscription.SubscriptionRepository') as mock_sub_repo_class:
            # Create subscription with plan at limit
            mock_subscription = MagicMock()
            mock_plan = MagicMock()
            mock_plan.max_sub_pages = 2
            mock_subscription.plan = mock_plan

            mock_sub_repo = MagicMock()
            mock_sub_repo.get_active_subscription.return_value = mock_subscription
            mock_sub_repo_class.return_value = mock_sub_repo

            # Mock the database count query to return 2 (at limit)
            mock_result = MagicMock()
            mock_result.count = 2
            mock_db.execute.return_value.fetchone.return_value = mock_result

            # Should raise 402 error
            with pytest.raises(HTTPException) as exc_info:
                # Call the endpoint function directly
                import asyncio
                result = asyncio.run(add_subpage(
                    knowledge_id=1,
                    subpage_name="test",
                    content="test content",
                    current_user=mock_user,
                    db=mock_db
                ))

            assert exc_info.value.status_code == 402
            assert "Subpage limit reached" in exc_info.value.detail


# Test cases for update_page_content / delete_page endpoints
def test_update_page_empty_content(client: TestClient, test_knowledge):
    """Empty page content is rejected before any DB work."""
    response = client.put(
        f"/api/v1/knowledge/{test_knowledge.id}/page/some-page",
        json={"content": "   "}
    )
    assert response.status_code == 400
    assert "Page content cannot be empty" in response.json()["detail"]


def test_update_page_knowledge_not_found(client: TestClient):
    """Updating a page on a non-existent knowledge source returns 404."""
    response = client.put(
        "/api/v1/knowledge/999999/page/some-page",
        json={"content": "New content"}
    )
    assert response.status_code == 404
    assert "Knowledge source not found" in response.json()["detail"]


def test_update_page_no_table_name(client: TestClient, test_knowledge):
    """Updating a page when the source has no vector table returns 400."""
    response = client.put(
        f"/api/v1/knowledge/{test_knowledge.id}/page/some-page",
        json={"content": "New content"}
    )
    assert response.status_code == 400
    assert "Knowledge source has no vector database table" in response.json()["detail"]


def test_delete_page_knowledge_not_found(client: TestClient):
    """Deleting a page on a non-existent knowledge source returns 404."""
    response = client.delete("/api/v1/knowledge/999999/page/some-page")
    assert response.status_code == 404
    assert "Knowledge source not found" in response.json()["detail"]


def test_delete_page_no_table_name(client: TestClient, test_knowledge):
    """Deleting a page when the source has no vector table returns 400."""
    response = client.delete(f"/api/v1/knowledge/{test_knowledge.id}/page/some-page")
    assert response.status_code == 400
    assert "Knowledge source has no vector database table" in response.json()["detail"]


def test_page_endpoints_registered():
    """The page update/delete endpoints are registered on the router."""
    from app.api.knowledge import router
    paths = [route.path for route in router.routes if hasattr(route, 'path')]
    assert any('/{knowledge_id}/page/{page_id:path}' in p for p in paths), \
        "page endpoints should be registered"


def test_replace_page_upserts_before_deleting(monkeypatch):
    """replace_page persists the new content (upsert) before clearing old chunks,
    so an insert failure can never leave the page empty. Ordering is asserted via
    a call log."""
    from types import SimpleNamespace
    from unittest.mock import MagicMock
    from app.knowledge import page_editor

    calls = []

    knowledge = SimpleNamespace(
        schema="ai", table_name="d_test", source="site.com",
        organization_id=uuid4(), agent_links=[],
    )

    # One existing chunk for the page so replace_page proceeds.
    existing_chunk = SimpleNamespace(id="site.com/docs", content="old", meta_data={"url": "site.com/docs"})
    monkeypatch.setattr(page_editor, "get_page_chunks", lambda db, k, pid: [existing_chunk])

    fake_manager = MagicMock()
    fake_manager.vector_db.embedder = object()
    fake_manager.vector_db.upsert.side_effect = lambda *a, **kw: calls.append("upsert")
    monkeypatch.setattr(page_editor, "get_manager", lambda org_id: fake_manager)
    new_chunks = [
        SimpleNamespace(id="site.com/docs::1", embedding=[0.1]),
        SimpleNamespace(id="site.com/docs::2", embedding=[0.2]),
    ]
    monkeypatch.setattr(page_editor, "embed_page", lambda *a, **kw: new_chunks)

    def fake_delete(db, k, pid, keep_ids):
        calls.append(("delete", keep_ids))
        return 1
    monkeypatch.setattr(page_editor, "delete_page_chunks_except", fake_delete)

    db = MagicMock()
    db.commit.side_effect = lambda: calls.append("commit")

    replaced = page_editor.replace_page(db, knowledge, "site.com/docs", "new content", title="Docs")

    assert replaced == 1
    # upsert (persist new content) must happen before the destructive delete,
    # and the delete must spare exactly the rows just written.
    assert calls[0] == "upsert"
    assert calls[1] == ("delete", ["site.com/docs::1", "site.com/docs::2"])
    assert "commit" in calls


# Test cases for crawl-scope (max_links) on add/urls
def test_add_urls_crawl_scope_clamps_max_links(client: TestClient, test_organization, db):
    """A website queued with max_links=1 ('this page only') records that cap."""
    response = client.post("/api/v1/knowledge/add/urls", json={
        "org_id": str(test_organization.id),
        "websites": ["https://scope.example.com"],
        "max_links": 1,
    })
    assert response.status_code == 200
    item = db.query(KnowledgeQueue).filter(
        KnowledgeQueue.source == "https://scope.example.com"
    ).first()
    assert item is not None
    assert item.queue_metadata.get("max_links") == 1


def test_add_urls_rejects_zero_max_links(client: TestClient, test_organization):
    """max_links below 1 is rejected by request validation."""
    response = client.post("/api/v1/knowledge/add/urls", json={
        "org_id": str(test_organization.id),
        "websites": ["https://scope2.example.com"],
        "max_links": 0,
    })
    assert response.status_code == 422


def test_create_text_source_creates_the_vector_table_before_writing(monkeypatch):
    """An org whose first knowledge source is pasted text has no vector table yet
    (only URL/PDF ingestion used to create it), so the write must create it first."""
    from types import SimpleNamespace
    from unittest.mock import MagicMock
    from app.knowledge import page_editor

    calls = []
    fake_manager = MagicMock()
    fake_manager.vector_db.table_name = "d_test"
    fake_manager.vector_db.schema = "ai"
    fake_manager.vector_db.create.side_effect = lambda: calls.append("create")
    fake_manager.vector_db.upsert.side_effect = lambda *a, **kw: calls.append("upsert")
    monkeypatch.setattr(page_editor, "KnowledgeManager", lambda **kw: fake_manager)
    monkeypatch.setattr(page_editor, "embed_page", lambda *a, **kw: [SimpleNamespace(id="t", embedding=[0.1])])

    repo = MagicMock()
    repo.get_by_sources.return_value = []
    repo.create.side_effect = lambda knowledge: knowledge
    monkeypatch.setattr(page_editor, "KnowledgeRepository", lambda db: repo)

    page_editor.create_text_source(MagicMock(), uuid4(), "Returns", "Returns within 45 days.")

    assert calls == ["create", "upsert"]


# Test cases for add/text endpoint
def test_add_text_unauthorized_org(client: TestClient):
    """Adding a text source for a different org is rejected."""
    response = client.post("/api/v1/knowledge/add/text", json={
        "org_id": str(uuid4()),
        "title": "Refund policy",
        "content": "Full refund within 14 days.",
    })
    assert response.status_code == 403


def test_add_text_empty_content_rejected(client: TestClient, test_organization):
    """Blank content is rejected by request validation."""
    response = client.post("/api/v1/knowledge/add/text", json={
        "org_id": str(test_organization.id),
        "title": "Refund policy",
        "content": "   ",
    })
    assert response.status_code == 422


def test_add_text_agent_not_found(client: TestClient, test_organization):
    """A text source targeting a non-existent agent returns 404."""
    response = client.post("/api/v1/knowledge/add/text", json={
        "org_id": str(test_organization.id),
        "title": "Refund policy",
        "content": "Full refund within 14 days.",
        "agent_id": str(uuid4()),
    })
    assert response.status_code == 404


def test_add_text_endpoint_registered():
    """The add/text endpoint is registered on the router."""
    from app.api.knowledge import router
    assert any(
        hasattr(route, 'path') and route.path == '/add/text' for route in router.routes
    ), "add/text endpoint should be registered" 

class TestExploreProgressIsScopedToTheExploreOrg:
    """The explore progress poll is public, so it must only ever expose the
    shared explore organization's jobs — queue ids are sequential."""

    def test_another_orgs_queue_item_is_not_exposed(self, client, test_knowledge_queue):
        """test_knowledge_queue belongs to a normal tenant, not the explore org"""
        response = client.get(
            f"/api/v1/knowledge/explore/progress/{test_knowledge_queue.id}")

        assert response.status_code == 404
        body = response.json()
        # The tenant's crawl source must not leak in any form
        assert "test.pdf" not in str(body)

    def test_explore_org_queue_item_is_exposed(self, client, db, test_user):
        """The public explore flow still gets its own job's progress"""
        from app.models.organization import Organization

        explore_org = Organization(
            id=UUID(settings.EXPLORE_SOURCE_ORG_ID),
            name="Explore Org",
            domain="explore.example.com",
            timezone="UTC"
        )
        db.add(explore_org)
        db.commit()

        queue = KnowledgeQueue(
            organization_id=explore_org.id,
            user_id=test_user.id,
            source_type="website",
            source="https://explore.example.com",
            status=QueueStatus.PENDING,
            created_at=datetime.now(timezone.utc)
        )
        db.add(queue)
        db.commit()
        db.refresh(queue)

        response = client.get(f"/api/v1/knowledge/explore/progress/{queue.id}")

        assert response.status_code == 200


class TestKnowledgeLinkAgentIsOrgScoped:
    """A knowledge source may only be linked to an agent in the same org."""

    def _foreign_agent(self, db):
        from app.models.organization import Organization
        other_org = Organization(
            id=uuid4(), name="Other Org", domain="other-kn.example.com", timezone="UTC")
        db.add(other_org)
        db.commit()
        agent = Agent(
            id=uuid4(),
            name="Foreign Agent",
            agent_type=AgentType.CUSTOMER_SUPPORT,
            instructions=["x"],
            organization_id=other_org.id,
            is_active=True,
        )
        db.add(agent)
        db.commit()
        db.refresh(agent)
        return agent

    def test_link_rejects_agent_from_another_org(self, client, db, test_knowledge):
        foreign_agent = self._foreign_agent(db)

        response = client.post(
            f"/api/v1/knowledge/link?knowledge_id={test_knowledge.id}"
            f"&agent_id={foreign_agent.id}")

        assert response.status_code == 404
        assert response.json()["detail"] == "Agent not found"

    def test_unlink_rejects_agent_from_another_org(self, client, db, test_knowledge):
        foreign_agent = self._foreign_agent(db)

        response = client.delete(
            f"/api/v1/knowledge/unlink?knowledge_id={test_knowledge.id}"
            f"&agent_id={foreign_agent.id}")

        assert response.status_code == 404
        assert response.json()["detail"] == "Agent not found"