feat(modules): 帮助中心、工单、工作流与多渠道接入

补齐 FAQ、帮助中心、工单、可视化工作流及 Telegram/WhatsApp/Slack 等渠道模块。

backend/app/services/channel_chat.py

@@ -0,0 +1,532 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import uuid
+from datetime import datetime, timezone
+from typing import Optional
+
+from sqlalchemy.orm import Session
+
+from app.agents.chat_agent import ChatAgent
+from app.channels import InboundMessage, ChannelInteraction, get_adapter
+from app.channels.app_hooks import HandledBy
+from app.channels.base import InteractiveMessage
+from app.channels.constants import MessageSource
+from app.channels.interactive import from_options
+from app.channels.interactive_limits import InteractiveLimits
+from app.core.security import decrypt_api_key
+from app.core.logger import get_logger
+from app.database import SessionLocal
+from app.models.ai_config import AIModelType
+from app.models.channels import ChannelAccount
+from app.models.session_to_agent import SessionStatus
+from app.repositories.agent import AgentRepository
+from app.repositories.ai_config import AIConfigRepository
+from app.repositories.chat import ChatRepository
+from app.repositories.customer import CustomerRepository
+from app.utils.phone import normalize_msisdn
+from app.repositories.channels import (
+    ChannelAccountRepository,
+    ChannelConversationRepository,
+    AgentChannelConfigRepository,
+)
+from app.repositories.session_to_agent import SessionToAgentRepository
+from app.services.app_first_routing import announce_inbound, app_claims
+from app.services.chat_notifications import notify_new_channel_chat_once, team_not_yet_notified
+from app.services.human_routing import HUMAN_ONLY_ACK, notify_customer
+from app.services.lead_capture import record_lead_from_response
+from app.services.conversation_token import VERIFIED_SCOPE
+from app.services.agent_rooms import USER_MESSAGE, relay_stored_message
+from app.services.business_message import business_message_context
+from app.services.channel_replies import inbound_attributes, reply_text
+from app.services.message_delivery import DeliveryResult, deliver_to_customer
+
+# Optional enterprise message-limit check (same seam as the widget path)
+try:
+    from app.enterprise.services.message_limit import check_message_limit
+    HAS_ENTERPRISE = True
+except ImportError:
+    HAS_ENTERPRISE = False
+
+logger = get_logger(__name__)
+
+
+async def process_channel_message(account_id, inbound: InboundMessage) -> None:
+    """Process one inbound customer message from an external channel.
+
+    Channel-agnostic core shared by every webhook route: resolves the
+    customer and session, stores the message, runs the AI agent (or relays
+    to the handling human), and delivers the reply through the channel's
+    adapter. Runs from a BackgroundTask with its own DB session.
+    """
+    db = SessionLocal()
+    try:
+        account_repo = ChannelAccountRepository(db)
+        account = account_repo.get_by_id(account_id)
+        if account is None or not account.is_active:
+            logger.warning(f"Ignoring message for missing/inactive channel account {account_id}")
+            return
+
+        adapter = get_adapter(account.channel_type)
+
+        agent_id = AgentChannelConfigRepository(db).get_active_agent_id(account.id)
+        if agent_id is None:
+            logger.warning(f"No agent configured for {account.channel_type} account {account.id}")
+            return
+
+        org_id = str(account.organization_id)
+        customer_id = _get_or_create_customer(db, account, inbound, org_id)
+        # If the customer still has a synthesized placeholder name, resolve the
+        # real one via the platform API — only then, not on every message.
+        if adapter is not None:
+            await _enrich_customer_name(db, adapter, account, inbound, customer_id)
+        session_record, conversation, is_new_session = _get_or_create_session(
+            db, account, inbound, agent_id, customer_id, org_id
+        )
+        session_id = str(session_record.session_id)
+
+        # Merge channel-specific per-conversation state (e.g. email threading
+        # headers) so outbound replies can use it.
+        if adapter is not None:
+            state = adapter.conversation_state(inbound)
+            if state:
+                ChannelConversationRepository(db).merge_extra(conversation, state)
+
+        # A human-only agent answers nothing — the chat waits in the inbox for
+        # someone to pick it up, so it takes the same path as one a human
+        # already holds.
+        agent = AgentRepository(db).get_agent(agent_id)
+        ai_replies_enabled = agent.ai_replies_enabled if agent else True
+        # Buttons are the agent's choice (off unless turned on), where the
+        # channel can show them
+        choice_limits = (adapter.choice_limits_for(conversation)
+                         if adapter is not None and agent is not None and agent.channel_quick_replies else None)
+
+        # An installed extension (an external app) may answer first. Its reply
+        # can land while we wait, so the customer's message keeps the time it
+        # arrived and still sorts before it.
+        received_at = datetime.now(timezone.utc)
+        if await app_claims(db, account, session_record, customer_id, inbound, received_at):
+            stored = _store_customer_message(db, inbound, session_record, agent_id,
+                                             customer_id, org_id, account, created_at=received_at)
+            await relay_stored_message(session_record, stored, USER_MESSAGE)
+            if adapter is not None:
+                await adapter.mark_read(account, inbound.external_message_id)
+            await announce_inbound(account, session_record, customer_id, inbound,
+                                   HandledBy.APP, received_at)
+            return
+
+        # From here the team (AI or inbox) is answering
+        await notify_new_channel_chat_once(db, session_record, conversation)
+
+        # Human is handling (or a transfer is pending, or this agent never uses
+        # AI): store + relay to the agent dashboard, never run the bot.
+        if (session_record.user_id is not None
+                or session_record.status == SessionStatus.TRANSFERRED
+                or not ai_replies_enabled):
+            stored = _store_customer_message(db, inbound, session_record, agent_id,
+                                             customer_id, org_id, account)
+            await relay_stored_message(session_record, stored, USER_MESSAGE)
+            await announce_inbound(account, session_record, customer_id, inbound,
+                                   HandledBy.HUMAN, received_at)
+
+            # Nothing will answer this on its own, so acknowledge once — on a
+            # channel an unanswered message reads as a dead number. Only on the
+            # opening message of a brand-new conversation.
+            if is_new_session and not ai_replies_enabled:
+                await notify_customer(db, session_record, HUMAN_ONLY_ACK,
+                                      channels_only=False)
+            return
+
+        await announce_inbound(account, session_record, customer_id, inbound,
+                               HandledBy.AI, received_at)
+
+        ai_config = AIConfigRepository(db).get_active_config(account.organization_id)
+        if ai_config is None:
+            logger.error(f"No AI config for org {org_id}; dropping {account.channel_type} message")
+            return
+
+        if not await _within_message_limit(db, org_id, ai_config):
+            await _send_via_adapter(
+                db, session_record,
+                "Sorry, this assistant has reached its message limit. Please try again later."
+            )
+            return
+
+        # Show a typing indicator while the agent composes its reply
+        if adapter is not None:
+            await adapter.send_typing(account, conversation)
+
+        response, bot_message_id = await _run_chat_agent(
+            db, account, ai_config, inbound,
+            org_id=org_id, agent_id=str(agent_id),
+            customer_id=customer_id, session_id=session_id,
+            conversation=conversation,
+            choice_limits=choice_limits,
+        )
+        if response is None:
+            # The agent hard-failed. Still send a reply so a typing indicator /
+            # placeholder (Slack, LINE) resolves instead of dangling.
+            await _deliver(db, session_record, {
+                'message': "Sorry, I hit an error handling that — please try again.",
+                'type': 'chat_response',
+            }, source=MessageSource.SYSTEM)
+            return
+
+        record_lead_from_response(
+            db, response,
+            organization_id=account.organization_id,
+            agent_id=agent_id,
+            customer_id=customer_id,
+            session_id=session_id,
+            channel=account.channel_type,
+            # The platform vouches for who is messaging.
+            scope=VERIFIED_SCOPE,
+        )
+
+        delivery = await _deliver(db, session_record, {
+            'message': response.message,
+            'type': 'chat_response',
+            'transfer_to_human': getattr(response, 'transfer_to_human', False),
+            'end_chat': getattr(response, 'end_chat', False),
+        }, source=MessageSource.AI, message_id=bot_message_id,
+            interactive=_offered_choices(response, choice_limits))
+        if not delivery.ok and bot_message_id is not None:
+            ChatRepository(db).mark_delivery_failed(bot_message_id, delivery.reason)
+
+        await _send_channel_prompts(db, adapter, account, conversation, response)
+    except Exception as e:
+        logger.error(f"Error processing channel message: {e}", exc_info=True)
+    finally:
+        db.close()
+
+
+async def _send_channel_prompts(db, adapter, account, conversation, response) -> None:
+    """After the reply, surface channel-native prompts. Rating is deliberately
+    NOT asked on external channels — it is a web-widget-only feature; here we
+    only request a phone number when the agent asks for contact."""
+    if adapter is None:
+        return
+    try:
+        if getattr(response, 'request_contact', False):
+            await adapter.request_phone(
+                account, conversation,
+                "To help us follow up, tap below to share your phone number.")
+    except Exception as e:
+        logger.error(f"Failed sending channel prompt: {e}")
+
+
+async def process_channel_interaction(account_id, interaction: ChannelInteraction) -> None:
+    """Handle a non-message interaction (a shared phone number) from a channel.
+    Runs from a BackgroundTask with its own DB session."""
+    db = SessionLocal()
+    try:
+        account = ChannelAccountRepository(db).get_by_id(account_id)
+        if account is None or not account.is_active:
+            return
+        conversation = ChannelConversationRepository(db).get_latest(
+            account.id, interaction.external_conversation_id)
+        if conversation is None:
+            return
+
+        if interaction.type == "contact":
+            _handle_contact(db, conversation, interaction)
+    except Exception as e:
+        logger.error(f"Error processing channel interaction: {e}", exc_info=True)
+    finally:
+        db.close()
+
+
+def _handle_contact(db, conversation, interaction: ChannelInteraction) -> None:
+    """Store a shared phone number on the customer record."""
+    if not interaction.phone or conversation.customer_id is None:
+        return
+    from app.models.customer import Customer
+    customer = db.query(Customer).filter(Customer.id == conversation.customer_id).first()
+    if customer is None:
+        return
+    meta = dict(customer.meta_data or {})
+    meta["phone"] = interaction.phone
+    customer.meta_data = meta
+    # Also promote to the identity column (set-if-absent, conflict-guarded) so
+    # the person becomes phone-addressable; meta_data keeps the raw value for
+    # anything already reading it there. msisdn-lenient because the number is
+    # platform-relayed (Telegram sends it with or without '+'), and one commit
+    # for both writes — separate transactions left a crash window where the
+    # two stores disagreed.
+    phone = normalize_msisdn(interaction.phone)
+    if phone:
+        CustomerRepository(db).set_phone_if_absent(customer, phone)
+    db.commit()
+    logger.info(f"Stored shared phone for customer {conversation.customer_id}")
+
+
+def _get_or_create_customer(db: Session, account: ChannelAccount,
+                            inbound: InboundMessage, org_id: str) -> str:
+    """Resolve the platform user to a Customer.
+
+    When the channel already gives us the customer's real email (e.g. the
+    email channel), use it verbatim so the same person is unified across
+    channels and the widget. Otherwise synthesize a stable per-channel address.
+
+    Phone is the second identity key: adapters that truthfully know the
+    customer's number (WhatsApp, SMS) declare it in profile['phone'], and the
+    repository resolves by phone before email — which is what unifies one human
+    across phone-bearing channels and outbound sends.
+    """
+    real_email = (inbound.profile or {}).get('email')
+    real_name = (inbound.profile or {}).get('name')
+    if real_email and '@' in real_email:
+        channel_email = real_email.lower()
+    else:
+        channel_email = f"{inbound.external_user_id}@{account.channel_type}.channel"
+    customer = CustomerRepository(db).get_or_create_customer(
+        email=channel_email,
+        organization_id=uuid.UUID(org_id),
+        full_name=real_name or placeholder_name(account.channel_type,
+                                                inbound.external_user_id),
+        # msisdn-lenient: adapters supply platform ids (wa_id, SMS sender),
+        # which are E.164-without-plus — trusted in a way typed digits aren't.
+        phone=normalize_msisdn((inbound.profile or {}).get('phone')),
+    )
+    # Upgrade an existing placeholder name once we've resolved the real one
+    # (get_or_create doesn't update an existing customer's name).
+    if real_name and _is_placeholder_name(customer.full_name, account.channel_type):
+        try:
+            customer.full_name = real_name
+            db.commit()
+        except Exception as e:
+            db.rollback()
+            logger.error(f"Failed to update customer name: {e}")
+    return str(customer.id)
+
+
+def placeholder_name(channel_type: str, external_user_id: str) -> str:
+    """The stand-in name for a channel person we don't know yet.
+
+    Paired with _is_placeholder_name deliberately: one builds the string, the
+    other recognises it, and the two only work if they agree exactly. Spelling
+    either out a second time somewhere else (an outbound send, say) makes them
+    agree by coincidence — and the day the format changes, the detector
+    silently stops matching and every real name upgrade stops happening.
+    """
+    return f"{channel_type.capitalize()} user {external_user_id[:8]}"
+
+
+def _is_placeholder_name(name: str, channel_type: str) -> bool:
+    """True when the stored name is the synthesized '<Channel> user xxxx' or empty."""
+    if not name:
+        return True
+    return name.startswith(f"{channel_type.capitalize()} user ")
+
+
+async def _enrich_customer_name(db: Session, adapter, account: ChannelAccount,
+                                inbound: InboundMessage, customer_id: str) -> None:
+    """Resolve the customer's real name via the platform API, but only while
+    the stored name is still a placeholder.
+
+    Channels like Slack send just a user id in the webhook, so we look the name
+    up via e.g. users.info. Gating on the placeholder means we make that network
+    call once — for a new/unresolved customer — not on every inbound message.
+    """
+    customer = CustomerRepository(db).get_by_id(uuid.UUID(customer_id))
+    if customer is None or not _is_placeholder_name(customer.full_name, account.channel_type):
+        return
+    try:
+        enrichment = await adapter.fetch_profile(account, inbound.external_user_id)
+    except Exception as e:
+        logger.debug(f"Profile enrichment failed (non-critical): {e}")
+        return
+    name = (enrichment or {}).get('name')
+    if not name:
+        return
+    try:
+        customer.full_name = name
+        db.commit()
+    except Exception as e:
+        db.rollback()
+        logger.error(f"Failed to update customer name: {e}")
+
+
+def _agent_changed(session_record, agent_id) -> bool:
+    """Whether this channel now answers with a different agent than the one the
+    open session was started with.
+
+    A session is bound to one agent: its stored history, and the AI runtime's
+    own memory for that session id, belong to that agent. Carrying it over to a
+    newly configured agent makes the new agent answer out of the previous one's
+    retrieved knowledge, and splits the inbox list in two — the conversations
+    query groups by agent *and* session, while the detail view keys on session
+    alone, so the same thread appears twice and neither row opens the other.
+
+    A missing session likewise needs a new one. A human-handled or
+    transferred chat is left alone: rotating it would drop the agent out of a
+    live conversation mid-reply.
+    """
+    if session_record is None:
+        return True
+    if session_record.user_id is not None:
+        return False
+    if session_record.status == SessionStatus.TRANSFERRED:
+        return False
+    return str(session_record.agent_id) != str(agent_id)
+
+
+def _get_or_create_session(db: Session, account: ChannelAccount, inbound: InboundMessage,
+                           agent_id, customer_id: str, org_id: str):
+    """Find the open session for this platform conversation or start a new one.
+
+    Returns (session, conversation, is_new) — callers use is_new to notify the
+    team only when a conversation actually starts.
+    """
+    conv_repo = ChannelConversationRepository(db)
+    session_repo = SessionToAgentRepository(db)
+
+    conversation = conv_repo.get_active(account.id, inbound.external_conversation_id)
+    if conversation is not None:
+        session_record = session_repo.get_session(conversation.session_id)
+        if not _agent_changed(session_record, agent_id):
+            conv_repo.touch_inbound(conversation)
+            return session_record, conversation, False
+        # Retire the thread so the newly configured agent starts clean. Closing
+        # the session is what makes get_active skip it, so the create below
+        # opens a fresh conversation for the same platform thread.
+        logger.info(
+            f"Agent changed for {account.channel_type} account {account.id}; "
+            f"starting a new session for conversation {inbound.external_conversation_id}")
+        session_repo.close_session(conversation.session_id)
+
+    session_id = str(uuid.uuid4())
+    session_record = session_repo.create_session(
+        session_id=session_id,
+        agent_id=str(agent_id),
+        customer_id=customer_id,
+        organization_id=org_id,
+        channel=account.channel_type,
+    )
+    conversation = conv_repo.create(
+        channel_account_id=account.id,
+        channel_type=account.channel_type,
+        external_conversation_id=inbound.external_conversation_id,
+        external_user_id=inbound.external_user_id,
+        session_id=uuid.UUID(session_id),
+        organization_id=account.organization_id,
+        agent_id=agent_id,
+        customer_id=uuid.UUID(customer_id),
+        extra=team_not_yet_notified(),
+    )
+    return session_record, conversation, True
+
+
+def _store_customer_message(db: Session, inbound: InboundMessage, session_record,
+                            agent_id, customer_id: str, org_id: str,
+                            account: ChannelAccount, created_at: Optional[datetime] = None):
+    """Persist a customer message on the paths the AI doesn't answer (the AI
+    path persists inside ChatAgent.get_response). Returns the stored row for
+    the relay."""
+    message = {
+        "message": reply_text(db, account.id, inbound),
+        "message_type": "user",
+        "session_id": str(session_record.session_id),
+        "organization_id": org_id,
+        "agent_id": str(agent_id),
+        "customer_id": customer_id,
+        "attributes": inbound_attributes(inbound, account.channel_type),
+    }
+    if created_at is not None:
+        message["created_at"] = created_at
+    return ChatRepository(db).create_message(message)
+
+
+async def _within_message_limit(db: Session, org_id: str, ai_config) -> bool:
+    """Enterprise subscription limit; permissive on any failure so the
+    community edition and enterprise errors never block conversations."""
+    if not (HAS_ENTERPRISE and ai_config.model_type == AIModelType.REPLYPIER):
+        return True
+    try:
+        return await check_message_limit(db, org_id, None, None)
+    except Exception as e:
+        logger.error(f"Message limit check failed (allowing message): {e}")
+        return True
+
+
+async def _run_chat_agent(db: Session, account: ChannelAccount, ai_config,
+                          inbound: InboundMessage, org_id: str, agent_id: str,
+                          customer_id: str, session_id: str, conversation=None,
+                          choice_limits: Optional[InteractiveLimits] = None):
+    """Run the AI agent for one turn. Workflow-enabled agents fall back to the
+    plain agent on external channels (workflow forms are widget-only).
+
+    Returns (response, id of the stored reply row); response is None when the
+    agent hard-failed."""
+    chat_agent = await ChatAgent.create_async(
+        api_key=decrypt_api_key(ai_config.encrypted_api_key),
+        model_name=ai_config.model_name,
+        model_type=ai_config.model_type.value if hasattr(ai_config.model_type, 'value') else ai_config.model_type,
+        base_url=(ai_config.settings or {}).get('base_url'),
+        org_id=org_id,
+        agent_id=agent_id,
+        customer_id=customer_id,
+        session_id=session_id,
+        channel=account.channel_type,
+        extra_context=business_message_context(conversation),
+        choice_limits=choice_limits,
+    )
+    try:
+        response = await chat_agent.get_response(
+            message=reply_text(db, account.id, inbound),
+            session_id=session_id,
+            org_id=org_id,
+            agent_id=agent_id,
+            customer_id=customer_id,
+            user_attributes=inbound_attributes(inbound, account.channel_type),
+        )
+        return response, chat_agent.last_bot_message_id
+    except Exception as e:
+        logger.error(f"Chat agent error on {account.channel_type} session {session_id}: {e}")
+        return None, None
+    finally:
+        await chat_agent.safe_cleanup_mcp_tools()
+
+
+def _offered_choices(response, limits: Optional[InteractiveLimits]) -> Optional[InteractiveMessage]:
+    """The agent's choices as the interactive message to send with its reply."""
+    options = getattr(response, 'quick_replies', None)
+    return from_options(response.message, options, limits) if options and limits is not None else None
+
+
+async def _deliver(db: Session, session_record, payload: dict, *, source: MessageSource,
+                   message_id: Optional[int] = None,
+                   interactive: Optional[InteractiveMessage] = None) -> DeliveryResult:
+    """Deliver a reply and log any failure.
+
+    Unlike the widget path there is no agent socket to notify here, so a failed
+    send would otherwise be silent: the customer gets nothing and nothing says
+    so. Callers that have a stored message act further on the result.
+    """
+    result = await deliver_to_customer(db, session_record, payload, message_id=message_id,
+                                       interactive=interactive, source=source)
+    if not result.ok:
+        logger.warning(
+            f"Undelivered reply on {getattr(session_record, 'channel', 'unknown')} "
+            f"session {session_record.session_id}: {result.reason}")
+    return result
+
+
+async def _send_via_adapter(db: Session, session_record, text: str) -> None:
+    """Send a plain service message to the customer's channel."""
+    await _deliver(db, session_record, {'message': text, 'type': 'chat_response'},
+                   source=MessageSource.SYSTEM)

backend/app/services/channel_replies.py

@@ -0,0 +1,103 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+What a channel customer's message carries beyond its text — a tapped
+button, a shared location — and how a tap on one of the agent's own choices
+reads back as the full option.
+"""
+
+from typing import Optional
+from uuid import UUID
+
+from sqlalchemy.orm import Session
+
+from app.channels.base import InboundMessage
+from app.channels.sent_choices import (
+    SENT_CHOICES_ATTRIBUTE, decode_sent_choices, sent_choice_text, sent_numbered_choice,
+)
+from app.repositories.chat import ChatRepository
+
+# chat_history.attributes key for a tap: only its type and id. The visible
+# text is already the (encrypted) message, and a shared location's details
+# live there too — nothing customer-identifying goes in plaintext attributes.
+REPLY_ATTRIBUTE = "interactive_reply"
+# How far back a tap without reply context is matched to the reply that offered it
+RECENT_OFFERS = 10
+
+
+def inbound_attributes(inbound: InboundMessage, channel_type: str) -> dict:
+    """The chat_history attributes for a customer's channel message."""
+    attributes = {
+        "channel": channel_type,
+        "external_message_id": inbound.external_message_id,
+    }
+    if inbound.interactive is not None:
+        attributes[REPLY_ATTRIBUTE] = {
+            "type": inbound.interactive.type.value,
+            "id": inbound.interactive.id,
+        }
+    if inbound.reply_to_message_id:
+        attributes["reply_to_message_id"] = inbound.reply_to_message_id
+    return attributes
+
+
+def reply_text(db: Session, channel_account_id: UUID, inbound: InboundMessage) -> str:
+    """The text to store and answer for this message.
+
+    A tap arrives as the button's title, which may have been cut to fit (or,
+    on Telegram, as just the button's id); a reply to choices shown as a
+    numbered list may be just the number. The full choice is in the record
+    of what the offering message sent, looked up within this customer's chat
+    on this account — by the tap's reply context where the platform gives
+    one, otherwise as the latest reply that offered the choice.
+    """
+    choice = _tapped_choice(db, channel_account_id, inbound) or _numbered_choice(db, channel_account_id, inbound)
+    if choice is not None:
+        return choice
+    return inbound.text or (inbound.interactive.title if inbound.interactive else "") or ""
+
+
+def _tapped_choice(db: Session, channel_account_id: UUID, inbound: InboundMessage) -> Optional[str]:
+    if inbound.interactive is None:
+        return None
+    repo = ChatRepository(db)
+    if inbound.reply_to_message_id:
+        offered = repo.in_channel_chat_by_external_message_id(
+            channel_account_id, inbound.external_conversation_id, inbound.reply_to_message_id)
+        candidates = [offered] if offered is not None else []
+    else:
+        candidates = repo.latest_replies_in_channel_chat(
+            channel_account_id, inbound.external_conversation_id, RECENT_OFFERS)
+    for offered in candidates:
+        text = sent_choice_text(_record(offered), inbound.interactive.id)
+        if text is not None:
+            return text
+    return None
+
+
+def _numbered_choice(db: Session, channel_account_id: UUID, inbound: InboundMessage) -> Optional[str]:
+    """A bare number answering our latest reply — the AI's or a person's —
+    when that reply showed its choices numbered (as text, or alongside
+    buttons a desktop app hides)."""
+    typed = (inbound.text or "").strip().rstrip(".")
+    if inbound.interactive is not None or not typed.isdecimal():
+        return None
+    latest = ChatRepository(db).latest_replies_in_channel_chat(
+        channel_account_id, inbound.external_conversation_id, 1)
+    return sent_numbered_choice(_record(latest[0]), typed) if latest else None
+
+
+def _record(message) -> Optional[dict]:
+    return decode_sent_choices((message.attributes or {}).get(SENT_CHOICES_ATTRIBUTE))

backend/app/services/channel_status.py

@@ -0,0 +1,186 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Channel delivery receipts (sent / delivered / read / failed) applied to the
+stored messages they belong to.
+"""
+
+import json
+from datetime import datetime, timezone
+from typing import List, Set
+from uuid import UUID
+
+from sqlalchemy.orm import Session
+
+from app.channels.app_hooks import ChannelEvent, EventKind, emit_channel_event, status_event_id
+from app.channels.base import DeliveryState, StatusEvent
+from app.core.logger import get_logger
+from app.core.redis import get_redis
+from app.database import SessionLocal
+from app.models.chat_history import ChatHistory
+from app.repositories.channels import ChannelAccountRepository
+from app.repositories.chat import ChatRepository
+from app.repositories.session_to_agent import SessionToAgentRepository
+from app.services.agent_rooms import emit_to_org_inbox
+
+logger = get_logger(__name__)
+
+# A receipt can arrive before the send that produced it has stored the
+# message id (the Graph response and the webhook race). It waits here until
+# record_sent picks it up; long enough to cover the race, short enough that a
+# receipt for a message we never stored doesn't linger.
+PENDING_KEY = "channel_status_pending:{account_id}:{message_id}"
+PENDING_TTL_SECONDS = 600
+
+# Receipts can arrive out of order and more than once, so a status only
+# replaces one it outranks. FAILED replaces sent/delivered but never READ:
+# a message the customer read was delivered, whatever arrives after.
+# Keyed by the stored value (chat_history.channel_status).
+_RANK = {DeliveryState.SENT.value: 0, DeliveryState.DELIVERED.value: 1, DeliveryState.READ.value: 2}
+
+
+def replaceable_by(new: DeliveryState) -> Set[str]:
+    """The stored statuses receipt `new` may overwrite (no status at all is
+    always overwritten)."""
+    if new == DeliveryState.FAILED:
+        return {DeliveryState.SENT.value, DeliveryState.DELIVERED.value}
+    return {state for state, rank in _RANK.items() if rank < _RANK[new.value]}
+
+
+async def process_status_event(account_id: UUID, event: StatusEvent) -> None:
+    """Apply one receipt from a channel webhook. Runs from a BackgroundTask
+    with its own DB session."""
+    db = SessionLocal()
+    try:
+        account = ChannelAccountRepository(db).get_by_id(account_id)
+        if account is None:
+            return
+        # Announced before matching: extensions count every receipt, including
+        # ones for messages ReplyPier never stored.
+        await emit_channel_event(ChannelEvent(
+            kind=EventKind.STATUS,
+            event_id=status_event_id(account.id, event.external_message_id, event.status.value),
+            organization_id=account.organization_id,
+            channel_account_id=account.id,
+            occurred_at=event.timestamp or datetime.now(timezone.utc),
+            status=event,
+        ))
+        chat_repo = ChatRepository(db)
+        message = chat_repo.get_by_external_message_id(
+            account.organization_id, event.external_message_id)
+        if message is None:
+            _park(account_id, event)
+            # record_sent may have stored the id and drained the queue between
+            # the lookup and the park; look again so the receipt isn't stranded.
+            message = chat_repo.get_by_external_message_id(
+                account.organization_id, event.external_message_id)
+            if message is None:
+                return
+            for parked in _take_parked(account_id, event.external_message_id):
+                await _apply(db, message, parked)
+            return
+        await _apply(db, message, event)
+    except Exception as e:
+        logger.error(f"Error processing channel status event: {e}", exc_info=True)
+    finally:
+        db.close()
+
+
+async def record_sent(db: Session, message_id: int, account_id: UUID,
+                      external_message_id: str) -> None:
+    """Store the platform id of a message we just sent, then apply any
+    receipts that arrived before it."""
+    message = ChatRepository(db).set_external_message_id(message_id, external_message_id)
+    if message is None:
+        return
+    for event in _take_parked(account_id, external_message_id):
+        await _apply(db, message, event)
+
+
+async def _apply(db: Session, message: ChatHistory, event: StatusEvent) -> None:
+    chat_repo = ChatRepository(db)
+    advanced = chat_repo.advance_channel_status(
+        message.id, event.status.value, event.timestamp or datetime.now(timezone.utc),
+        replaceable_by(event.status))
+    if not advanced:
+        return
+    db.refresh(message)
+    if event.status == DeliveryState.FAILED:
+        # Meta can accept a send and fail it later; this is the only place
+        # that failure becomes visible.
+        chat_repo.mark_delivery_failed(message.id, event.error)
+    session = SessionToAgentRepository(db).get_session(message.session_id)
+    await emit_to_org_inbox('message_status', {
+        'session_id': str(message.session_id),
+        'message_id': message.id,
+        'external_message_id': message.external_message_id,
+        'channel_status': message.channel_status,
+        'delivery_status': (message.attributes or {}).get('delivery_status'),
+    }, message.organization_id, session.user_id if session else None)
+
+
+def _pending_key(account_id: UUID, external_message_id: str) -> str:
+    return PENDING_KEY.format(account_id=account_id, message_id=external_message_id)
+
+
+def _park(account_id: UUID, event: StatusEvent) -> None:
+    redis = get_redis()
+    if redis is None:
+        return
+    key = _pending_key(account_id, event.external_message_id)
+    try:
+        pipe = redis.pipeline()
+        pipe.rpush(key, _serialize(event))
+        pipe.expire(key, PENDING_TTL_SECONDS)
+        pipe.execute()
+    except Exception as e:
+        logger.warning(f"Could not park channel status event: {e}")
+
+
+def _take_parked(account_id: UUID, external_message_id: str) -> List[StatusEvent]:
+    redis = get_redis()
+    if redis is None:
+        return []
+    key = _pending_key(account_id, external_message_id)
+    try:
+        pipe = redis.pipeline()
+        pipe.lrange(key, 0, -1)
+        pipe.delete(key)
+        raw_events, _ = pipe.execute()
+    except Exception as e:
+        logger.warning(f"Could not read parked channel status events: {e}")
+        return []
+    return [_deserialize(raw, external_message_id) for raw in raw_events or []]
+
+
+def _serialize(event: StatusEvent) -> str:
+    # Only what _apply uses — no recipient number or pricing sits in Redis
+    return json.dumps({
+        'status': event.status.value,
+        'timestamp': event.timestamp.isoformat() if event.timestamp else None,
+        'error': event.error,
+    })
+
+
+def _deserialize(raw: str, external_message_id: str) -> StatusEvent:
+    data = json.loads(raw)
+    return StatusEvent(
+        external_account_id="",
+        external_message_id=external_message_id,
+        recipient_id="",
+        status=DeliveryState(data['status']),
+        timestamp=datetime.fromisoformat(data['timestamp']) if data['timestamp'] else None,
+        error=data.get('error'),
+    )

backend/app/services/db_connector_service.py

@@ -0,0 +1,319 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Execution layer for guardrailed ticket DB connectors. All functions here are
+synchronous (drivers are sync) — callers run them via asyncio.to_thread.
+
+The read-only guarantee is enforced at the SESSION level, independently of
+the SQL guardrails: default_transaction_read_only for Postgres, SESSION
+TRANSACTION READ ONLY for MySQL, plus statement timeouts on both.
+"""
+
+import io
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional, Tuple
+from uuid import UUID
+
+from app.core.logger import get_logger
+from app.core.outbound_policy import pin_outbound_target
+from app.core.security import decrypt_api_key
+
+logger = get_logger(__name__)
+
+CONNECT_TIMEOUT_SECONDS = 8
+SSH_CONNECT_TIMEOUT_SECONDS = 10
+# Schemas that are never discovered or queried.
+SYSTEM_SCHEMAS = (
+    "pg_catalog", "information_schema", "pg_toast",
+    "mysql", "performance_schema", "sys",
+)
+
+
+@dataclass
+class DBConnectorConfig:
+    """Plain snapshot of a TicketDBConnector row (secrets decrypted) so the
+    executor never holds ORM objects across sessions/threads."""
+    id: Optional[UUID]
+    organization_id: Optional[UUID]
+    name: str
+    engine: str
+    host: str
+    port: int
+    database: str
+    username: str
+    password: str
+    allowed_tables: List[str] = field(default_factory=list)
+    masked_columns: List[str] = field(default_factory=list)
+    # {"table": "customer_column"} — tables restricted to the ticket
+    # customer's own rows. See sql_guardrails._apply_row_scope.
+    row_scope: dict = field(default_factory=dict)
+    row_scope_key: str = "email"
+    max_rows: int = 100
+    statement_timeout_ms: int = 5000
+    # SSH tunnel (bastion/jump host) — the common production topology where
+    # the database isn't directly reachable.
+    ssh_enabled: bool = False
+    ssh_host: Optional[str] = None
+    ssh_port: int = 22
+    ssh_username: Optional[str] = None
+    ssh_password: Optional[str] = None
+    ssh_private_key: Optional[str] = None
+    ssh_private_key_passphrase: Optional[str] = None
+
+    @classmethod
+    def from_model(cls, connector) -> "DBConnectorConfig":
+        return cls(
+            id=connector.id,
+            organization_id=connector.organization_id,
+            name=connector.name,
+            engine=str(connector.engine),
+            host=connector.host,
+            port=connector.port,
+            database=connector.database,
+            username=connector.username,
+            password=decrypt_api_key(connector.encrypted_password),
+            allowed_tables=list(connector.allowed_tables or []),
+            masked_columns=list(connector.masked_columns or []),
+            row_scope=dict(connector.row_scope or {}),
+            row_scope_key=connector.row_scope_key or "email",
+            max_rows=connector.max_rows or 100,
+            statement_timeout_ms=connector.statement_timeout_ms or 5000,
+            ssh_enabled=bool(connector.ssh_enabled),
+            ssh_host=connector.ssh_host,
+            ssh_port=connector.ssh_port or 22,
+            ssh_username=connector.ssh_username,
+            ssh_password=(
+                decrypt_api_key(connector.encrypted_ssh_password)
+                if connector.encrypted_ssh_password else None
+            ),
+            ssh_private_key=(
+                decrypt_api_key(connector.encrypted_ssh_private_key)
+                if connector.encrypted_ssh_private_key else None
+            ),
+            ssh_private_key_passphrase=(
+                decrypt_api_key(connector.encrypted_ssh_key_passphrase)
+                if connector.encrypted_ssh_key_passphrase else None
+            ),
+        )
+
+
+def _load_ssh_key(key_text: str, passphrase: Optional[str]):
+    """Parse a PEM/OpenSSH private key of any common type from a string,
+    without writing it to disk."""
+    import paramiko
+
+    last_error = None
+    # DSA is obsolete — Ed25519/RSA/ECDSA cover every key type a modern host
+    # issues.
+    for key_class in (paramiko.Ed25519Key, paramiko.RSAKey, paramiko.ECDSAKey):
+        try:
+            return key_class.from_private_key(
+                io.StringIO(key_text), password=passphrase or None
+            )
+        except Exception as e:  # wrong type or bad passphrase — try the next
+            last_error = e
+    raise ValueError(f"Unsupported or unreadable SSH private key: {last_error}")
+
+
+@contextmanager
+def _ssh_tunnel(config: DBConnectorConfig, ssh_address: Optional[str] = None):
+    """Open an SSH local-forward tunnel to the database and yield the local
+    (host, port) to connect through. ``ssh_address`` is the bastion's vetted
+    IP when the outbound policy pinned one. Always torn down on exit."""
+    from sshtunnel import SSHTunnelForwarder
+
+    kwargs = {
+        "ssh_username": config.ssh_username,
+        "remote_bind_address": (config.host, config.port),
+        # 127.0.0.1:0 — the OS assigns a free local port per tunnel.
+        "local_bind_address": ("127.0.0.1", 0),
+        "set_keepalive": 15.0,
+    }
+    if config.ssh_private_key:
+        kwargs["ssh_pkey"] = _load_ssh_key(
+            config.ssh_private_key, config.ssh_private_key_passphrase
+        )
+    elif config.ssh_password:
+        kwargs["ssh_password"] = config.ssh_password
+    else:
+        raise ValueError("SSH tunnel needs a private key or password")
+
+    tunnel = SSHTunnelForwarder((ssh_address or config.ssh_host, config.ssh_port), **kwargs)
+    tunnel.start()
+    try:
+        yield "127.0.0.1", tunnel.local_bind_port
+    finally:
+        try:
+            tunnel.stop()
+        except Exception as e:
+            logger.warning(f"SSH tunnel teardown warning (non-critical): {e}")
+
+
+def _driver_connect(config: DBConnectorConfig, host: str, port: int,
+                    hostaddr: Optional[str] = None):
+    """Open a read-only DB session with a statement timeout, to (host, port)
+    — which is the tunnel's local endpoint when tunneling. ``hostaddr`` is
+    the vetted IP to dial instead of resolving ``host`` again."""
+    if config.engine == "mysql":
+        import pymysql
+        conn = pymysql.connect(
+            # pymysql has no separate address field; no TLS is configured
+            # here, so the name isn't needed for verification.
+            host=hostaddr or host,
+            port=port,
+            user=config.username,
+            password=config.password,
+            database=config.database,
+            connect_timeout=CONNECT_TIMEOUT_SECONDS,
+            read_timeout=max(1, config.statement_timeout_ms // 1000 + 1),
+        )
+        with conn.cursor() as cursor:
+            cursor.execute("SET SESSION TRANSACTION READ ONLY")
+            # max_execution_time bounds SELECTs server-side (milliseconds).
+            cursor.execute(f"SET SESSION max_execution_time = {int(config.statement_timeout_ms)}")
+        return conn
+    import psycopg
+    # libpq dials hostaddr and keeps host for TLS verification.
+    pinned = {"hostaddr": hostaddr} if hostaddr else {}
+    return psycopg.connect(
+        host=host,
+        **pinned,
+        port=port,
+        dbname=config.database,
+        user=config.username,
+        password=config.password,
+        connect_timeout=CONNECT_TIMEOUT_SECONDS,
+        options=(
+            f"-c statement_timeout={int(config.statement_timeout_ms)} "
+            "-c default_transaction_read_only=on"
+        ),
+    )
+
+
+def first_hop(ssh_enabled: bool, ssh_host: Optional[str], host: Optional[str]) -> Tuple[Optional[str], str]:
+    """The host this server dials first, with its field label, for the
+    outbound policy: the SSH bastion when tunnelling, else the database. A
+    database behind a bastion is reached from the bastion, and may be private."""
+    if ssh_enabled:
+        return ssh_host, "SSH host"
+    return host, "Database host"
+
+
+@contextmanager
+def _connection(config: DBConnectorConfig):
+    """Yield a read-only DB connection, opening an SSH tunnel first when the
+    connector is configured for one. Tunnel and connection are both closed on
+    exit — a fresh pair per call keeps the (short-lived) investigation queries
+    isolated."""
+    # Vetted on every connect, not only on save, and the vetted address is
+    # the one dialled: DNS can change, even between two lookups.
+    pinned = pin_outbound_target(*first_hop(config.ssh_enabled, config.ssh_host, config.host))
+    if config.ssh_enabled:
+        if not config.ssh_host or not config.ssh_username:
+            raise ValueError("SSH tunnel is enabled but host/username are missing")
+        with _ssh_tunnel(config, pinned) as (host, port):
+            conn = _driver_connect(config, host, port)
+            try:
+                yield conn
+            finally:
+                try:
+                    conn.close()
+                except Exception:
+                    pass
+    else:
+        conn = _driver_connect(config, config.host, config.port, pinned)
+        try:
+            yield conn
+        finally:
+            try:
+                conn.close()
+            except Exception:
+                pass
+
+
+def run_readonly_query(
+    config: DBConnectorConfig, sql: str
+) -> Tuple[List[str], List[tuple]]:
+    """Execute validated SQL in a read-only session; fetches at most
+    max_rows + 1 (the validator already forced a LIMIT — this is belt and
+    braces)."""
+    with _connection(config) as conn:
+        with conn.cursor() as cursor:
+            cursor.execute(sql)
+            columns = [d[0] for d in (cursor.description or [])]
+            rows = cursor.fetchmany(config.max_rows + 1)
+        return columns, list(rows)
+
+
+def test_connection(config: DBConnectorConfig) -> Dict:
+    """Connect and discover user tables + columns (for the allowlist picker).
+    Returns {"tables": [{"schema", "table", "columns": [{"name", "type"}]}]}."""
+    placeholders = ", ".join(["%s"] * len(SYSTEM_SCHEMAS))
+    sql = (
+        "SELECT table_schema, table_name, column_name, data_type "
+        "FROM information_schema.columns "
+        f"WHERE table_schema NOT IN ({placeholders}) "
+        "ORDER BY table_schema, table_name, ordinal_position"
+    )
+    with _connection(config) as conn:
+        with conn.cursor() as cursor:
+            cursor.execute(sql, SYSTEM_SCHEMAS)
+            rows = cursor.fetchall()
+
+    tables: Dict[Tuple[str, str], List[Dict]] = {}
+    for schema, table, column, data_type in rows:
+        tables.setdefault((schema, table), []).append({"name": column, "type": data_type})
+    return {
+        "tables": [
+            {"schema": schema, "table": table, "columns": columns}
+            for (schema, table), columns in tables.items()
+        ]
+    }
+
+
+def describe_columns(config: DBConnectorConfig, schema: str, table: str) -> List[Dict]:
+    """Column names/types of one table via information_schema."""
+    with _connection(config) as conn:
+        with conn.cursor() as cursor:
+            cursor.execute(
+                "SELECT column_name, data_type FROM information_schema.columns "
+                "WHERE table_schema = %s AND table_name = %s ORDER BY ordinal_position",
+                (schema, table),
+            )
+            return [{"name": name, "type": data_type} for name, data_type in cursor.fetchall()]
+
+
+def format_result_table(
+    columns: List[str], rows: List[tuple], max_rows: int, max_value_chars: int = 200
+) -> str:
+    """Compact text rendering of a result set for the model."""
+    if not columns:
+        return "(no result)"
+    shown = rows[:max_rows]
+    lines = [" | ".join(columns)]
+    for row in shown:
+        lines.append(
+            " | ".join(
+                (str(value)[:max_value_chars] if value is not None else "NULL")
+                for value in row
+            )
+        )
+    suffix = f"({len(shown)} row{'s' if len(shown) != 1 else ''}"
+    if len(rows) > max_rows:
+        suffix += ", truncated"
+    suffix += ")"
+    lines.append(suffix)
+    return "\n".join(lines)

backend/app/services/faq_article_import.py

@@ -0,0 +1,581 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Article-mode help-center import: crawl the linked article pages of an existing
+help-center/FAQ index and import each one AS-IS — HTML converted straight to
+Markdown (no LLM, no credits), images downloaded and re-hosted on our storage,
+links absolutized. One draft FAQ per article: question = page title, answer =
+the article body in Markdown.
+"""
+
+import asyncio
+import re
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional, Tuple
+from urllib.parse import urldefrag, urljoin, urlparse
+
+import httpx
+from bs4 import BeautifulSoup, Tag
+from markdownify import MarkdownConverter
+
+from app.core.config import settings
+from app.core.logger import get_logger
+from app.knowledge.main_content import select_main_node
+from app.knowledge.sitemap_parser import _registrable_domain
+from app.knowledge.url_safety import safe_get
+from app.models.faq import FAQ, FAQ_SOURCE_URL_MAX_LENGTH, FAQStatus
+from app.models.faq_generation_job import FAQGenerationJob, FAQJobStage
+from app.models.schemas.faq import MAX_ANSWER_LENGTH, MAX_QUESTION_LENGTH
+from app.repositories.faq import FAQRepository
+from app.repositories.faq_generation_job import FAQGenerationJobRepository
+from app.services.faq_generation import CategoryMerger, insert_draft_rows, normalize_question
+from app.services.faq_import import _FETCH_HEADERS, MIN_STATIC_TEXT_CHARS
+from app.services.help_center_images import (
+    FAQ_IMAGE_TYPES,
+    MAX_FAQ_IMAGE_BYTES,
+    store_article_image,
+)
+from app.services.help_center_settings import assign_faq_url_paths
+
+logger = get_logger(__name__)
+
+DEFAULT_ARTICLE_CATEGORY = "Imported"
+
+# Link targets that can never be article pages.
+_SKIP_EXTENSIONS = (
+    ".pdf", ".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".zip", ".mp4",
+    ".css", ".js", ".xml", ".ico", ".woff", ".woff2", ".txt", ".json",
+)
+
+# In-article chrome that shouldn't survive into the imported Markdown.
+# Word-boundary anchored: matches "docs-toc"/"breadcrumbs" but never innocent
+# substrings like "stock-levels" or "autocomplete".
+_TOC_RE = re.compile(r"\b(toc|table-of-contents|on-this-page|breadcrumbs?)\b", re.IGNORECASE)
+
+# Help-center chrome text (platform branding / article metadata) to drop.
+# Platform-specific so it never strips legitimate prose like "Powered by our
+# AI engine" — only footer branding for known help-center platforms and the
+# "Last updated on …" article-metadata line.
+_CHROME_TEXT_RE = re.compile(
+    r"^last updated on\b"
+    r"|^(made with|powered by)\b.*\b(chatwoot|intercom|zendesk|help ?scout|gitbook|freshdesk|document360)\b",
+    re.IGNORECASE,
+)
+_CHROME_HREFS = ("chatwoot.com", "chatwoot.help")
+
+# Path markers separating leaf article pages from category/section listings
+# across Chatwoot / Zendesk / Intercom-style help centers. Only /articles/
+# pages hold real content; the listings are followed for their article links.
+_ARTICLE_MARKER = "/articles/"
+_CATEGORY_MARKERS = ("/categories/", "/sections/", "/collections/")
+_HEADING_TAGS = ("h1", "h2", "h3", "h4")
+_LEADING_SYMBOLS_RE = re.compile(r"^[^\w]+", re.UNICODE)
+
+# Homepage section headings that are curated cross-cuts, not real categories —
+# their articles' true category comes from the category listing pages instead.
+_GENERIC_SECTIONS = frozenset({
+    "featured articles", "featured", "popular", "popular articles", "all articles",
+    "browse", "recent articles", "recently updated", "top articles",
+})
+
+# Internal scheme marking images collected during conversion, replaced with the
+# re-hosted URL afterwards (conversion is sync; storage is async). The trailing
+# ".img" delimits the index so "…//1.img" is never a prefix of "…//10.img".
+_IMG_PLACEHOLDER = "rp-pending-image://{index}.img"
+
+
+@dataclass
+class Article:
+    url: str
+    title: str
+    markdown: str
+    category_hint: str
+    # index -> (bytes, content_type) awaiting re-host.
+    pending_images: Dict[int, Tuple[bytes, str]] = field(default_factory=dict)
+
+
+class _ArticleConverter(MarkdownConverter):
+    """HTML→Markdown converter that absolutizes links and swaps images for
+    placeholders after downloading them (SSRF-guarded, size/type-capped)."""
+
+    def __init__(self, base_url: str, client: httpx.Client, **options):
+        super().__init__(**options)
+        self._base_url = base_url
+        self._client = client
+        self.images: Dict[int, Tuple[bytes, str]] = {}
+
+    def convert_a(self, el, text, parent_tags):
+        # Absolutize every link; same-site source links are remapped to the
+        # destination help center in a second pass (run_article_import_job),
+        # once every article's new FAQ slug is known.
+        href = el.get("href")
+        if href:
+            el["href"] = urljoin(self._base_url, href)
+        return super().convert_a(el, text, parent_tags)
+
+    def convert_img(self, el, text, parent_tags):
+        alt = (el.get("alt") or "").strip()
+        src = (el.get("src") or "").strip()
+        if not src or src.startswith("data:"):
+            return alt  # inline/base64 images are stripped by the renderer anyway
+        if len(self.images) >= settings.FAQ_ARTICLE_IMPORT_MAX_IMAGES:
+            return alt
+        image = self._download(urljoin(self._base_url, src))
+        if not image:
+            return alt
+        index = len(self.images)
+        self.images[index] = image
+        return f"![{alt}]({_IMG_PLACEHOLDER.format(index=index)})"
+
+    def _download(self, url: str) -> Optional[Tuple[bytes, str]]:
+        try:
+            response = safe_get(self._client, url)
+            response.raise_for_status()
+            content_type = (response.headers.get("content-type") or "").split(";")[0].strip()
+            if content_type not in FAQ_IMAGE_TYPES or len(response.content) > MAX_FAQ_IMAGE_BYTES:
+                return None
+            return response.content, content_type
+        except Exception as e:
+            logger.warning(f"Article image skipped ({url}): {e}")
+            return None
+
+
+def _fetch_soup(client: httpx.Client, url: str) -> Tuple[Optional[BeautifulSoup], str]:
+    """Static SSRF-guarded fetch → (soup, final hop-validated URL)."""
+    try:
+        response = safe_get(client, url)
+        response.raise_for_status()
+        return BeautifulSoup(response.text, "html.parser"), str(response.url)
+    except httpx.HTTPError as e:
+        logger.warning(f"Article fetch failed for {url}: {e}")
+        return None, url
+
+
+def _fetch_index_soup(client: httpx.Client, url: str) -> Tuple[Optional[BeautifulSoup], str]:
+    """Index fetch with the headless-browser fallback for JS-rendered indexes
+    (article pages themselves are static-only — one browser per crawl, not 30)."""
+    soup, final_url = _fetch_soup(client, url)
+    if soup and len(soup.get_text(" ", strip=True)) >= MIN_STATIC_TEXT_CHARS:
+        return soup, final_url
+    try:
+        from app.knowledge.crawl4ai_fallback import get_crawl4ai_fallback
+
+        fallback = get_crawl4ai_fallback(timeout=settings.FAQ_IMPORT_FETCH_TIMEOUT)
+        if fallback.is_available:
+            html, browser_soup, _ = fallback.fetch_with_browser(final_url)
+            if browser_soup is not None:
+                return browser_soup, final_url
+            if isinstance(html, str) and html:
+                return BeautifulSoup(html, "html.parser"), final_url
+    except Exception as e:
+        logger.warning(f"Article index browser fetch failed for {url}: {e}")
+    return soup, final_url
+
+
+def _clean_category(text: str) -> str:
+    """Strip a leading emoji/symbol (Chatwoot section headings are '📞 Help
+    and support') and cap length."""
+    cleaned = _LEADING_SYMBOLS_RE.sub("", (text or "").strip()).strip()
+    return cleaned[:100] or DEFAULT_ARTICLE_CATEGORY
+
+
+def _same_site_href(anchor: Tag, base_url: str, root_domain: str) -> Optional[str]:
+    """Absolute same-site content URL for an anchor, or None if it's off-site,
+    a fragment/query, a bare root, or a binary asset."""
+    href = anchor.get("href")
+    if not href:
+        return None
+    url = urldefrag(urljoin(base_url, href))[0].rstrip("/")
+    parsed = urlparse(url)
+    if parsed.scheme not in ("http", "https") or parsed.query:
+        return None
+    if parsed.path in ("", "/"):
+        return None
+    if _registrable_domain(parsed.hostname or "") != root_domain:
+        return None
+    if parsed.path.lower().endswith(_SKIP_EXTENSIONS):
+        return None
+    return url
+
+
+def _is_article_url(url: str) -> bool:
+    return _ARTICLE_MARKER in urlparse(url).path.lower()
+
+
+def _is_category_url(url: str) -> bool:
+    path = urlparse(url).path.lower()
+    return any(marker in path for marker in _CATEGORY_MARKERS)
+
+
+def _sectioned_article_links(node: Tag, base_url: str, root_domain: str) -> List[Tuple[str, Optional[str]]]:
+    """Article links in document order, each tagged with the nearest preceding
+    section heading (a help-center homepage's category cards). Headings nested
+    inside a link are article titles, not sections, so they don't reset it."""
+    results: List[Tuple[str, Optional[str]]] = []
+    current: Optional[str] = None
+    for el in node.find_all([*_HEADING_TAGS, "a"]):
+        if el.name in _HEADING_TAGS:
+            if not el.find_parent("a"):
+                cat = _clean_category(el.get_text(" ", strip=True))
+                # Curated cross-cut sections ("Featured Articles") aren't real
+                # categories — leave those articles for the category pages.
+                current = None if cat.lower() in _GENERIC_SECTIONS else cat
+            continue
+        url = _same_site_href(el, base_url, root_domain)
+        if url and _is_article_url(url):
+            results.append((url, current))
+    return results
+
+
+def discover_article_links(
+    client: httpx.Client, index_url: str, limit: int
+) -> List[Tuple[str, Optional[str]]]:
+    """Discover (article_url, category) pairs from a help-center index.
+
+    Standard help centers (Chatwoot/Zendesk/Intercom) nest articles under
+    /articles/ and group them into /categories/ (or /sections//collections/).
+    We follow the category listing pages FIRST for the complete per-category
+    list (homepages truncate each section to a few articles), so hitting the
+    page cap yields whole categories rather than a thin slice of every one.
+    Homepage section links then fill any remainder (e.g. featured-only
+    articles). A non-standard/flat page with no /articles/ links falls back to
+    importing every same-site content link.
+    """
+    soup, final_url = _fetch_index_soup(client, index_url)
+    if soup is None:
+        raise ValueError("Could not read that page.")
+    root_domain = _registrable_domain(urlparse(final_url).hostname or "")
+    index_clean = urldefrag(final_url)[0].rstrip("/")
+    main = select_main_node(soup) or soup
+
+    ordered: List[str] = []
+    category_of: Dict[str, Optional[str]] = {}
+
+    def record(url: str, category: Optional[str]) -> None:
+        if url == index_clean:
+            return
+        if url not in category_of:
+            category_of[url] = category
+            ordered.append(url)
+        elif category and not category_of[url]:
+            # Upgrade a still-uncategorised article (seen first under a generic
+            # homepage section) with the category page's real name.
+            category_of[url] = category
+
+    # 1. Follow category/section listing pages first — the complete per-category
+    #    lists (a homepage shows only ~5 of a category's articles).
+    category_links: List[str] = []
+    seen_cat = set()
+    for anchor in main.find_all("a", href=True):
+        url = _same_site_href(anchor, final_url, root_domain)
+        if url and _is_category_url(url) and url not in seen_cat:
+            seen_cat.add(url)
+            category_links.append(url)
+    for cat_url in category_links[: settings.FAQ_ARTICLE_IMPORT_MAX_CATEGORIES]:
+        if len(ordered) >= limit:
+            break
+        cat_soup, cat_final = _fetch_soup(client, cat_url)
+        if cat_soup is None:
+            continue
+        og_title, doc_title = _head_titles(cat_soup)
+        cat_node = select_main_node(cat_soup) or cat_soup
+        cat_name = _clean_category(_page_title(og_title, doc_title, cat_node, cat_soup))
+        for anchor in cat_node.find_all("a", href=True):
+            url = _same_site_href(anchor, cat_final, root_domain)
+            if url and _is_article_url(url):
+                record(url, cat_name)
+
+    # 2. Homepage section links — fills any remainder (featured-only articles,
+    #    or the index itself being a single category/flat listing page).
+    for url, category in _sectioned_article_links(main, final_url, root_domain):
+        record(url, category)
+
+    # 3. Fallback: no standard article URLs — import every same-site link,
+    #    resolving each article's category from its own breadcrumb/URL.
+    if not ordered:
+        for anchor in main.find_all("a", href=True):
+            url = _same_site_href(anchor, final_url, root_domain)
+            if url:
+                record(url, None)
+
+    ordered = ordered[:limit]
+    if not ordered:
+        raise ValueError("No linked article pages found on that page.")
+    return [(url, category_of[url]) for url in ordered]
+
+
+def _head_titles(soup: BeautifulSoup) -> Tuple[str, str]:
+    """(og:title, <title>) — must run BEFORE select_main_node, which strips
+    <head> from the soup."""
+    og = soup.find("meta", property="og:title")
+    og_title = og["content"].strip() if og and og.get("content", "").strip() else ""
+    doc_title = ""
+    if soup.title and soup.title.get_text(strip=True):
+        # Drop the " | Site Name" tail commonly appended to <title>.
+        doc_title = soup.title.get_text(strip=True).split("|")[0].strip()
+    return og_title, doc_title
+
+
+def _page_title(og_title: str, doc_title: str, node: Tag, soup: BeautifulSoup) -> str:
+    if og_title:
+        return og_title
+    h1 = node.find("h1") or soup.find("h1")
+    if h1 and h1.get_text(strip=True):
+        return h1.get_text(" ", strip=True)
+    return doc_title
+
+
+def _category_hint(soup: BeautifulSoup, url: str) -> str:
+    """Breadcrumb (second-to-last crumb = the article's section) or a URL path
+    segment, else the generic import bucket."""
+    crumbs = soup.select('[aria-label*="readcrumb"] a, nav.breadcrumb a, .breadcrumbs a, .breadcrumb a')
+    texts = [c.get_text(" ", strip=True) for c in crumbs if c.get_text(strip=True)]
+    if len(texts) >= 2:
+        # Last crumb is the page itself; the one before is its section.
+        return texts[-2][:100]
+    segments = [s for s in urlparse(url).path.split("/") if s]
+    if len(segments) >= 2:
+        return segments[-2].replace("-", " ").replace("_", " ").title()[:100]
+    return DEFAULT_ARTICLE_CATEGORY
+
+
+def _strip_chrome(node: Tag) -> None:
+    """Remove in-article chrome that shouldn't become FAQ content: nav bars,
+    TOCs, footers, help-center platform branding ('Made with Chatwoot') and
+    'Last updated' metadata lines. Best-effort — a detached element mid-loop
+    just skips. (Breadcrumb "Home › Category" nav links are dropped later, in
+    _remap_source_links, since they often share a container with the title.)"""
+    for chrome in node.find_all("nav") + node.find_all("footer"):
+        chrome.decompose()
+    for chrome in node.find_all(class_=_TOC_RE) + node.find_all(id=_TOC_RE):
+        try:
+            chrome.decompose()
+        except Exception:
+            pass
+    # Platform-branding link ("Made with Chatwoot" logo): remove only a SMALL
+    # enclosing block (the footer badge), never a large content div that merely
+    # links out to the platform.
+    for anchor in list(node.find_all("a", href=True)):
+        try:
+            href = (anchor.get("href") or "").lower()
+            if any(brand in href for brand in _CHROME_HREFS):
+                container = anchor.find_parent(["div", "footer", "section"])
+                target = container if container and len(container.get_text(" ", strip=True)) <= 40 else anchor
+                target.decompose()
+        except Exception:
+            pass
+    for el in list(node.find_all(["p", "div", "span", "small", "time"])):
+        try:
+            text = el.get_text(" ", strip=True)
+            if text and len(text) <= 60 and _CHROME_TEXT_RE.search(text):
+                el.decompose()
+        except Exception:
+            pass
+
+
+def fetch_article(
+    client: httpx.Client, url: str, category_override: Optional[str] = None,
+) -> Optional[Article]:
+    """One article page → title + Markdown body (+ images pending re-host).
+    category_override (from the category listing that linked here) wins over the
+    per-page breadcrumb/URL guess. None when the page can't be read or is empty."""
+    soup, final_url = _fetch_soup(client, url)
+    if soup is None:
+        return None
+    og_title, doc_title = _head_titles(soup)  # before <head> is stripped below
+    node = select_main_node(soup)
+    if node is None:
+        return None
+    title = _page_title(og_title, doc_title, node, soup)
+    if not title:
+        return None
+    category = category_override or _category_hint(soup, final_url)
+    # The page's own h1 would duplicate the FAQ question.
+    first_h1 = node.find("h1")
+    if first_h1:
+        first_h1.extract()
+    _strip_chrome(node)
+    converter = _ArticleConverter(
+        base_url=final_url, client=client, heading_style="ATX", bullets="-"
+    )
+    markdown = converter.convert_soup(node).strip()
+    if not markdown:
+        return None
+    return Article(
+        url=final_url,
+        title=title[:MAX_QUESTION_LENGTH],
+        markdown=markdown,
+        category_hint=category,
+        pending_images=converter.images,
+    )
+
+
+async def _rehost_images(article: Article) -> str:
+    markdown = article.markdown
+    for index, (content, content_type) in article.pending_images.items():
+        placeholder = _IMG_PLACEHOLDER.format(index=index)
+        try:
+            markdown = markdown.replace(placeholder, await store_article_image(content, content_type))
+        except Exception as e:
+            logger.warning(f"Article image re-host failed ({article.url}): {e}")
+            markdown = markdown.replace(f"({placeholder})", "()")
+    return markdown
+
+
+# Markdown links, excluding image embeds (`![alt](src)`). Group 1 = label,
+# group 2 = the http(s) URL; an optional markdown title (`[x](url "title")`, which
+# markdownify emits for anchors with a title attribute) is tolerated and dropped.
+_MD_LINK_RE = re.compile(r'(?<!!)\[([^\]]*)\]\((https?://[^)\s]+)(?:\s+"[^"]*")?\)')
+
+
+def _article_key(path: str) -> Optional[str]:
+    """Stable identifier for an article from its URL path — the numeric id in
+    `/articles/{id}-{slug}` (falling back to the whole segment). Lets a cross-link
+    resolve to the same article regardless of the slug suffix or a missing slug."""
+    lower = path.lower()
+    idx = lower.find(_ARTICLE_MARKER)
+    if idx == -1:
+        return None
+    segment = path[idx + len(_ARTICLE_MARKER):].split("/")[0].split("?")[0]
+    numeric = re.match(r"\d+", segment)
+    return numeric.group(0) if numeric else (segment.lower() or None)
+
+
+# Leading breadcrumb residue after "Home › Account" links are dropped: chevron /
+# middot separators (never markdown-structural chars like - > | so hr/blockquote/
+# tables are safe).
+_BREADCRUMB_RESIDUE_RE = re.compile(r"^[\s›»·]+")
+
+
+def _remap_source_links(markdown: str, source_hosts: set, key_to_slug: dict) -> str:
+    """Rewrite links to the SOURCE HELP CENTER as ROOT-RELATIVE help-center paths:
+      - article link → /a/{new-slug} (or /  when the article wasn't imported),
+        kept as a link;
+      - home/category link (non-article) → DROPPED — these are breadcrumb/nav
+        chrome ("Home › Account") that shouldn't appear in article bodies.
+
+    Links are stored WITHOUT an origin or /help/{slug} prefix so they survive a
+    move to a custom domain or subdomain — the serving prefix is added at render
+    time (help_center_content.render_article_html). Matching is by EXACT host
+    against the set of hosts the articles were actually fetched from (handles
+    redirects and main-site→help-subdomain splits), so links to the main product
+    site and every other external link are left untouched."""
+    def repl(match):
+        label, url = match.group(1), match.group(2)
+        parsed = urlparse(url)
+        if (parsed.hostname or "").lower() not in source_hosts:
+            return match.group(0)  # off-site (incl. the main product site) — keep
+        key = _article_key(parsed.path)
+        if key is None:
+            return ""  # home / category breadcrumb link — drop entirely
+        slug = key_to_slug.get(key)
+        return f"[{label}](/a/{slug})" if slug else f"[{label}](/)"
+
+    result = _MD_LINK_RE.sub(repl, markdown)
+    # Tidy up after dropped breadcrumbs: strip leading separator residue and
+    # collapse the blank lines those drops leave behind.
+    result = _BREADCRUMB_RESIDUE_RE.sub("", result)
+    return re.sub(r"\n{3,}", "\n\n", result)
+
+
+async def run_article_import_job(db, job: FAQGenerationJob) -> int:
+    """Execute an IMPORT_ARTICLES job. Returns FAQs created. No LLM involved —
+    articles are imported verbatim as Markdown drafts, then a second pass remaps
+    intra-help-center links to the org's new help center."""
+    job_repo = FAQGenerationJobRepository(db)
+    job_repo.update_progress(job.id, stage=FAQJobStage.ANALYZING_SOURCES, progress_percentage=5.0)
+
+    client = httpx.Client(timeout=settings.FAQ_IMPORT_FETCH_TIMEOUT, headers=_FETCH_HEADERS)
+    try:
+        links = await asyncio.to_thread(
+            discover_article_links, client, job.source_url, settings.FAQ_ARTICLE_IMPORT_MAX_PAGES
+        )
+        job_repo.update_progress(job.id, stage=FAQJobStage.EXTRACTING, progress_percentage=10.0)
+
+        existing = {normalize_question(q) for q in FAQRepository(db).get_existing_questions(job.organization_id)}
+        categories = CategoryMerger(db, job.organization_id)
+        source_label = f"Imported from {urlparse(job.source_url).netloc}"
+
+        rows: List[FAQ] = []
+        row_sources: List[str] = []  # each row's source article URL, for link remapping
+        for index, (link, category) in enumerate(links):
+            article = await asyncio.to_thread(fetch_article, client, link, category)
+            job_repo.update_progress(
+                job.id, progress_percentage=10.0 + 78.0 * (index + 1) / len(links)
+            )
+            if not article:
+                continue
+            key = normalize_question(article.title)
+            if not key or key in existing:
+                continue
+            existing.add(key)
+            answer = (await _rehost_images(article))[:MAX_ANSWER_LENGTH]
+            rows.append(
+                FAQ(
+                    organization_id=job.organization_id,
+                    question=article.title,
+                    answer=answer,
+                    category=categories.merge(article.category_hint),
+                    status=FAQStatus.DRAFT,
+                    knowledge_id=None,
+                    source_label=source_label,
+                    # Provenance on every import, whether or not the original
+                    # URLs are being preserved — so a later "apply preserved
+                    # paths" pass is a pure data migration.
+                    source_url=article.url[:FAQ_SOURCE_URL_MAX_LENGTH],
+                    generation_job_id=job.id,
+                    created_by=job.user_id,
+                )
+            )
+            row_sources.append(article.url)
+    finally:
+        client.close()
+
+    if not rows:
+        return 0
+    if job.preserve_source_urls:
+        # Before the insert, so the per-org unique index can never reject the
+        # batch. Rows that can't keep their path just fall back to /a/{slug} —
+        # one unusable URL must not cost the org the whole import.
+        applied = assign_faq_url_paths(db, rows, row_sources)
+        logger.info(f"Article import job {job.id}: preserved {applied}/{len(rows)} original URLs")
+    created = insert_draft_rows(db, job, rows)  # assigns each row a unique slug in place
+    _remap_imported_links(db, rows, row_sources)
+    return created
+
+
+def _remap_imported_links(db, rows, row_sources) -> None:
+    """Second pass: rewrite intra-help-center links in the just-imported answers to
+    root-relative help-center paths (article → /a/{slug}, else → home)."""
+    # Match against the hosts the articles were ACTUALLY fetched from (post-redirect,
+    # possibly a help subdomain) — NOT the typed index URL, which may redirect or be
+    # main-site-hosted. Using the real article hosts fixes redirect/subdomain splits
+    # without risking a match on main-site content links.
+    source_hosts = {(urlparse(src).hostname or "").lower() for src in row_sources}
+    source_hosts.discard("")
+    key_to_slug = {}
+    for src, faq in zip(row_sources, rows):
+        key = _article_key(urlparse(src).path)
+        if key:
+            key_to_slug[key] = faq.slug
+
+    dirty = False
+    for faq in rows:
+        remapped = _remap_source_links(faq.answer, source_hosts, key_to_slug)
+        if remapped != faq.answer:
+            faq.answer = remapped[:MAX_ANSWER_LENGTH]
+            dirty = True
+    if dirty:
+        db.commit()

backend/app/services/faq_generation.py

@@ -0,0 +1,431 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+FAQ generation pipeline: read the org's stored knowledge text (no re-crawl),
+extract grounded FAQs with the org's configured model, dedup against existing
+questions and insert drafts for review. Additive only — regeneration never
+edits or deletes existing FAQs. The drafting loop and dedup/category helpers
+here are shared with the import service (faq_import.py).
+"""
+
+import re
+from collections import deque
+from difflib import SequenceMatcher
+from typing import Awaitable, Callable, List, Optional, Tuple, TypeVar
+
+from fastapi import HTTPException
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+from app.agents.faq_generator import FAQGeneratorAgent, GeneratedFAQ
+from app.core.config import settings
+from app.core.logger import get_logger
+from app.core.security import decrypt_api_key
+from app.knowledge.page_editor import NATURAL_ID_ORDER, PAGE_ID_EXPR
+from app.models.faq import FAQ, DEFAULT_FAQ_CATEGORY, FAQStatus
+from app.models.faq_generation_job import FAQGenerationJob, FAQJobStage, FAQJobType
+from app.models.knowledge import Knowledge
+from app.repositories.ai_config import AIConfigRepository
+from app.repositories.faq import FAQRepository
+from app.repositories.faq_generation_job import FAQGenerationJobRepository
+
+logger = get_logger(__name__)
+
+# Dedup lists injected into prompts are a small recent window for intra-run
+# steering only — the in-Python DedupState below (exact + fuzzy) is the real
+# guard covering every existing question, so the prompt stays cheap.
+MAX_PROMPT_QUESTIONS = 30
+
+# Fuzzy-dedup tuning: token-set Jaccard on content words; borderline scores are
+# confirmed with difflib on the normalized strings. Very short questions skip
+# fuzzy matching entirely (too collision-prone).
+FUZZY_JACCARD_DUP = 0.8
+FUZZY_JACCARD_MAYBE = 0.65
+FUZZY_SEQUENCE_DUP = 0.87
+FUZZY_MIN_CONTENT_TOKENS = 3
+
+# Tiny inline stopword set — enough to keep filler words from inflating
+# question similarity, without an NLP dependency.
+_STOPWORDS = frozenset(
+    "a an and are can could do does for from how i in is it my of on or the "
+    "there to we what when where which who why will with you your".split()
+)
+
+
+_PUNCT_RE = re.compile(r"[^\w\s]", re.UNICODE)
+
+BatchT = TypeVar("BatchT")
+
+
+def normalize_question(question: str) -> str:
+    """Casefold + strip punctuation/whitespace, so trivial rephrasings of the
+    same question dedup against each other."""
+    return " ".join(_PUNCT_RE.sub(" ", question.casefold()).split())
+
+
+class NoAIConfigError(Exception):
+    """The org has no active AI configuration to generate with."""
+
+
+def build_generator(db: Session, organization_id, job: Optional[FAQGenerationJob] = None) -> FAQGeneratorAgent:
+    """The org's configured extraction agent. When a job is given, every actual
+    provider call (retries and the plain-JSON fallback's second call included)
+    increments the job's llm_calls — the metering source of truth."""
+    config = AIConfigRepository(db).get_active_config(organization_id)
+    if not config:
+        raise NoAIConfigError(
+            "No active AI configuration. Set up an AI model in AI Configuration first."
+        )
+    generator = FAQGeneratorAgent(
+        api_key=decrypt_api_key(config.encrypted_api_key),
+        model_name=config.model_name,
+        model_type=config.model_type.value if hasattr(config.model_type, "value") else str(config.model_type),
+        base_url=(config.settings or {}).get('base_url'),
+    )
+    if job is not None:
+        job_repo = FAQGenerationJobRepository(db)
+        generator.on_llm_call = lambda: job_repo.increment_llm_calls(job.id)
+    return generator
+
+
+def load_source_pages(db: Session, knowledge: Knowledge, max_chars: Optional[int] = None) -> List[str]:
+    """One string of aggregated text per crawled page/sub-page of a source,
+    read straight from the org's vector table (content is stored alongside the
+    embeddings — no re-crawl needed). Pages kept in crawl order when the cap
+    truncates; chunk text within a page is naturally ordered. max_chars caps
+    each page and defaults to the model-agnostic floor."""
+    if not knowledge.schema or not knowledge.table_name:
+        # Source rows without a vector table (failed/partial ingestion).
+        return []
+    # Chunks written by app.knowledge.chunking (`::n` ids) are verbatim slices
+    # and join back with nothing between them; legacy agno `_n` chunks were
+    # cut on whitespace they dropped, so those keep a newline.
+    query = text(
+        f"SELECT {PAGE_ID_EXPR} AS page_id, "
+        "string_agg(content, CASE WHEN id ~ '::[0-9]+$' THEN '' ELSE E'\n' END "
+        f"ORDER BY {NATURAL_ID_ORDER}) AS body "
+        f'FROM {knowledge.schema}."{knowledge.table_name}" '
+        "WHERE name = :source GROUP BY 1 ORDER BY min(created_at) LIMIT :max_pages"
+    )
+    rows = db.execute(
+        query, {"source": knowledge.source, "max_pages": settings.FAQ_MAX_PAGES_PER_SOURCE}
+    ).fetchall()
+    max_chars = max_chars or settings.FAQ_MAX_BATCH_CHARS
+    return [row.body[:max_chars] for row in rows if row.body and row.body.strip()]
+
+
+def pack_batches(items: List[str], max_chars: Optional[int] = None, sep: str = "\n\n---\n\n") -> List[str]:
+    """Greedy-pack whole items (pages, lines) into batches of ~max_chars.
+    Oversized single items are truncated to max_chars, never split across
+    batches. Shared by generation (page items) and import (line items)."""
+    max_chars = max_chars or settings.FAQ_MAX_BATCH_CHARS
+    batches: List[str] = []
+    current: List[str] = []
+    size = 0
+    for item in items:
+        item = item[:max_chars]
+        if current and size + len(item) + len(sep) > max_chars:
+            batches.append(sep.join(current))
+            current, size = [], 0
+        current.append(item)
+        size += len(item) + len(sep)
+    if current:
+        batches.append(sep.join(current))
+    return batches
+
+
+def _content_tokens(normalized: str) -> frozenset:
+    return frozenset(t for t in normalized.split() if t not in _STOPWORDS)
+
+
+class DedupState:
+    """Tracks every question seen (existing + accepted this run).
+
+    Two layers: an exact normalized-set match, plus fuzzy similarity (token-set
+    Jaccard via an inverted index, difflib confirmation on borderline scores)
+    so rephrasings are caught without shipping the whole question list to the
+    LLM. A bounded recent-questions window is still exposed for prompts."""
+
+    def __init__(self, existing_questions: List[str]):
+        self.seen: set = set()
+        self._token_index: dict = {}  # content token -> set of seen keys
+        self._key_tokens: dict = {}  # seen key -> its content-token frozenset
+        self._recent = deque(existing_questions, maxlen=MAX_PROMPT_QUESTIONS)
+        for q in existing_questions:
+            self._remember(normalize_question(q))
+
+    def _remember(self, key: str) -> None:
+        if not key or key in self.seen:
+            return
+        self.seen.add(key)
+        tokens = _content_tokens(key)
+        self._key_tokens[key] = tokens
+        for token in tokens:
+            self._token_index.setdefault(token, set()).add(key)
+
+    def is_similar(self, key: str) -> bool:
+        """True when key fuzzily matches a seen question. Exact hits are the
+        caller's job (cheap set test); this covers rephrasings."""
+        tokens = _content_tokens(key)
+        if len(tokens) < FUZZY_MIN_CONTENT_TOKENS:
+            return False
+        candidates: set = set()
+        for token in tokens:
+            candidates |= self._token_index.get(token, set())
+        for candidate in candidates:
+            other = self._key_tokens[candidate]
+            union = len(tokens | other)
+            if not union:
+                continue
+            jaccard = len(tokens & other) / union
+            if jaccard >= FUZZY_JACCARD_DUP:
+                return True
+            if jaccard >= FUZZY_JACCARD_MAYBE and (
+                SequenceMatcher(None, key, candidate).ratio() >= FUZZY_SEQUENCE_DUP
+            ):
+                return True
+        return False
+
+    def accept_new(self, faqs: List[GeneratedFAQ]) -> List[GeneratedFAQ]:
+        accepted = []
+        for faq in faqs:
+            key = normalize_question(faq.question)
+            if not key or key in self.seen or self.is_similar(key):
+                continue
+            self._remember(key)
+            self._recent.append(faq.question)
+            accepted.append(faq)
+        return accepted
+
+    @property
+    def for_prompt(self) -> List[str]:
+        return list(self._recent)
+
+
+class CategoryMerger:
+    """Case-insensitive merge of model-emitted categories into the org's
+    existing spellings."""
+
+    def __init__(self, db: Session, organization_id):
+        self._known = {c.casefold(): c for c in FAQRepository(db).get_categories(organization_id)}
+
+    def merge(self, category: str) -> str:
+        cleaned = (category or "").strip() or DEFAULT_FAQ_CATEGORY
+        return self._known.setdefault(cleaned.casefold(), cleaned)
+
+    @property
+    def names(self) -> List[str]:
+        return list(self._known.values())
+
+
+async def draft_batches(
+    db: Session,
+    job: FAQGenerationJob,
+    batches: List[BatchT],
+    extract: Callable[[BatchT, DedupState], Awaitable[List[GeneratedFAQ]]],
+    retries_per_batch: int = 1,
+) -> List[Tuple[BatchT, GeneratedFAQ]]:
+    """Shared DRAFTING stage: run `extract` per batch with retry, dedup as we
+    go, report progress 30→90%. Raises only when every batch failed."""
+    job_repo = FAQGenerationJobRepository(db)
+    job_repo.update_progress(job.id, stage=FAQJobStage.DRAFTING, progress_percentage=30.0)
+    dedup = DedupState(FAQRepository(db).get_existing_questions(job.organization_id))
+
+    accepted: List[Tuple[BatchT, GeneratedFAQ]] = []
+    failures = 0
+    last_error: Optional[Exception] = None
+    for index, batch in enumerate(batches):
+        faqs = None
+        for attempt in range(1 + retries_per_batch):
+            try:
+                faqs = await extract(batch, dedup)
+                break
+            except Exception as e:
+                last_error = e
+                logger.warning(f"FAQ batch {index + 1}/{len(batches)} attempt {attempt + 1} failed: {e}")
+        if faqs is None:
+            failures += 1
+        else:
+            accepted.extend((batch, faq) for faq in dedup.accept_new(faqs))
+        job_repo.update_progress(
+            job.id, progress_percentage=30.0 + 60.0 * (index + 1) / len(batches)
+        )
+    if batches and failures == len(batches):
+        # Surface the underlying provider error (e.g. an unsupported/decommissioned
+        # model) instead of a generic message, so the UI can tell the user what to fix.
+        detail = f" — {last_error}" if last_error else ""
+        raise RuntimeError(f"The AI model could not generate FAQs from this content{detail}")
+    return accepted
+
+
+def insert_draft_rows(db: Session, job: FAQGenerationJob, rows: List[FAQ]) -> int:
+    """Shared GROUPING stage tail: bulk-insert the drafted rows."""
+    FAQGenerationJobRepository(db).update_progress(
+        job.id, stage=FAQJobStage.GROUPING, progress_percentage=92.0
+    )
+    if rows:
+        FAQRepository(db).bulk_create(rows)
+    return len(rows)
+
+
+async def run_generation_job(db: Session, job: FAQGenerationJob) -> int:
+    """Execute a GENERATE_ALL / GENERATE_SOURCE job. Returns FAQs created.
+    Status transitions and notifications are the worker's responsibility."""
+    job_repo = FAQGenerationJobRepository(db)
+
+    # Stage 1: resolve sources + model config.
+    job_repo.update_progress(job.id, stage=FAQJobStage.ANALYZING_SOURCES, progress_percentage=5.0)
+    generator = build_generator(db, job.organization_id, job=job)
+
+    knowledge_query = db.query(Knowledge).filter(Knowledge.organization_id == job.organization_id)
+    if job.job_type == FAQJobType.GENERATE_SOURCE.value:
+        knowledge_query = knowledge_query.filter(Knowledge.id == job.knowledge_id)
+    elif job.knowledge_ids:
+        # Explicit selection overrides the skip-already-generated default.
+        knowledge_query = knowledge_query.filter(Knowledge.id.in_(job.knowledge_ids))
+    sources = knowledge_query.all()
+    if not sources:
+        raise ValueError("No knowledge sources to generate from. Add knowledge first.")
+
+    if job.job_type == FAQJobType.GENERATE_ALL.value and not job.knowledge_ids:
+        # Regenerate touches only sources that haven't produced FAQs yet —
+        # deleting a source's FAQs makes it eligible again.
+        generated = FAQRepository(db).knowledge_ids_with_faqs(job.organization_id)
+        sources = [s for s in sources if s.id not in generated]
+        if not sources:
+            raise ValueError(
+                "All knowledge sources already have FAQs. "
+                "Delete a source's FAQs to regenerate it."
+            )
+
+    # Stage 2: read stored page text per source. A broken source (missing
+    # vector table etc.) is skipped, not fatal — unless nothing is readable.
+    job_repo.update_progress(job.id, stage=FAQJobStage.EXTRACTING, progress_percentage=10.0)
+    source_batches: List[Tuple[Knowledge, str]] = []
+    for source in sources:
+        try:
+            # Batch size follows the org's model context window — big-context
+            # models get fewer, larger LLM calls.
+            pages = load_source_pages(db, source, max_chars=generator.batch_chars)
+        except Exception as e:
+            logger.warning(f"Skipping unreadable knowledge source {source.id} ({source.source}): {e}")
+            db.rollback()
+            continue
+        for batch in pack_batches(pages, max_chars=generator.batch_chars):
+            source_batches.append((source, batch))
+    if not source_batches:
+        raise ValueError("The selected knowledge sources contain no readable content.")
+
+    # Stage 3: draft per batch (shared loop).
+    categories = CategoryMerger(db, job.organization_id)
+
+    async def extract(batch: Tuple[Knowledge, str], dedup: DedupState) -> List[GeneratedFAQ]:
+        return await generator.generate_from_text(
+            batch[1],
+            existing_questions=dedup.for_prompt,
+            existing_categories=categories.names,
+        )
+
+    accepted = await draft_batches(db, job, source_batches, extract)
+
+    # Stage 4: group + insert drafts.
+    rows = [
+        FAQ(
+            organization_id=job.organization_id,
+            question=faq.question,
+            answer=faq.answer,
+            category=categories.merge(faq.category),
+            status=FAQStatus.DRAFT,
+            knowledge_id=source.id,
+            source_label=source.source,
+            generation_job_id=job.id,
+            created_by=job.user_id,
+        )
+        for (source, _batch_text), faq in accepted
+    ]
+    return insert_draft_rows(db, job, rows)
+
+
+def maybe_enqueue_auto_faq_job(db: Session, queue_item) -> Optional[FAQGenerationJob]:
+    """Auto-generate draft FAQs when a new knowledge source finishes processing.
+
+    Only fires for orgs that have adopted the feature (a help-center settings
+    row or any FAQ exists), have auto-generate on, are allowed by their plan
+    (cloud), and don't already have an active job for this source. Never raises
+    and always leaves the shared session usable — a hook failure must not fail
+    knowledge processing.
+    """
+    try:
+        from app.repositories.help_center import HelpCenterRepository
+        from app.services.help_center_access import help_center_allowed
+
+        org_id = queue_item.organization_id
+        settings_row = HelpCenterRepository(db).get_by_org(org_id)
+        faq_repo = FAQRepository(db)
+        job_repo_gate = FAQGenerationJobRepository(db)
+        # Adoption gate: auto-generation only after the org explicitly ran a
+        # generate/import (or added an FAQ). A settings row alone is NOT
+        # opt-in — merely opening the admin page creates one.
+        if not job_repo_gate.has_user_initiated_job(org_id) and not faq_repo.exists_for_org(org_id):
+            return None
+        if settings_row and not settings_row.auto_generate:
+            return None
+        if not help_center_allowed(db, org_id):
+            return None
+        knowledge = (
+            db.query(Knowledge)
+            .filter(Knowledge.organization_id == org_id, Knowledge.source == queue_item.source)
+            .order_by(Knowledge.id.desc())  # deterministic: newest row wins on duplicate sources
+            .first()
+        )
+        if not knowledge:
+            return None
+        job_repo = FAQGenerationJobRepository(db)
+        if job_repo.get_active_for_org(org_id, FAQJobType.GENERATE_SOURCE, knowledge_id=knowledge.id):
+            return None
+        from app.services.faq_usage import (
+            count_source_pages,
+            ensure_generation_budget,
+            generation_is_metered,
+            PAGES_PER_CALL,
+        )
+        from math import ceil
+
+        try:
+            pages = count_source_pages(db, knowledge)
+            ensure_generation_budget(db, org_id, max(1, ceil((pages or 1) / PAGES_PER_CALL)))
+        except HTTPException:
+            logger.info(f"Auto FAQ generation skipped for org {org_id}: message budget exhausted")
+            return None
+        job = job_repo.create(
+            FAQGenerationJob(
+                organization_id=org_id,
+                user_id=queue_item.user_id,
+                job_type=FAQJobType.GENERATE_SOURCE.value,
+                knowledge_id=knowledge.id,
+                metered=generation_is_metered(db, org_id),
+            )
+        )
+        logger.info(f"Auto-enqueued FAQ generation job {job.id} for knowledge {knowledge.id}")
+        return job
+    except Exception as e:
+        logger.error(f"Auto FAQ enqueue failed (non-fatal): {e}")
+        # A failed flush would otherwise poison the caller's session and turn
+        # an already-successful knowledge run into a FAILED one.
+        try:
+            db.rollback()
+        except Exception:
+            pass
+        return None

backend/app/services/faq_import.py

@@ -0,0 +1,243 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+FAQ page migration: fetch the customer's existing FAQ/help-center page
+(SSRF-guarded) or read an uploaded PDF, extract Q&A pairs with the org's model
+and insert them as drafts. URL mode is single page only — for linked article
+pages use the article importer (faq_article_import.py).
+"""
+
+import asyncio
+from io import BytesIO
+from typing import List, Optional
+from urllib.parse import urlparse
+
+import httpx
+from bs4 import BeautifulSoup
+from sqlalchemy.orm import Session
+
+from app.core.config import settings
+from app.core.logger import get_logger
+from app.knowledge.url_safety import guard_url, safe_get
+from app.models.faq import FAQ, FAQStatus
+from app.models.faq_generation_job import FAQGenerationJob, FAQJobStage
+from app.repositories.faq_generation_job import FAQGenerationJobRepository
+from app.services.faq_generation import (
+    CategoryMerger,
+    DedupState,
+    build_generator,
+    draft_batches,
+    insert_draft_rows,
+    pack_batches,
+)
+
+logger = get_logger(__name__)
+
+# Below this, the static HTML likely lacks the real content (JS-rendered page).
+MIN_STATIC_TEXT_CHARS = 500
+# Block-level elements get blank-line separators so a question heading stays
+# attached to its answer text when batches are split on blocks.
+_BLOCK_TAGS = (
+    "p", "div", "section", "article", "li", "h1", "h2", "h3", "h4", "h5", "h6",
+    "dt", "dd", "summary", "details", "tr",
+)
+_BOILERPLATE_TAGS = ("script", "style", "noscript", "nav", "footer", "header", "form")
+
+_FETCH_HEADERS = {"User-Agent": "ReplyPier-FAQ-Import/1.0 (+https://replypier.site)"}
+
+
+def _soup_to_text(soup: BeautifulSoup) -> str:
+    """Clean text with one BLANK line between block elements, so downstream
+    blank-line batching keeps each Q&A block intact."""
+    for tag in soup.find_all(_BOILERPLATE_TAGS):
+        tag.decompose()
+    for tag in soup.find_all(_BLOCK_TAGS):
+        tag.append("\n\n")
+    lines = [line.strip() for line in soup.get_text("\n").split("\n")]
+    text = "\n".join(lines)
+    # Collapse 3+ newlines to exactly one blank line.
+    while "\n\n\n" in text:
+        text = text.replace("\n\n\n", "\n\n")
+    return text.strip()
+
+
+def fetch_page_text(url: str) -> str:
+    """Fetch the page and reduce it to clean text. Static fetch first (every
+    redirect hop SSRF-checked by safe_get); crawl4ai's headless browser as a
+    fallback for JS-rendered pages. Blocking — callers run it in a thread."""
+    text = ""
+    final_url = url
+    try:
+        with httpx.Client(timeout=settings.FAQ_IMPORT_FETCH_TIMEOUT, headers=_FETCH_HEADERS) as client:
+            response = safe_get(client, url)
+            response.raise_for_status()
+            # The hop-validated final URL — the browser fallback fetches this,
+            # not the original, so a redirect chain can't smuggle it inward.
+            final_url = str(response.url)
+            text = _soup_to_text(BeautifulSoup(response.text, "html.parser"))
+    except httpx.HTTPError as e:
+        logger.warning(f"FAQ import static fetch failed for {url}: {e}")
+
+    if len(text) < MIN_STATIC_TEXT_CHARS:
+        rendered = _fetch_rendered_text(final_url)
+        if rendered and len(rendered) > len(text):
+            text = rendered
+
+    if not text.strip():
+        raise ValueError("Could not read any text from that page.")
+    return text[: settings.FAQ_IMPORT_MAX_PAGE_CHARS]
+
+
+def _fetch_rendered_text(url: str) -> Optional[str]:
+    """Headless-browser fallback for JS-rendered pages. The URL is re-checked
+    against the SSRF guard, but a browser follows redirects/JS navigation
+    unguarded — same residual exposure as the knowledge crawler's fallback."""
+    try:
+        guard_url(url)
+        from app.knowledge.crawl4ai_fallback import get_crawl4ai_fallback
+
+        fallback = get_crawl4ai_fallback(timeout=settings.FAQ_IMPORT_FETCH_TIMEOUT)
+        if not fallback.is_available:
+            return None
+        _, soup, _ = fallback.fetch_with_browser(url)
+        return _soup_to_text(soup) if soup else None
+    except Exception as e:
+        logger.warning(f"FAQ import browser fetch failed for {url}: {e}")
+        return None
+
+
+def _split_blocks(text: str) -> List[str]:
+    """Blank-line-separated blocks (see _soup_to_text) — the batching unit."""
+    return [block for block in text.split("\n\n") if block.strip()]
+
+
+async def _extract_and_insert(
+    db: Session, job: FAQGenerationJob, generator, text: str, source_label: str,
+    mode: str = "extract",
+) -> int:
+    """Shared LLM tail for URL/PDF imports: batch → extract → dedup → insert.
+
+    mode="extract" migrates a page that already contains Q&A pairs (verbatim).
+    mode="generate" turns free-form documentation (a product guide, manual…)
+    into FAQs — an existing-FAQ extraction prompt would find almost nothing in
+    prose that isn't already a Q&A list."""
+    batches = pack_batches(_split_blocks(text), max_chars=generator.batch_chars, sep="\n\n")
+    categories = CategoryMerger(db, job.organization_id)
+
+    if mode == "generate":
+        async def extract(batch: str, dedup: DedupState):
+            return await generator.generate_from_text(
+                batch, existing_questions=dedup.for_prompt, existing_categories=categories.names
+            )
+    else:
+        async def extract(batch: str, dedup: DedupState):
+            faqs = await generator.extract_from_faq_page(batch, existing_questions=dedup.for_prompt)
+            if not faqs:
+                # The page had no verbatim Q&A — an index/category listing or free-form
+                # prose. Rather than import nothing, draft FAQs from the same content.
+                faqs = await generator.generate_from_text(
+                    batch, existing_questions=dedup.for_prompt, existing_categories=categories.names
+                )
+            return faqs
+
+    accepted = await draft_batches(db, job, batches, extract, retries_per_batch=0)
+
+    rows = [
+        FAQ(
+            organization_id=job.organization_id,
+            question=faq.question,
+            answer=faq.answer,
+            category=categories.merge(faq.category),
+            status=FAQStatus.DRAFT,
+            knowledge_id=None,
+            source_label=source_label,
+            generation_job_id=job.id,
+            created_by=job.user_id,
+        )
+        for _batch, faq in accepted
+    ]
+    return insert_draft_rows(db, job, rows)
+
+
+async def run_import_job(db: Session, job: FAQGenerationJob) -> int:
+    """Execute an IMPORT_URL job. Returns FAQs created. Status transitions and
+    notifications are the worker's responsibility."""
+    job_repo = FAQGenerationJobRepository(db)
+    # ANALYZING = fail fast on a missing model config, before any external I/O.
+    job_repo.update_progress(job.id, stage=FAQJobStage.ANALYZING_SOURCES, progress_percentage=5.0)
+    generator = build_generator(db, job.organization_id, job=job)
+
+    job_repo.update_progress(job.id, stage=FAQJobStage.EXTRACTING, progress_percentage=15.0)
+    # Re-validated at fetch time (not only at the API layer) so a job forged or
+    # replayed against the queue can't reach internal hosts; run in a thread so
+    # the slow fetch doesn't block the worker's event loop.
+    page_text = await asyncio.to_thread(fetch_page_text, job.source_url)
+    return await _extract_and_insert(
+        db, job, generator, page_text,
+        source_label=f"Imported from {urlparse(job.source_url).netloc}",
+    )
+
+
+def _pdf_to_text(content: bytes) -> str:
+    """Page-by-page text extraction with pypdf directly — agno's PDFReader
+    chunks/summarizes for the vector KB, which we don't want here. Blank-line
+    separators between pages keep block batching intact."""
+    from pypdf import PdfReader
+
+    reader = PdfReader(BytesIO(content))
+    pages = []
+    for page in reader.pages:
+        try:
+            text = page.extract_text() or ""
+        except Exception as e:  # a single corrupt page shouldn't sink the file
+            logger.warning(f"PDF page extraction failed: {e}")
+            continue
+        if text.strip():
+            pages.append(text.strip())
+    return "\n\n".join(pages)[: settings.FAQ_IMPORT_MAX_PAGE_CHARS]
+
+
+async def run_pdf_import_job(db: Session, job: FAQGenerationJob) -> int:
+    """Execute an IMPORT_PDF job: read the stored upload (job.source_url),
+    extract text and run the shared Q&A extraction. The stored file is
+    best-effort deleted afterwards — it's an import buffer, not a document
+    library."""
+    from app.services.file_storage import delete_upload, load_upload
+
+    job_repo = FAQGenerationJobRepository(db)
+    # Captured before the try: after a DB-originated failure the ORM instance
+    # is expired and reading job attributes from the finally would raise,
+    # masking the real error and leaking the buffer.
+    stored = job.source_url
+    source_label = f"Imported from {job.source_file_name or 'PDF'}"
+    # ANALYZING = fail fast on a missing model config, before file I/O.
+    job_repo.update_progress(job.id, stage=FAQJobStage.ANALYZING_SOURCES, progress_percentage=5.0)
+
+    try:
+        generator = build_generator(db, job.organization_id, job=job)
+        content = await load_upload(stored)
+        job_repo.update_progress(job.id, stage=FAQJobStage.EXTRACTING, progress_percentage=15.0)
+        text = await asyncio.to_thread(_pdf_to_text, content)
+        if not text.strip():
+            raise ValueError(
+                "Could not read any text from that PDF (it may be scanned images)."
+            )
+        # A PDF is usually documentation/marketing prose, not an existing Q&A
+        # list — generate FAQs from it rather than extracting verbatim pairs.
+        return await _extract_and_insert(
+            db, job, generator, text, source_label=source_label, mode="generate"
+        )
+    finally:
+        await delete_upload(stored)

backend/app/services/faq_usage.py

@@ -0,0 +1,180 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+FAQ generation usage/credits: estimate how many LLM calls a run will make,
+decide whether those calls are metered (hosted model), and enforce the org's
+message budget before enqueueing. OSS/self-hosted (no enterprise module) is
+never metered and never blocked.
+"""
+
+import math
+from dataclasses import dataclass
+from typing import List, Optional
+from uuid import UUID
+
+from fastapi import HTTPException
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+from app.core.config import settings
+from app.core.logger import get_logger
+from app.knowledge.page_editor import PAGE_ID_EXPR
+from app.models.knowledge import Knowledge
+from app.repositories.faq import FAQRepository
+# Shared hosted-model metering helpers; re-exported here so existing FAQ
+# callers keep importing them from faq_usage.
+from app.services.usage_metering import (  # noqa: F401
+    HOSTED_MODEL_TYPE,
+    is_hosted_model,
+    remaining_message_credits,
+)
+
+logger = get_logger(__name__)
+
+# Rough pages-per-LLM-call heuristic for the confirm-dialog estimate (matches
+# today's floor batch of ~3 average pages). An estimate, not billing.
+PAGES_PER_CALL = 3
+
+OVER_BUDGET_MESSAGE = (
+    "Not enough message credits left in your plan for this generation "
+    "(~{estimated} needed, {remaining} remaining). Upgrade your plan or "
+    "switch to your own AI model key."
+)
+
+
+@dataclass
+class SourceEstimate:
+    id: int
+    name: str
+    source_type: str
+    has_faqs: bool
+    pages: int
+    estimated_calls: int
+
+
+@dataclass
+class GenerationEstimate:
+    total_sources: int
+    new_sources: int
+    pages: int
+    estimated_calls: int
+    sources: List[SourceEstimate]
+
+
+def generation_is_metered(db: Session, organization_id: UUID) -> bool:
+    """Generation costs credits only on the hosted model — own-key orgs pay
+    their provider directly. FAQ_METER_OWN_KEY flips to always-meter."""
+    return settings.FAQ_METER_OWN_KEY or is_hosted_model(db, organization_id)
+
+
+def count_source_pages(db: Session, knowledge: Knowledge) -> Optional[int]:
+    """Distinct stored pages for a source, capped like generation itself.
+    None when the vector table is missing/unreadable."""
+    if not knowledge.schema or not knowledge.table_name:
+        return None
+    query = text(
+        f"SELECT COUNT(*) FROM (SELECT {PAGE_ID_EXPR} "
+        f'FROM {knowledge.schema}."{knowledge.table_name}" '
+        "WHERE name = :source GROUP BY 1 LIMIT :max_pages) pages"
+    )
+    try:
+        count = db.execute(
+            query, {"source": knowledge.source, "max_pages": settings.FAQ_MAX_PAGES_PER_SOURCE}
+        ).scalar()
+        return int(count or 0)
+    except Exception as e:
+        logger.warning(f"Page count failed for knowledge {knowledge.id}: {e}")
+        db.rollback()
+        return None
+
+
+def estimate_generation_calls(
+    db: Session,
+    organization_id: UUID,
+    knowledge_ids: Optional[List[int]] = None,
+    count_pages: bool = True,
+) -> GenerationEstimate:
+    """Confirm-dialog numbers for a GENERATE_ALL run: which sources it would
+    read (new-only unless explicitly targeted) and roughly how many LLM calls
+    that costs. Unreadable sources count as one call each.
+
+    Always returns the FULL per-source list (for the generation source picker),
+    each with its own page/call estimate and whether it already has FAQs. The
+    aggregate counts (new_sources/pages/estimated_calls) cover the run's TARGET
+    sources: the explicit `knowledge_ids` selection, or — when none is given —
+    every source that doesn't yet have FAQs.
+
+    count_pages=False skips the per-source page COUNT over the vector tables
+    (cheap — for surfaces that only need counts, like the Generate button
+    label); per-source estimated_calls is then a 1-per-source lower bound."""
+    all_sources = (
+        db.query(Knowledge).filter(Knowledge.organization_id == organization_id).all()
+    )
+    generated = FAQRepository(db).knowledge_ids_with_faqs(organization_id)
+    existing_ids = {s.id for s in all_sources}
+    target_ids = (
+        (set(knowledge_ids) & existing_ids)
+        if knowledge_ids
+        else {s.id for s in all_sources if s.id not in generated}
+    )
+
+    source_list: List[SourceEstimate] = []
+    agg_pages = 0
+    agg_calls = 0
+    for source in all_sources:
+        source_pages = count_source_pages(db, source) if count_pages else None
+        pages = source_pages or 0
+        if source_pages is None:
+            # Unreadable now, or page-count skipped (count_pages=False) — assume
+            # one call so the estimate isn't zero.
+            calls = 1
+        else:
+            # A readable but empty source produces nothing → costs no call.
+            calls = max(1, math.ceil(pages / PAGES_PER_CALL)) if pages else 0
+        source_list.append(
+            SourceEstimate(
+                id=source.id,
+                name=source.source,
+                source_type=source.source_type.value if hasattr(source.source_type, "value") else str(source.source_type),
+                has_faqs=source.id in generated,
+                pages=pages,
+                estimated_calls=calls,
+            )
+        )
+        if source.id in target_ids:
+            agg_pages += pages
+            agg_calls += calls
+
+    return GenerationEstimate(
+        total_sources=len(all_sources),
+        new_sources=len(target_ids),
+        pages=agg_pages,
+        estimated_calls=agg_calls,
+        sources=source_list,
+    )
+
+
+def ensure_generation_budget(db: Session, organization_id: UUID, estimated_calls: int) -> None:
+    """402 when a metered run would exceed the org's remaining credits."""
+    if estimated_calls <= 0 or not generation_is_metered(db, organization_id):
+        return
+    remaining = remaining_message_credits(db, organization_id)
+    if remaining is None:
+        return
+    if estimated_calls > remaining:
+        raise HTTPException(
+            status_code=402,
+            detail=OVER_BUDGET_MESSAGE.format(estimated=estimated_calls, remaining=remaining),
+        )

backend/app/services/firebase.py

@@ -0,0 +1,63 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from firebase_admin import credentials, initialize_app, get_app
+from app.core.config import settings
+from app.core.logger import get_logger
+import os
+
+logger = get_logger(__name__)
+
+
+def initialize_firebase():
+    """Initialize Firebase Admin SDK"""
+    try:
+        # Check if already initialized
+        try:
+            get_app()
+            logger.info("Firebase already initialized")
+            return
+        except ValueError:
+            logger.info("Initializing Firebase...")
+
+        # Get credentials file path
+        cred_path = settings.FIREBASE_CREDENTIALS
+
+        if not cred_path or not os.path.isfile(cred_path):
+            logger.warning(f"Firebase credentials not found at {cred_path}. Running in development mode.")
+            # Use a default configuration for development
+            cred = credentials.Certificate({
+                "type": "service_account",
+                "project_id": "demo-project",
+                "private_key_id": "demo123456789",
+                "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDK/eDH0MDgBNHs\nYuXwHMnMYGz+86YJjYoXVbKdUGHQZXYVEVlc8rYA+VXbU1AqhD+c7yT9V9EaEjPO\nXGVWS6dWQh4Kh5g7y1O5SHIwQhDHK+AAqMgBfZhzEHJBtO8CXG8TDyxhfHXxjWpD\nGPz/RJ7GhW9cHYQOvP4yZAp3ZNGYQJwUlhNLGr80XEhGVlkZ5WPL0YFzwOxvqGQs\nQVVxGGj5h7YHJt1YmZwF6cBGGX8AUetJkMTgQEJSNsaK4Pl+MZUGQOhQEynK3qO2\nWf+7XZl4XPx9Qw+WPYEoHHlGgGQR4YUpL+qzVxYwNAXhVxd9k+YPp5JZHtGQAXeO\nZhxDo1QLAgMBAAECggEABBj/H/JGXuVGoCjR6J0XF+kQTOuTgZBvgapkpPTYwfXQ\nPRKzGWEpGUPVnVb3gMDqbqQe7EVJZ8Nj/zHmXf9c4qKxpXwNl7kYmPpuQOHmA8Nq\nXPE3QZH3YSRz1YXF/pA9RQqG5F4JBJjz5LcUR+ax9aSxPu26xwDHOpR4EIR9Aovx\nJfcg7MTAY4YuGBe0oLQbEWoUQqXgUIVEzWcqiDGXF0uuMZzpW6YVk5ZvWHXz7SVK\nE7e1XQN6sMTYqrmnFGkBhuvXiBkPZHsOEHBVxXVqytE8FJAqXwPwFqxWj4RhNS/w\nkMUv5ZyH6UOFuBXtSe1XGGEqktNVGPBhAkEo4BbhAQKBgQD1+fdC9zzDwA4KoNnk\nVxQbCpC7Zr5F8Mycer0SV/3Gg6okzo9vXl+vqVHXqYNGnAEFkLLhEzwpZB0cnqWk\nQswrGvZ5FvUSKd7rMfI0Uh5FxhYCuUwwZW6gVKEZDCXF6N7KBbMPaYVZuGwD7oQk\nUQKBgQDTKj+RKqFrwhZM4cVJVgqHgQmMvHXC7Hy3Sj8Yl9ZHxgsnrkoNXXXRDNwp\nEWVoTy4Xr1BXuBxFEgEBpuPGBlHVXLZRzaP1pCZVqftxiHDqGlAh4y3yZtKHSTGj\n1B6OZlYXxvD+9K5Ki1ZwM3JxpY4YE5zASL7cUrV+gUUE2VoengKBgQDxIrXXFR8l\nAgMBAAECggEABBj/H/JGXuVGoCjR6J0XF+kQTOuTgZBvgapkpPTYwfXQ\n-----END PRIVATE KEY-----\n",
+                "client_email": "firebase-adminsdk-demo@demo-project.iam.gserviceaccount.com",
+                "client_id": "123456789",
+                "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+                "token_uri": "https://oauth2.googleapis.com/token",
+                "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+                "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/firebase-adminsdk-demo%40demo-project.iam.gserviceaccount.com",
+                "universe_domain": "googleapis.com"
+            })
+        else:
+            logger.info("Loading Firebase credentials from file " + cred_path)
+            cred = credentials.Certificate(cred_path)
+        
+        initialize_app(cred)
+        logger.info("Firebase initialized successfully")
+    except Exception as e:
+        logger.error(f"Error initializing Firebase: {str(e)}")
+        logger.warning("Continuing without Firebase initialization")

backend/app/services/help_center_access.py

@@ -0,0 +1,42 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Help-center plan gating: Pro-only on cloud, unrestricted for OSS/self-hosted.
+Thin wrappers over the generic feature gate.
+"""
+
+from uuid import UUID
+
+from sqlalchemy.orm import Session
+
+from app.services.feature_gate import check_feature_access, feature_allowed
+
+HELP_CENTER_FEATURE = "help_center"
+
+UPGRADE_MESSAGE = (
+    "The Help Center is not available in your current plan. "
+    "Please upgrade to Pro to access this feature."
+)
+
+
+def help_center_allowed(db: Session, organization_id: UUID) -> bool:
+    """Non-raising check — used by the public site (which 404s instead of
+    403ing) and the auto-generation hook. Fails closed on gating errors."""
+    return feature_allowed(db, organization_id, HELP_CENTER_FEATURE)
+
+
+def check_help_center_access(db: Session, organization_id: UUID) -> None:
+    """Raising check for admin endpoints (403 with the upgrade message)."""
+    check_feature_access(db, organization_id, HELP_CENTER_FEATURE, UPGRADE_MESSAGE)

backend/app/services/help_center_content.py

@@ -0,0 +1,131 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+FAQ answers are authored and stored as Markdown so a single Text column can hold
+rich help-center articles (headings, images, links, ordered steps, callouts).
+This module turns that Markdown into HTML for the public page, and into plain
+text for previews, meta descriptions and FAQPage structured data.
+
+Security: the rendered HTML is dropped onto the *public* help center, so an org
+admin's Markdown is untrusted input for visitors. Everything is run through an
+nh3 (ammonia) allowlist clean — the template must mark this output `| safe`, so
+this is the ONLY place org content is ever trusted as HTML.
+"""
+
+import re
+from typing import Mapping, Optional
+
+import markdown as _markdown
+import nh3
+from bs4 import BeautifulSoup
+
+# `extra` enables tables/fenced code/etc; `sane_lists` fixes mixed list quirks;
+# `nl2br` treats single newlines as <br> so plain-text FAQ answers (the common
+# case) keep their line breaks without the author needing blank lines.
+_MD_EXTENSIONS = ["extra", "sane_lists", "nl2br"]
+
+# Allowlist of tags a help-center article may contain. Anything else (script,
+# style, iframe, form, event handlers, …) is stripped by nh3.
+_ALLOWED_TAGS = {
+    "p", "br", "hr", "h1", "h2", "h3", "h4", "h5", "h6",
+    "strong", "b", "em", "i", "u", "s", "del", "code", "pre", "blockquote",
+    "ul", "ol", "li", "a", "img",
+    "table", "thead", "tbody", "tr", "th", "td",
+    "figure", "figcaption", "span",
+}
+_ALLOWED_ATTRS = {
+    "a": {"href", "title"},
+    "img": {"src", "alt", "title"},
+}
+# Block javascript:/data: URLs; images and links may only point at real hosts.
+_URL_SCHEMES = {"http", "https", "mailto"}
+
+_WHITESPACE_RE = re.compile(r"\s+")
+
+# Intra-help-center links are stored ROOT-RELATIVE (/a/{slug}, or / for the home)
+# so they survive a move between path mode and a subdomain/custom domain. The
+# serving prefix is added here at render time. `/a/` (with the trailing slash)
+# never matches an image path like /api/v1/uploads/...
+_INTERNAL_HREF_RE = re.compile(r'(href=")(/a/[^"]*|/)(")')
+
+
+def render_article_html(
+    text: str, base_path: str = "", link_paths: Optional[Mapping[str, str]] = None
+) -> str:
+    """Markdown -> sanitized HTML for the public article body. Safe to render
+    with Jinja's `| safe` because nh3 strips every tag/attr/scheme off-allowlist
+    and rewrites link rel to neutralise tab-nabbing.
+
+    `base_path` is the serving prefix ("" at a domain root, "/help/{slug}" in path
+    mode) prepended to root-relative intra-help-center links so cross-article links
+    resolve under whichever mode/domain serves the page.
+
+    `link_paths` maps slug -> the path that article is actually served at, for
+    the articles whose original URL was preserved on a help-center migration.
+    Cross-article links stay STORED as /a/{slug} (stable when a path is later
+    edited, and the only form the regex above can prefix) and are resolved here,
+    so a link lands directly on the preserved URL instead of via its 301."""
+    if not text or not text.strip():
+        return ""
+    raw = _markdown.markdown(text, extensions=_MD_EXTENSIONS, output_format="html")
+    html = nh3.clean(
+        raw,
+        tags=_ALLOWED_TAGS,
+        attributes=_ALLOWED_ATTRS,
+        url_schemes=_URL_SCHEMES,
+        link_rel="noopener noreferrer nofollow",
+    )
+    if base_path or link_paths:
+        html = _INTERNAL_HREF_RE.sub(
+            lambda m: f"{m.group(1)}{base_path}{_resolve_internal_href(m.group(2), link_paths)}{m.group(3)}",
+            html,
+        )
+    return html
+
+
+def _resolve_internal_href(href: str, link_paths: Optional[Mapping[str, str]]) -> str:
+    """A stored /a/{slug} link swapped for that article's preserved path, if it
+    has one. Unknown slugs (and the "/" home link) pass straight through."""
+    if not link_paths or not href.startswith("/a/"):
+        return href
+    return link_paths.get(href[len("/a/"):], href)
+
+
+def to_plain_text(text: str) -> str:
+    """Strip Markdown/HTML to a single line of readable text (previews, meta
+    descriptions, JSON-LD answer text)."""
+    if not text or not text.strip():
+        return ""
+    html = _markdown.markdown(text, extensions=_MD_EXTENSIONS, output_format="html")
+    plain = BeautifulSoup(html, "html.parser").get_text(" ")
+    return _WHITESPACE_RE.sub(" ", plain).strip()
+
+
+def excerpt(text: str, length: int = 150) -> str:
+    """A short one-line preview of an answer, cut on a word boundary."""
+    plain = to_plain_text(text)
+    if len(plain) <= length:
+        return plain
+    return plain[:length].rsplit(" ", 1)[0].rstrip() + "…"
+
+
+def read_time_minutes(text: str) -> int:
+    """Estimated read time in whole minutes (>= 1), ~200 words/min."""
+    words = len(to_plain_text(text).split())
+    return max(1, round(words / 200))
+
+
+def read_time_label(text: str) -> str:
+    return f"{read_time_minutes(text)} min read"

backend/app/services/help_center_images.py

@@ -0,0 +1,124 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Help-center article images. Images embedded in an article's Markdown must
+resolve to a STABLE URL that is baked inline in the answer forever — so no
+signed/expiring URLs, and no raw S3 URLs either now the bucket is private.
+Both storage backends therefore return the same host-RELATIVE path, which
+app/api/help_center_images.py resolves per request (redirect to a freshly
+signed URL on S3, direct file on local). Host-relative so it resolves against
+whichever origin serves the page: the main app in path mode, public_app on a
+help-center subdomain / custom domain. Shared by the admin upload endpoint and
+the article importer's re-hosting path.
+"""
+
+import re
+from uuid import uuid4
+
+from app.core.config import settings
+from app.services.file_storage import store_upload
+
+def strip_upload_host(text: str) -> str:
+    """Convert the app's OWN absolute upload URLs to relative /api/v1/uploads/...
+    paths, leaving everything else untouched. Only strips the configured
+    BACKEND_URL origin or a loopback host (localhost/127.0.0.1/0.0.0.0) — a
+    third-party URL that merely contains /api/v1/uploads/ (e.g. a link pasted
+    into an FAQ answer) is NOT rewritten. S3 and already-relative URLs never
+    match. Used by the one-time backfill migration."""
+    if not text:
+        return text
+    backend = settings.BACKEND_URL.rstrip("/")
+    # Match our own backend origin OR any loopback host (with optional port),
+    # only when it sits directly in front of an /api/v1/uploads/ path.
+    pattern = re.compile(
+        r'(?:' + re.escape(backend)
+        + r'|https?://(?:localhost|127\.0\.0\.1|0\.0\.0\.0)(?::\d+)?)'
+        + r'(?=/api/v1/uploads/)'
+    )
+    return pattern.sub("", text)
+
+MAX_FAQ_IMAGE_BYTES = 5 * 1024 * 1024
+# Reject larger-than-this on a side up front (bomb guard); downscale the stored
+# image to fit FAQ_IMAGE_FIT_DIM so article pages stay light.
+FAQ_IMAGE_MAX_DIM = 6000
+FAQ_IMAGE_FIT_DIM = 1600
+FAQ_IMAGE_TYPES = {
+    "image/png": ".png",
+    "image/jpeg": ".jpg",
+    "image/gif": ".gif",
+    "image/webp": ".webp",
+}
+UPLOAD_FOLDER = "help_center"
+
+# store_article_image names files "<uuid4><ext>". The delivery route matches
+# against this before interpolating the name into an S3 key or a filesystem
+# path, so it is what stops a traversal ("../agents/x.png") being served.
+_UUID4_PATTERN = (
+    r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
+)
+_EXTENSION_PATTERN = "|".join(
+    re.escape(ext.lstrip(".")) for ext in sorted(set(FAQ_IMAGE_TYPES.values()))
+)
+ARTICLE_IMAGE_NAME_RE = re.compile(rf"^{_UUID4_PATTERN}\.(?:{_EXTENSION_PATTERN})$")
+
+# Absolute S3 URLs baked into article Markdown before delivery moved behind
+# article_image_path(). Both URL shapes (virtual-hosted and path-style) end in
+# ".../help_center/<name>", so anchor on that.
+_BAKED_S3_IMAGE_RE = re.compile(
+    r"https?://[^\s\"'()]*amazonaws\.com/[^\s\"'()]*"
+    + re.escape(UPLOAD_FOLDER)
+    + rf"/({_UUID4_PATTERN}\.(?:{_EXTENSION_PATTERN}))"
+)
+
+
+def article_image_path(file_name: str) -> str:
+    """The stable, host-relative URL baked into article Markdown.
+
+    Identical for local and S3 storage — app/api/help_center_images.py resolves
+    it per request. Host-relative so it works both in path mode (main app) and
+    on a help-center subdomain / custom domain (public_app)."""
+    return f"{settings.API_V1_STR}/help-center/images/{file_name}"
+
+
+def absolute_upload_url(stored: str) -> str:
+    """Make a stored upload path absolute. S3 returns an absolute http(s) URL
+    already; local mode returns a path under the /api/v1/uploads mount, which we
+    anchor to the backend's public origin so it loads cross-domain."""
+    if stored.startswith("http"):
+        return stored
+    return f"{settings.BACKEND_URL.rstrip('/')}{stored}"
+
+
+def rewrite_baked_s3_image_urls(text: str) -> str:
+    """Point previously-baked absolute S3 article-image URLs at article_image_path.
+
+    Those URLs 403 now the bucket is private, and would have needed re-signing
+    anyway. Used by the one-time backfill migration."""
+    if not text:
+        return text
+    return _BAKED_S3_IMAGE_RE.sub(lambda m: article_image_path(m.group(1)), text)
+
+
+async def store_article_image(content: bytes, content_type: str) -> str:
+    """Persist validated image bytes and return the stable URL to bake into the
+    answer markdown. Caller is responsible for size/content-type validation
+    against MAX_FAQ_IMAGE_BYTES / FAQ_IMAGE_TYPES.
+
+    The returned path is storage-agnostic and never expires. It must not be a
+    signed S3 URL (those expire) nor a raw S3 URL (the bucket is private, so
+    those 403) — the delivery route resolves it per request instead."""
+    file_name = f"{uuid4()}{FAQ_IMAGE_TYPES[content_type]}"
+    await store_upload(content, folder=UPLOAD_FOLDER, file_name=file_name, content_type=content_type)
+    return article_image_path(file_name)

backend/app/services/help_center_public.py

@@ -0,0 +1,238 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Public help-center domain logic: resolve the Host header to an org's
+published help center, group its FAQs, and answer "Ask AI" questions grounded
+in the mapped agent's knowledge. No admin auth here — everything must stay
+read-only against published data.
+"""
+
+import asyncio
+from typing import List, Optional, Tuple
+
+from sqlalchemy.orm import Session
+
+from app.agents.guardrail_policy import (
+    GuardrailContext,
+    INJECTION_CLAUSE,
+    visitor_data_block,
+)
+from app.core.help_center_host import normalize_host, slug_for_host  # noqa: F401 — canonical host helpers re-exported for the public app
+from app.core.logger import get_logger
+from app.utils.guardrail_runtime import Surface, check_inbound
+from app.database import SessionLocal
+from app.models.faq import FAQ
+from app.models.help_center import HelpCenterSettings
+from app.repositories.faq import FAQRepository
+from app.repositories.help_center import HelpCenterQueryRepository, HelpCenterRepository
+from app.services.help_center_access import help_center_allowed
+
+logger = get_logger(__name__)
+
+MAX_QUESTION_CHARS = 500
+ANSWER_MAX_TOKENS = 600
+# Unauthenticated LLM calls are expensive: bound how many run at once per
+# worker so /ask can't drain the DB pool or the provider budget.
+_ask_concurrency = asyncio.Semaphore(4)
+
+_ASK_INSTRUCTIONS = """You are the support assistant on a company's public help center. A visitor has asked a question.
+
+You have tools to look up the answer:
+- search_faqs: search the published help-center FAQs.
+- search_knowledge_base: search the company's wider knowledge base (only available when the help center is mapped to an agent).
+
+ALWAYS use the tools before answering — start with search_faqs, and also try
+search_knowledge_base for anything the FAQs don't cover. You may call a tool more
+than once with different wording if the first result isn't relevant.
+
+Answer ONLY from what the tools return. When they surface a relevant answer, reply
+in 1-4 short sentences of plain text (no markdown), in a friendly second-person
+voice. If, after genuinely searching, you find nothing relevant, briefly say you
+couldn't find that in the help center and suggest starting a chat or contacting
+support. Never invent facts, prices, policies, or URLs.
+
+""" + INJECTION_CLAUSE
+
+
+def resolve_help_center(
+    db: Session, host: Optional[str] = None, slug: Optional[str] = None
+) -> Optional[HelpCenterSettings]:
+    """Enabled help center row, or None (renders as 404). Resolves by explicit
+    `slug` (path dispatch) when given, else by Host header (subdomain slug or a
+    verified custom domain). Custom domains only match once verified; plan
+    lapses hide the site."""
+    repo = HelpCenterRepository(db)
+    if slug is not None:
+        row = repo.get_by_slug(slug)
+    else:
+        host_slug = slug_for_host(host or "")
+        if host_slug:
+            row = repo.get_by_slug(host_slug)
+        else:
+            row = repo.get_by_verified_domain(host or "")
+    if not row or not row.enabled:
+        return None
+    if not help_center_allowed(db, row.organization_id):
+        return None
+    return row
+
+
+# Topic accent colors, applied by category display order (matches the index and
+# article templates). Kept here so list cards, sidebar topics, the article tag
+# and related cards all resolve the SAME color for a given category.
+CATEGORY_PALETTE = [
+    "#6d5bd0", "#0e8c8c", "#cf5b38", "#2a6fdb", "#1f8a5b", "#b0468a", "#c98a1e", "#3a6f8f",
+]
+
+
+def published_faq_groups(
+    db: Session, row: HelpCenterSettings, search: Optional[str] = None
+) -> List[Tuple[str, List[FAQ]]]:
+    """Published FAQs grouped by category, in display order."""
+    faqs = FAQRepository(db).get_published_for_org(row.organization_id, search=search)
+    groups: dict = {}
+    for faq in faqs:
+        groups.setdefault(faq.category, []).append(faq)
+    return list(groups.items())
+
+
+def category_colors(categories) -> dict:
+    """category name -> accent hex, by display order."""
+    return {c: CATEGORY_PALETTE[i % len(CATEGORY_PALETTE)] for i, c in enumerate(categories)}
+
+
+def get_published_article(db: Session, row: HelpCenterSettings, slug: str) -> Optional[FAQ]:
+    """A single published FAQ by slug for the /a/{slug} article page."""
+    return FAQRepository(db).get_published_by_slug(row.organization_id, slug)
+
+
+def get_published_article_by_path(
+    db: Session, row: HelpCenterSettings, url_path: str
+) -> Optional[FAQ]:
+    """A single published FAQ by its preserved original path — the lookup for a
+    URL the org carried over from the help center it migrated from."""
+    return FAQRepository(db).get_published_by_url_path(row.organization_id, url_path)
+
+
+def related_articles(db: Session, row: HelpCenterSettings, faq: FAQ, limit: int = 4) -> List[FAQ]:
+    """Other published FAQs in the same category — the article's related list."""
+    return FAQRepository(db).get_published_related(
+        row.organization_id, faq.category, faq.id, limit=limit
+    )
+
+
+def contrast_ink(hex_color: str) -> str:
+    """Dark or light text over the brand color (WCAG relative luminance)."""
+    try:
+        value = hex_color.lstrip("#")
+        if len(value) == 3:
+            value = "".join(ch * 2 for ch in value)
+        r, g, b = (int(value[i:i + 2], 16) / 255 for i in (0, 2, 4))
+
+        def linear(c: float) -> float:
+            return c / 12.92 if c <= 0.04045 else ((c + 0.055) / 1.055) ** 2.4
+
+        luminance = 0.2126 * linear(r) + 0.7152 * linear(g) + 0.0722 * linear(b)
+        return "#12131A" if luminance > 0.45 else "#FFFFFF"
+    except (ValueError, IndexError):
+        return "#FFFFFF"
+
+
+def widget_id_for(row: HelpCenterSettings) -> Optional[str]:
+    """The chat widget embedded on the page: the mapped agent's widget. Gated by
+    its own toggle, independent of the AI quick-summary in search."""
+    if not (row.chat_widget_enabled and row.agent and row.agent.widgets):
+        return None
+    return row.agent.widgets[0].id
+
+
+def ask_available(row: HelpCenterSettings) -> bool:
+    return bool(row.ai_search_enabled and row.agent_id)
+
+
+async def answer_question(organization_id, agent_id, question: str) -> Optional[str]:
+    """Grounded answer for a public help-center question. The agent is given
+    FAQ + knowledge search tools and decides what to look up, so it can find and
+    combine answers instead of relying on a single pre-stuffed context. Returns
+    None when no confident answer could be produced. Writes the ask-log row
+    (metering/analytics); never touches chat history or the KB.
+
+    The whole tool-using run happens in a worker thread: the tools do synchronous
+    DB work, and holding a pooled connection through a multi-second model call
+    would let this unauthenticated endpoint drain the pool for the backend.
+    """
+    from app.core.security import decrypt_api_key
+    from app.repositories.ai_config import AIConfigRepository
+
+    question = question.strip()[:MAX_QUESTION_CHARS]
+
+    # Pre-LLM guardrail on the only unauthenticated LLM entry point: a
+    # blocked question never reaches the semaphore, the DB pool, or the model.
+    # None renders as the standard "couldn't find that" fallback.
+    guardrail_ctx = GuardrailContext(
+        org_id=str(organization_id), agent_id=str(agent_id) if agent_id else None
+    )
+    if check_inbound(question, ctx=guardrail_ctx, surface=Surface.HELP_CENTER).block:
+        return None
+
+    async with _ask_concurrency:
+        with SessionLocal() as db:
+            config = AIConfigRepository(db).get_active_config(organization_id)
+            if not config:
+                return None
+            model_type = config.model_type.value if hasattr(config.model_type, "value") else str(config.model_type)
+            model_name = config.model_name
+            api_key = decrypt_api_key(config.encrypted_api_key)
+            base_url = (config.settings or {}).get('base_url')
+
+        def _run() -> Optional[str]:
+            from agno.agent import Agent
+
+            from app.tools.faq_search import FAQSearchTool
+            from app.tools.knowledge_search_byagent import KnowledgeSearchByAgent
+            from app.utils.agno_utils import create_model
+
+            tools = [FAQSearchTool(organization_id=organization_id)]
+            if agent_id:
+                tools.append(KnowledgeSearchByAgent(agent_id=str(agent_id), org_id=organization_id))
+            model = create_model(
+                model_type=model_type, api_key=api_key, model_name=model_name, max_tokens=ANSWER_MAX_TOKENS, base_url=base_url
+            )
+            agent = Agent(
+                name="Help Center Answers",
+                model=model,
+                tools=tools,
+                instructions=_ASK_INSTRUCTIONS,
+                markdown=False,
+            )
+            response = agent.run(
+                message=visitor_data_block("VISITOR QUESTION", question),
+                stream=False,
+            )
+            content = getattr(response, "content", None)
+            return content.strip() if isinstance(content, str) and content.strip() else None
+
+        answer = None
+        try:
+            answer = await asyncio.to_thread(_run)
+        except Exception as e:
+            logger.error(f"Help center ask failed for org {organization_id}: {e}")
+        finally:
+            try:
+                with SessionLocal() as log_db:
+                    HelpCenterQueryRepository(log_db).log(organization_id, question, answered=bool(answer))
+            except Exception as log_err:
+                logger.error(f"Help center ask log failed: {log_err}")
+    return answer

backend/app/services/help_center_seo.py

@@ -0,0 +1,267 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+SEO surface of the public help center: the <head> values (title, description,
+canonical, Open Graph) and the schema.org JSON-LD graph. Kept out of the route
+module so every page derives its metadata by the same rules.
+
+Two URL spaces are in play and must not be mixed up:
+  - ABSOLUTE (live_url-based) for canonical, og:*, sitemap and every JSON-LD
+    @id/url — crawlers and social scrapers need a fetchable address.
+  - RELATIVE (base_path-prefixed) for links the page actually renders, so the
+    same markup works on a subdomain, a custom domain and /help/{slug}.
+Breadcrumbs carry both, which is how the BreadcrumbList markup is guaranteed to
+match the visible trail (a Google requirement).
+"""
+
+from datetime import datetime
+from typing import List, Optional, Sequence
+from urllib.parse import quote, urlsplit
+
+from app.models.faq import FAQ
+from app.models.help_center import HelpCenterSettings
+from app.services.help_center_content import excerpt, to_plain_text
+from app.services.help_center_settings import live_url
+
+HOME_CRUMB = "Home"
+# Meta descriptions get truncated by search engines around here; the excerpt
+# helper adds an ellipsis rather than cutting mid-word.
+META_DESCRIPTION_CHARS = 200
+
+
+def site_url(row: HelpCenterSettings) -> str:
+    """The help center's public base URL, never trailing-slashed."""
+    return live_url(row).rstrip("/")
+
+
+def asset_origin(site: str) -> str:
+    """Scheme + host the site's uploads are served from. Same origin as the
+    site in every mode — but NOT the same prefix in path mode, where the site
+    is {backend}/help/{slug} while uploads stay at {backend}/api/v1/uploads/…"""
+    parts = urlsplit(site)
+    return f"{parts.scheme}://{parts.netloc}"
+
+
+def absolute_asset_url(stored_url: Optional[str], site: str) -> Optional[str]:
+    """A stored logo/image URL made absolute for og:image and JSON-LD. S3 URLs
+    are already absolute and pass through; local uploads are relative and get
+    anchored to the serving origin."""
+    if not stored_url:
+        return None
+    if stored_url.startswith("http://") or stored_url.startswith("https://"):
+        return stored_url
+    return f"{asset_origin(site)}{stored_url}"
+
+
+def _isoformat(value: Optional[datetime]) -> Optional[str]:
+    return value.isoformat() if value else None
+
+
+def article_title(row: HelpCenterSettings, faq: FAQ) -> str:
+    """The article's <title>: the org's override, else question + site name."""
+    return faq.meta_title or f"{faq.question} · {row.organization.name} Help Center"
+
+
+def article_description(faq: FAQ) -> str:
+    """The article's meta description: the org's override, else an excerpt of
+    the answer (falling back to the question for answers with no prose)."""
+    return faq.meta_description or excerpt(faq.answer, META_DESCRIPTION_CHARS) or faq.question
+
+
+def index_title(row: HelpCenterSettings) -> str:
+    return row.title or f"{row.organization.name} Help Center"
+
+
+def index_description(row: HelpCenterSettings) -> str:
+    return row.description or f"Answers to common questions about {row.organization.name}."
+
+
+def category_url(site: str, category: str) -> str:
+    """Absolute URL of the index filtered to one category — the middle
+    breadcrumb. Mirrors the ?topic= link the templates render."""
+    return f"{site}/?topic={quote(category)}"
+
+
+def article_path(faq: FAQ) -> str:
+    """The root-relative path an article is served at, percent-DECODED.
+
+    An article migrated in with "preserve original URLs" keeps the path it had
+    on the old help center; everything else lives at /a/{slug}. Exactly one of
+    the two is canonical — /a/{slug} 301s to the preserved path when there is
+    one — so this is the single answer to "where does this article live?" and
+    every URL the site emits derives from it."""
+    return faq.url_path or f"/a/{faq.slug}"
+
+
+def article_href(faq: FAQ, base_path: str = "") -> str:
+    """What a page renders in href=: the serving prefix plus the ENCODED path."""
+    return f"{base_path}{quote(article_path(faq), safe='/')}"
+
+
+def article_url(site: str, faq: FAQ) -> str:
+    """The article's ABSOLUTE URL, for canonical/og:url/sitemap/JSON-LD."""
+    return f"{site}{quote(article_path(faq), safe='/')}"
+
+
+def breadcrumbs(
+    row: HelpCenterSettings, faq: FAQ, base_path: str, site: str
+) -> List[dict]:
+    """Home › Category › Article. Each crumb carries `href` (what the page
+    links to) and `url` (what BreadcrumbList advertises); the last crumb is the
+    current page and is rendered as text, so it has no href."""
+    return [
+        {"label": HOME_CRUMB, "href": f"{base_path}/", "url": f"{site}/"},
+        {
+            "label": faq.category,
+            "href": f"{base_path}/?topic={quote(faq.category)}",
+            "url": category_url(site, faq.category),
+        },
+        {"label": faq.question, "href": None, "url": article_url(site, faq)},
+    ]
+
+
+def _organization_node(row: HelpCenterSettings, site: str, logo_url: Optional[str]) -> dict:
+    node = {
+        "@type": "Organization",
+        "@id": f"{site}/#organization",
+        "name": row.organization.name,
+        "url": f"{site}/",
+    }
+    if logo_url:
+        node["logo"] = {"@type": "ImageObject", "url": logo_url}
+    return node
+
+
+def _website_node(row: HelpCenterSettings, site: str) -> dict:
+    return {
+        "@type": "WebSite",
+        "@id": f"{site}/#website",
+        "url": f"{site}/",
+        "name": index_title(row),
+        "publisher": {"@id": f"{site}/#organization"},
+    }
+
+
+def _question_entity(faq: FAQ) -> dict:
+    """One schema.org Question + acceptedAnswer, from the plain-text form of the
+    (Markdown) answer."""
+    return {
+        "@type": "Question",
+        "name": faq.question,
+        "acceptedAnswer": {"@type": "Answer", "text": to_plain_text(faq.answer)},
+    }
+
+
+def _breadcrumb_node(crumbs: Sequence[dict], page_url: str) -> dict:
+    return {
+        "@type": "BreadcrumbList",
+        "@id": f"{page_url}#breadcrumb",
+        "itemListElement": [
+            {"@type": "ListItem", "position": i, "name": crumb["label"], "item": crumb["url"]}
+            for i, crumb in enumerate(crumbs, start=1)
+        ],
+    }
+
+
+def index_json_ld(
+    row: HelpCenterSettings, faqs: Sequence[FAQ], site: str, logo_url: Optional[str]
+) -> dict:
+    """The landing page graph: Organization + WebSite + a CollectionPage that is
+    also the FAQPage carrying every published Q&A.
+
+    FAQPage - not QAPage - is the correct type for these owner-authored answers.
+    QAPage models community/forum pages where users submit competing answers and
+    requires answerCount/upvoteCount, which don't apply here. Google retired FAQ
+    rich results, but the markup still describes the page for other consumers.
+    """
+    page_url = f"{site}/"
+    page: dict = {
+        "@type": ["CollectionPage", "FAQPage"],
+        "@id": f"{page_url}#webpage",
+        "url": page_url,
+        "name": index_title(row),
+        "description": index_description(row),
+        "isPartOf": {"@id": f"{site}/#website"},
+        "about": {"@id": f"{site}/#organization"},
+    }
+    if faqs:
+        page["mainEntity"] = [_question_entity(faq) for faq in faqs]
+    return {
+        "@context": "https://schema.org",
+        "@graph": [
+            _organization_node(row, site, logo_url),
+            _website_node(row, site),
+            page,
+        ],
+    }
+
+
+def article_json_ld(
+    row: HelpCenterSettings,
+    faq: FAQ,
+    site: str,
+    crumbs: Sequence[dict],
+    logo_url: Optional[str],
+    image_url: Optional[str],
+) -> dict:
+    """The article page graph: Organization + WebSite + WebPage + BreadcrumbList
+    + TechArticle, wired together by @id.
+
+    TechArticle (not FAQPage) is what a single how-to/help article is: Google
+    dropped FAQ rich results, while BreadcrumbList still renders in search.
+    """
+    page_url = article_url(site, faq)
+    description = article_description(faq)
+    article: dict = {
+        "@type": "TechArticle",
+        "@id": f"{page_url}#article",
+        "headline": faq.question,
+        "description": description,
+        "articleSection": faq.category,
+        "inLanguage": "en",
+        "mainEntityOfPage": {"@id": f"{page_url}#webpage"},
+        "isPartOf": {"@id": f"{page_url}#webpage"},
+        "author": {"@id": f"{site}/#organization"},
+        "publisher": {"@id": f"{site}/#organization"},
+    }
+    published_at = _isoformat(faq.created_at)
+    # updated_at stays NULL until the first edit, so fall back to the creation
+    # date rather than claiming a modification date older than publication.
+    modified_at = _isoformat(faq.updated_at) or published_at
+    if published_at:
+        article["datePublished"] = published_at
+    if modified_at:
+        article["dateModified"] = modified_at
+    if image_url:
+        article["image"] = image_url
+    web_page = {
+        "@type": "WebPage",
+        "@id": f"{page_url}#webpage",
+        "url": page_url,
+        "name": article_title(row, faq),
+        "description": description,
+        "isPartOf": {"@id": f"{site}/#website"},
+        "breadcrumb": {"@id": f"{page_url}#breadcrumb"},
+    }
+    return {
+        "@context": "https://schema.org",
+        "@graph": [
+            _organization_node(row, site, logo_url),
+            _website_node(row, site),
+            web_page,
+            _breadcrumb_node(crumbs, page_url),
+            article,
+        ],
+    }

backend/app/services/help_center_settings.py

@@ -0,0 +1,252 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Help-center settings lifecycle: lazy get-or-create (which is also the
+"org has adopted the feature" marker), slug generation and the agent
+auto-default for AI search.
+"""
+
+import re
+from typing import Iterable, Optional, Sequence
+from urllib.parse import unquote, urlsplit
+from uuid import UUID
+
+from sqlalchemy.orm import Session
+
+from app.core.config import settings
+from app.core.logger import get_logger
+from app.models.agent import Agent
+from app.models.faq import FAQ, FAQ_SLUG_MAX_LENGTH, FAQ_URL_PATH_MAX_LENGTH
+from app.models.help_center import HelpCenterSettings
+from app.models.knowledge_to_agent import KnowledgeToAgent
+from app.repositories.faq import FAQRepository
+from app.repositories.help_center import HelpCenterRepository
+
+logger = get_logger(__name__)
+
+SLUG_MAX_LENGTH = 63
+_SLUG_CLEAN_RE = re.compile(r"[^a-z0-9]+")
+
+# First path segments the public help center already owns (app.api
+# help_center_public routes, its uploads/images mounts, the /help/{slug}
+# dispatch prefix, and /.well-known, which ACME needs for TLS issuance). A
+# preserved path may never start with one of these or it would shadow the real
+# route. Matching on the FIRST SEGMENT rather than a string prefix keeps
+# "/apix/..." usable while still rejecting "/api/...".
+FAQ_RESERVED_FIRST_SEGMENTS = frozenset(
+    {"a", "ask", "api", "help", "healthz", "robots.txt", "sitemap.xml", ".well-known", "static", "uploads"}
+)
+_CONTROL_CHARS_RE = re.compile(r"[\x00-\x1f\x7f]")
+
+
+def slugify_org_name(name: str) -> str:
+    slug = _SLUG_CLEAN_RE.sub("-", (name or "").casefold()).strip("-")
+    return slug[:SLUG_MAX_LENGTH].strip("-") or "help"
+
+
+def _faq_base_slug(question: str) -> str:
+    base = _SLUG_CLEAN_RE.sub("-", (question or "").casefold()).strip("-")
+    return base[:FAQ_SLUG_MAX_LENGTH].strip("-") or "article"
+
+
+def _dedupe_slug(base: str, is_taken) -> str:
+    """First free slug from `base`, appending -2, -3, … while is_taken(candidate)."""
+    candidate = base
+    suffix = 2
+    while is_taken(candidate):
+        tail = f"-{suffix}"
+        candidate = f"{base[:FAQ_SLUG_MAX_LENGTH - len(tail)]}{tail}"
+        suffix += 1
+    return candidate
+
+
+def generate_faq_slug(db: Session, organization_id: UUID, question: str) -> str:
+    """A per-org-unique URL slug from an FAQ question. Collisions get -2, -3, …
+    Assigned once at creation and kept stable afterwards so article URLs persist."""
+    repo = FAQRepository(db)
+    return _dedupe_slug(_faq_base_slug(question), lambda c: repo.slug_exists(organization_id, c))
+
+
+def resolve_faq_slug(
+    db: Session, organization_id: UUID, requested: str, exclude_id: Optional[UUID] = None
+) -> str:
+    """A hand-typed slug, reduced to the same shape generation produces
+    (lowercase, a-z0-9 and single hyphens, length-capped) and made unique within
+    the org. `exclude_id` is the FAQ being edited, so keeping its own slug is a
+    no-op rather than a collision that appends -2."""
+    repo = FAQRepository(db)
+    return _dedupe_slug(
+        _faq_base_slug(requested),
+        lambda c: repo.slug_exists(organization_id, c, exclude_id=exclude_id),
+    )
+
+
+def assign_faq_slugs(db: Session, faqs) -> None:
+    """Give every FAQ in a batch a unique slug in place, deduping against both the
+    DB and the other rows in the same batch. Rows that already have a slug are
+    left untouched (and reserved so batch-mates don't collide with them)."""
+    repo = FAQRepository(db)
+    taken: set = set()
+    for faq in faqs:
+        if faq.slug:
+            taken.add(faq.slug)
+            continue
+        slug = _dedupe_slug(
+            _faq_base_slug(faq.question),
+            lambda c: c in taken or repo.slug_exists(faq.organization_id, c),
+        )
+        faq.slug = slug
+        taken.add(slug)
+
+
+def normalize_url_path(raw: Optional[str]) -> Optional[str]:
+    """A full URL or hand-typed path reduced to the root-relative path an article
+    can be served at — or None when it can't be used at all, in which case the
+    caller falls back to /a/{slug}.
+
+    Returned percent-DECODED, because that is the form the ASGI scope hands the
+    router to match against; every emitted URL re-encodes it (see
+    help_center_seo.article_url). Getting that direction backwards silently 404s
+    every non-ASCII path."""
+    if not raw:
+        return None
+    candidate = raw.strip()
+    if "://" in candidate or candidate.startswith("//"):
+        candidate = urlsplit(candidate).path  # drop scheme/host/port
+    candidate = candidate.split("#", 1)[0].split("?", 1)[0]  # drop fragment/query
+    candidate = unquote(candidate)
+    if _CONTROL_CHARS_RE.search(candidate):
+        return None
+    if not candidate.startswith("/"):
+        candidate = f"/{candidate}"
+    candidate = re.sub(r"/{2,}", "/", candidate).rstrip("/")
+    if len(candidate) > FAQ_URL_PATH_MAX_LENGTH:
+        return None
+    segments = candidate.split("/")[1:]
+    if not segments or not segments[0]:
+        return None  # "" or "/" — that's the index, not an article
+    if "." in segments or ".." in segments:
+        return None  # no traversal, and nothing that normalises to another path
+    if segments[0].lower() in FAQ_RESERVED_FIRST_SEGMENTS:
+        return None
+    return candidate
+
+
+def resolve_faq_url_path(
+    db: Session, organization_id: UUID, requested: str, exclude_id: Optional[UUID] = None
+) -> Optional[str]:
+    """A validated preserved path that is free within the org, else None.
+
+    Deliberately does NOT dedupe with -2/-3 the way slugs do: a mangled path is
+    no longer the URL the org already ranks for, so silently altering it would
+    defeat the whole point. The caller decides what to do instead — fall back to
+    /a/{slug} on import, or reject the edit in the admin API."""
+    path = normalize_url_path(requested)
+    if path is None:
+        return None
+    repo = FAQRepository(db)
+    return None if repo.url_path_exists(organization_id, path, exclude_id=exclude_id) else path
+
+
+def assign_faq_url_paths(db: Session, faqs: Sequence[FAQ], sources: Iterable[str]) -> int:
+    """Give each FAQ in a batch its source URL's path, in place, deduping against
+    both the DB and the other rows in the batch. Returns how many were applied.
+
+    Rows whose path is unusable or taken keep url_path=None and are served at
+    /a/{slug} — a partial result still imports every article, which beats
+    failing the whole job over one bad URL.
+
+    MUST run before the batch insert, so the unique index can never reject it."""
+    repo = FAQRepository(db)
+    taken: set = set()
+    applied = 0
+    for faq, source in zip(faqs, sources):
+        path = normalize_url_path(source)
+        if path is None:
+            continue
+        if path in taken or repo.url_path_exists(faq.organization_id, path):
+            logger.warning(f"Preserved URL path already taken, using the slug instead: {path}")
+            continue
+        faq.url_path = path
+        taken.add(path)
+        applied += 1
+    return applied
+
+
+def generate_unique_slug(db: Session, name: str) -> str:
+    """Slug from the org name; reserved labels and collisions get -2, -3, …"""
+    repo = HelpCenterRepository(db)
+    base = slugify_org_name(name)
+    if base in settings.HELP_CENTER_RESERVED_SLUGS:
+        base = f"{base}-help"[:SLUG_MAX_LENGTH]
+    candidate = base
+    suffix = 2
+    while repo.slug_exists(candidate) or candidate in settings.HELP_CENTER_RESERVED_SLUGS:
+        tail = f"-{suffix}"
+        candidate = f"{base[:SLUG_MAX_LENGTH - len(tail)]}{tail}"
+        suffix += 1
+    return candidate
+
+
+def default_agent_id(db: Session, organization_id: UUID) -> Optional[UUID]:
+    """The org's only active agent; else the first agent with linked knowledge;
+    else None (user picks manually)."""
+    active_agents = (
+        db.query(Agent)
+        .filter(Agent.organization_id == organization_id, Agent.is_active.is_(True))
+        .order_by(Agent.id)
+        .all()
+    )
+    if len(active_agents) == 1:
+        return active_agents[0].id
+    with_knowledge = (
+        db.query(Agent)
+        .join(KnowledgeToAgent, KnowledgeToAgent.agent_id == Agent.id)
+        .filter(Agent.organization_id == organization_id)
+        .order_by(Agent.id)
+        .first()
+    )
+    return with_knowledge.id if with_knowledge else None
+
+
+def get_or_create_settings(db: Session, organization) -> HelpCenterSettings:
+    """Fetch the org's help-center settings, creating the row (slug + agent
+    auto-default) on first access."""
+    repo = HelpCenterRepository(db)
+    row = repo.get_by_org(organization.id)
+    if row:
+        return row
+    row = HelpCenterSettings(
+        organization_id=organization.id,
+        slug=generate_unique_slug(db, organization.name),
+        agent_id=default_agent_id(db, organization.id),
+    )
+    created = repo.create(row)
+    logger.info(f"Created help center settings for org {organization.id} (slug={created.slug})")
+    return created
+
+
+def live_url(row: HelpCenterSettings) -> str:
+    """The public URL the help center is (or will be) served at.
+
+    A verified custom domain always wins. Otherwise the URL shape follows
+    HELP_CENTER_PUBLIC_MODE: "subdomain" advertises {slug}.<base> (cloud); the
+    default "path" mode serves same-origin as the API at {BACKEND_URL}/help/{slug}
+    (self-host — no DNS/TLS/proxy needed, and correct scheme+port via BACKEND_URL)."""
+    if row.domain_verified:
+        return f"https://{row.custom_domain}"
+    if settings.HELP_CENTER_PUBLIC_MODE == "subdomain":
+        return f"https://{row.slug}.{settings.HELP_CENTER_BASE_DOMAIN}"
+    return f"{settings.BACKEND_URL.rstrip('/')}/help/{row.slug}"

backend/app/services/jira.py

@@ -0,0 +1,362 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from datetime import datetime, timedelta
+from typing import Optional, Dict, Any, List
+import requests
+from urllib.parse import urlencode
+
+from app.models.jira import JiraToken
+from app.core.config import settings
+from app.core.exceptions import JiraAuthError
+from app.core.external_urls import external_base
+from app.core.logger import get_logger
+logger = get_logger(__name__)
+
+# Atlassian hosts. Deployments never set these; the e2e suite points them at its
+# recording fake. The authorize URL is browser-facing, the rest are server calls.
+JIRA_AUTH_BASE = external_base("JIRA_AUTH_BASE", "https://auth.atlassian.com")
+JIRA_API_BASE = external_base("JIRA_API_BASE", "https://api.atlassian.com")
+
+
+def jira_authorize_url() -> str:
+    return f"{JIRA_AUTH_BASE}/authorize"
+
+
+def jira_token_url() -> str:
+    return f"{JIRA_AUTH_BASE}/oauth/token"
+
+
+def jira_accessible_resources_url() -> str:
+    return f"{JIRA_API_BASE}/oauth/token/accessible-resources"
+
+
+def jira_rest_url(cloud_id: str, path: str) -> str:
+    """Jira Cloud REST v3 URL, e.g. `jira_rest_url(cloud_id, "issue")`."""
+    return f"{JIRA_API_BASE}/ex/jira/{cloud_id}/rest/api/3/{path}"
+
+
+class JiraService:
+    """Service for handling Jira OAuth 2.0 flow and API interactions."""
+    
+    def __init__(self):
+        self.client_id = settings.JIRA_CLIENT_ID
+        self.client_secret = settings.JIRA_CLIENT_SECRET
+        self.redirect_uri = settings.JIRA_REDIRECT_URI
+        self.auth_url = jira_authorize_url()
+        self.token_url = jira_token_url()
+        self.scope = "read:jira-work write:jira-work read:jira-user offline_access"
+
+    def get_authorization_url(self, state: str) -> str:
+        """Generate the authorization URL for Jira OAuth flow."""
+        params = {
+            "audience": "api.atlassian.com",
+            "client_id": self.client_id,
+            "scope": self.scope,
+            "redirect_uri": self.redirect_uri,
+            "state": state,
+            "response_type": "code",
+            "prompt": "consent"
+        }
+        return f"{self.auth_url}?{urlencode(params)}"
+
+    def validate_token(self, token: JiraToken) -> bool:
+        """Check if the token is valid and not expired."""
+        if token is None:
+            return False
+        return token.expires_at > datetime.utcnow() + timedelta(minutes=5)
+
+    async def exchange_code_for_token(self, code: str) -> Dict[str, Any]:
+        """Exchange authorization code for access token."""
+        data = {
+            "grant_type": "authorization_code",
+            "client_id": self.client_id,
+            "client_secret": self.client_secret,
+            "code": code,
+            "redirect_uri": self.redirect_uri
+        }
+
+        response = requests.post(self.token_url, data=data)
+        if response.status_code != 200:
+            raise JiraAuthError("Failed to exchange code for token")
+        
+        token_data = response.json()
+        return {
+            "access_token": token_data["access_token"],
+            "refresh_token": token_data["refresh_token"],
+            "token_type": token_data["token_type"],
+            "expires_at": datetime.utcnow() + timedelta(seconds=token_data["expires_in"])
+        }
+
+    async def refresh_token(self, refresh_token: str) -> Dict[str, Any]:
+        """Refresh an expired access token."""
+        data = {
+            "grant_type": "refresh_token",
+            "client_id": self.client_id,
+            "client_secret": self.client_secret,
+            "refresh_token": refresh_token
+        }
+
+        response = requests.post(self.token_url, data=data)
+        if response.status_code != 200:
+            raise JiraAuthError("Failed to refresh token")
+
+        token_data = response.json()
+        return {
+            "access_token": token_data["access_token"],
+            "refresh_token": token_data.get("refresh_token", refresh_token),
+            "token_type": token_data["token_type"],
+            "expires_at": datetime.utcnow() + timedelta(seconds=token_data["expires_in"])
+        }
+
+    async def get_cloud_id(self, access_token: str) -> Dict[str, str]:
+        """Get Jira Cloud ID and site URL for the authenticated user."""
+        headers = {
+            "Authorization": f"Bearer {access_token}",
+            "Accept": "application/json"
+        }
+        response = requests.get(jira_accessible_resources_url(), headers=headers)
+        
+        if response.status_code != 200:
+            raise JiraAuthError("Failed to get Jira Cloud ID")
+
+        resources = response.json()
+        if not resources:
+            raise JiraAuthError("No Jira Cloud instances found")
+
+        # Use the first available cloud instance
+        return {
+            "cloud_id": resources[0]["id"],
+            "site_url": resources[0]["url"]
+        }
+
+    async def get_projects(self, access_token: str, cloud_id: str) -> List[Dict[str, Any]]:
+        """Get all projects from Jira."""
+        headers = {
+            "Authorization": f"Bearer {access_token}",
+            "Accept": "application/json",
+            "Content-Type": "application/json"
+        }
+        
+        url = jira_rest_url(cloud_id, "project")
+        response = requests.get(url, headers=headers)
+        
+        if response.status_code != 200:
+            raise JiraAuthError(f"Failed to get Jira projects: {response.text}")
+        
+        projects = response.json()
+        return [
+            {
+                "id": project["id"],
+                "key": project["key"],
+                "name": project["name"]
+            }
+            for project in projects
+        ]
+    
+    async def get_issue_types(self, access_token: str, cloud_id: str, project_key: str) -> List[Dict[str, Any]]:
+        """Get all issue types for a project from Jira."""
+        headers = {
+            "Authorization": f"Bearer {access_token}",
+            "Accept": "application/json",
+            "Content-Type": "application/json"
+        }
+        
+        # First, get the project to find its ID
+        url = jira_rest_url(cloud_id, f"project/{project_key}")
+        response = requests.get(url, headers=headers)
+        
+        if response.status_code != 200:
+            raise JiraAuthError(f"Failed to get Jira project: {response.text}")
+        
+        project = response.json()
+        
+        # Get issue types for the project, filtering out sub-tasks
+        issue_types = []
+        for issue_type in project.get("issueTypes", []):
+            # Skip sub-tasks as they require a parent issue
+            if issue_type.get("subtask", False):
+                logger.info(f"Skipping sub-task issue type: {issue_type['name']} (ID: {issue_type['id']})")
+                continue
+                
+            issue_types.append({
+                "id": issue_type["id"],
+                "name": issue_type["name"],
+                "description": issue_type.get("description", ""),
+                "iconUrl": issue_type.get("iconUrl", "")
+            })
+        
+        return issue_types
+        
+    async def create_issue(self, organization, db, issue_data):
+        """Wrapper method for create_issue that takes organization, db, and issue_data parameters."""
+        from app.models.jira import JiraToken
+        
+        token = db.query(JiraToken).filter(
+            JiraToken.organization_id == organization.id
+        ).first()
+        
+        if not token:
+            raise JiraAuthError("No Jira connection found")
+        
+        # Check if token is valid and refresh if needed
+        is_valid = self.validate_token(token)
+        if not is_valid:
+            try:
+                token_data = await self.refresh_token(token.refresh_token)
+                
+                # Update token in database
+                for key, value in token_data.items():
+                    setattr(token, key, value)
+                
+                db.commit()
+            except Exception as e:
+                logger.error(f"Failed to refresh Jira token: {e}")
+                raise JiraAuthError("Jira token expired and could not be refreshed")
+        
+        # Create the issue
+        result = await self.create_issue_internal(
+            token.access_token,
+            token.cloud_id,
+            issue_data.projectKey,
+            issue_data.issueTypeId,
+            issue_data.summary,
+            issue_data.description,
+            issue_data.priority
+        )
+        
+        return result
+    
+    async def create_issue_internal(self, access_token: str, cloud_id: str, project_key: str, issue_type_id: str, 
+                          summary: str, description: str, priority_id: Optional[str] = None) -> Dict[str, Any]:
+        """Create a new issue in Jira."""
+        headers = {
+            "Authorization": f"Bearer {access_token}",
+            "Accept": "application/json",
+            "Content-Type": "application/json"
+        }
+        
+        url = jira_rest_url(cloud_id, "issue")
+        
+        data = {
+            "fields": {
+                "project": {
+                    "key": project_key
+                },
+                "issuetype": {
+                    "id": issue_type_id
+                },
+                "summary": summary,
+                "description": {
+                    "type": "doc",
+                    "version": 1,
+                    "content": [
+                        {
+                            "type": "paragraph",
+                            "content": [
+                                {
+                                    "type": "text",
+                                    "text": description
+                                }
+                            ]
+                        }
+                    ]
+                }
+            }
+        }
+        
+        # Add priority if provided and available
+        if priority_id:
+            # Check if priority field is available for this project/issue type
+            priority_available = await self.is_priority_available(access_token, cloud_id, project_key, issue_type_id)
+            if priority_available:
+                data["fields"]["priority"] = {
+                    "id": priority_id
+                }
+            else:
+                logger.warning(f"Priority field not available for project {project_key} and issue type {issue_type_id}. Skipping priority.")
+        
+        response = requests.post(url, headers=headers, json=data)
+        
+        if response.status_code not in (200, 201):
+            raise JiraAuthError(f"Failed to create Jira issue: {response.text}")
+        
+        return response.json()
+        
+    async def get_priorities(self, access_token: str, cloud_id: str) -> List[Dict[str, Any]]:
+        """Get all priorities from Jira."""
+        headers = {
+            "Authorization": f"Bearer {access_token}",
+            "Accept": "application/json",
+            "Content-Type": "application/json"
+        }
+        
+        url = jira_rest_url(cloud_id, "priority")
+        response = requests.get(url, headers=headers)
+        
+        if response.status_code != 200:
+            raise JiraAuthError(f"Failed to get Jira priorities: {response.text}")
+        
+        priorities = response.json()
+        return [
+            {
+                "id": priority["id"],
+                "name": priority["name"],
+                "description": priority.get("description", ""),
+                "iconUrl": priority.get("iconUrl", "")
+            }
+            for priority in priorities
+        ]
+        
+    async def check_field_availability(self, access_token: str, cloud_id: str, project_key: str, issue_type_id: str, field_name: str) -> bool:
+        """Check if a field is available for a project/issue type combination."""
+        headers = {
+            "Authorization": f"Bearer {access_token}",
+            "Accept": "application/json",
+            "Content-Type": "application/json"
+        }
+        
+        # Get the create metadata for the project/issue type
+        url = jira_rest_url(cloud_id, f"issue/createmeta?projectKeys={project_key}&issuetypeIds={issue_type_id}&expand=projects.issuetypes.fields")
+        response = requests.get(url, headers=headers)
+        
+        if response.status_code != 200:
+            logger.error(f"Failed to get create metadata: {response.text}")
+            return False
+        
+        metadata = response.json()
+        
+        # Navigate through the response structure to find the field
+        try:
+            projects = metadata.get("projects", [])
+            if not projects:
+                return False
+                
+            issue_types = projects[0].get("issuetypes", [])
+            if not issue_types:
+                return False
+                
+            fields = issue_types[0].get("fields", {})
+            
+            # Check if the field exists
+            return field_name in fields
+        except Exception as e:
+            logger.error(f"Error checking field availability: {e}")
+            return False
+            
+    async def is_priority_available(self, access_token: str, cloud_id: str, project_key: str, issue_type_id: str) -> bool:
+        """Check if the priority field is available for a project/issue type combination."""
+        return await self.check_field_availability(access_token, cloud_id, project_key, issue_type_id, "priority") 

backend/app/services/meta_own_app.py

@@ -0,0 +1,189 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+WhatsApp numbers and Messenger Pages that live in the customer's own Meta
+app (connected with manual credentials) rather than in ReplyPier's.
+
+Meta signs their webhooks with the customer's app secret, which the shared
+/webhooks/meta endpoint can't check. So each such account gets its own
+webhook address, checked with its app's secret only. That address takes
+events for the account and for the same organization's other accounts from
+the same app (one Meta app has one Messenger callback, and possibly several
+numbers), never for anyone else's: a payload signed with a secret someone
+chose themselves can't reach another organization.
+
+A WhatsApp number is pointed at its address with a per-number callback
+override; Messenger has no override, so the customer pastes the address and
+verify token into their app.
+"""
+
+import hashlib
+import hmac
+from dataclasses import dataclass
+from typing import Callable, Optional
+
+from sqlalchemy.orm import Session
+
+from app.channels.meta_base import graph_detail, graph_get, graph_post_json
+from app.core.config import settings, webhooks_base_url
+from app.core.logger import get_logger
+from app.models.channels import ChannelAccount, ChannelType
+from app.core.security import decrypt_credentials
+from app.repositories.channels.accounts import ChannelAccountRepository
+
+logger = get_logger(__name__)
+
+# In the account's encrypted credentials
+APP_SECRET_KEY = "app_secret"
+# In its (non-secret) settings: which app it's from, and whether Meta
+# accepted our per-number callback for it
+APP_ID_SETTING = "meta_app_id"
+AUTO_WEBHOOK_SETTING = "webhook_auto_configured"
+OWN_APP_CHANNELS = (ChannelType.WHATSAPP.value, ChannelType.MESSENGER.value)
+
+# Meta's answer to an appsecret_proof made with another app's secret
+_PROOF_ERROR = "appsecret_proof"
+
+
+class AppSecretMismatch(Exception):
+    """The secret isn't the one of the app that issued the token."""
+
+
+class AppCheckUnavailable(Exception):
+    """Meta couldn't be asked (network, rate limit, an expired token…)."""
+
+
+@dataclass
+class OwnApp:
+    app_id: str
+    secret: str
+
+
+def is_own_app(account: Optional[ChannelAccount]) -> bool:
+    return (account is not None and account.channel_type in OWN_APP_CHANNELS
+            and bool((account.settings or {}).get(APP_ID_SETTING)))
+
+
+def own_app_secret(account: ChannelAccount) -> Optional[str]:
+    return decrypt_credentials(account.encrypted_credentials).get(APP_SECRET_KEY) if is_own_app(account) else None
+
+
+def own_app_token(account: Optional[ChannelAccount]) -> Optional[str]:
+    """The customer's token, read before a reconnect replaces it: only it can
+    take back a callback override made with it."""
+    return decrypt_credentials(account.encrypted_credentials).get("access_token") if is_own_app(account) else None
+
+
+def own_app_webhook_url(account: ChannelAccount) -> str:
+    return f"{webhooks_base_url()}/meta/{account.id}"
+
+
+async def resolve_own_app(access_token: str, entered_secret: Optional[str],
+                          previous: Optional[ChannelAccount]) -> Optional[OwnApp]:
+    """The customer's own app behind this token, proven with its secret —
+    the one entered, or else the one kept from the last connect (checked
+    again: the new token may be from another app). None without a secret, or
+    for ReplyPier's own app, whose webhooks the shared address already gets.
+
+    Raises AppSecretMismatch or AppCheckUnavailable."""
+    secret = entered_secret or (own_app_secret(previous) if previous is not None else None)
+    if not secret:
+        return None
+    proof = hmac.new(secret.encode(), access_token.encode(), hashlib.sha256).hexdigest()
+    # /app (the token's own app) needs no permission on the Page or number
+    ok, data = await graph_get("app", access_token, params={"fields": "id", "appsecret_proof": proof})
+    if not ok:
+        error = data.get("error") if isinstance(data.get("error"), dict) else {}
+        if _PROOF_ERROR in str(error.get("message", "")):
+            raise AppSecretMismatch()
+        # Meta's own errors carry a code; anything else is ours (a timeout,
+        # a socket error) and stays in the log
+        logger.warning(f"Could not check an app secret with Meta: {error.get('message')}")
+        raise AppCheckUnavailable(graph_detail(data, "") if "code" in error else "Meta did not answer")
+    app_id = str(data.get("id") or "")
+    if not app_id or app_id == settings.META_APP_ID:
+        return None
+    return OwnApp(app_id=app_id, secret=secret)
+
+
+async def sync_own_app(db: Session, account: ChannelAccount, own: Optional[OwnApp],
+                       access_token: str, previous_token: Optional[str]) -> None:
+    """After a connect: route the account's webhooks to its own address when
+    it's from the customer's app, or hand a number that used to be back."""
+    repo = ChannelAccountRepository(db)
+    is_whatsapp = account.channel_type == ChannelType.WHATSAPP.value
+    if own is not None:
+        # Moved to another app: the old app's override is still the old app's
+        if is_whatsapp and previous_token and (account.settings or {}).get(APP_ID_SETTING) != own.app_id:
+            await _set_number_callback(account, previous_token, "", None)
+        # Saved first: Meta checks the address during the callback call, and
+        # the address only answers for an account marked as from its own app
+        repo.update_settings(account, {APP_ID_SETTING: own.app_id})
+        if is_whatsapp:
+            pointed = await _set_number_callback(
+                account, access_token, own_app_webhook_url(account), account.webhook_secret)
+            repo.update_settings(account, {AUTO_WEBHOOK_SETTING: pointed})
+        return
+    if previous_token is not None:
+        if is_whatsapp:
+            await _set_number_callback(account, previous_token, "", None)
+        repo.update_settings(account, {}, remove=(APP_ID_SETTING, AUTO_WEBHOOK_SETTING))
+
+
+async def leave_own_app(account: ChannelAccount) -> None:
+    """Before a disconnect: send the number's webhooks back to the customer's
+    app. Best effort — a disconnect must never fail over it."""
+    if account.channel_type != ChannelType.WHATSAPP.value:
+        return
+    try:
+        token = own_app_token(account)
+    except Exception:
+        logger.exception(f"Could not read {account.id}'s credentials to release its callback")
+        return
+    if token:
+        await _set_number_callback(account, token, "", None)
+
+
+def same_app_accounts(db: Session, account: ChannelAccount) -> Callable[[str], Optional[ChannelAccount]]:
+    """Resolve an external id named in a webhook to one of this organization's
+    accounts from the same app (and channel), or to nothing."""
+    repo = ChannelAccountRepository(db)
+    app_id = account.settings[APP_ID_SETTING]
+
+    def resolve(external_id: str) -> Optional[ChannelAccount]:
+        if external_id == account.external_account_id:
+            return account
+        other = repo.get_by_external_id(account.channel_type, external_id)
+        if (other is not None and other.organization_id == account.organization_id
+                and (other.settings or {}).get(APP_ID_SETTING) == app_id):
+            return other
+        return None
+    return resolve
+
+
+async def _set_number_callback(account: ChannelAccount, access_token: str, url: str,
+                               verify_token: Optional[str]) -> bool:
+    """Point a WhatsApp number's webhooks at `url` ("" sends them back to the
+    app's own callback). Meta checks the address during the call."""
+    configuration = {"override_callback_uri": url}
+    if verify_token:
+        configuration["verify_token"] = verify_token
+    ok, data = await graph_post_json(account.external_account_id, access_token,
+                                     {"webhook_configuration": configuration})
+    if not ok:
+        action = "point" if url else "release"
+        logger.warning(f"Could not {action} the webhook callback of {account.external_account_id}: "
+                       f"{graph_detail(data, 'no reason given')}")
+    return ok

backend/app/services/notifications.py

@@ -0,0 +1,114 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import enum
+from typing import Iterable, Optional
+
+from sqlalchemy.orm import Session
+
+from app.core.logger import get_logger
+from app.models.notification import Notification, NotificationType
+from app.models.notification_settings import (
+    NOTIFY_CHAT_ASSIGNED,
+    NOTIFY_CHAT_TRANSFER,
+    NOTIFY_NEW_CHAT,
+)
+from app.repositories.notification_settings import UserNotificationSettingsRepository
+from app.services.user import send_fcm_notification
+
+logger = get_logger(__name__)
+
+
+class ChatNotificationEvent(str, enum.Enum):
+    """Chat events a user can mute. The value is both the preference column
+    and the `event` discriminator carried in the push metadata.
+    """
+    NEW_CHAT = NOTIFY_NEW_CHAT
+    CHAT_TRANSFER = NOTIFY_CHAT_TRANSFER
+    CHAT_ASSIGNED = NOTIFY_CHAT_ASSIGNED
+
+
+async def notify_user(
+    db: Session,
+    user_id,
+    type_: NotificationType,
+    title: str,
+    message: str,
+    metadata: Optional[dict] = None,
+) -> None:
+    """Persist an in-app notification and send its FCM push.
+
+    Never raises and no-ops without a user (background jobs may have none) —
+    a notification failure must not fail the work it reports on.
+
+    NOTE: this commits (and, on error, rolls back) the caller's `db` session,
+    so call it only once any work you want persisted is already committed.
+    Every current caller either commits first or has nothing pending.
+    """
+    if not user_id:
+        return
+    try:
+        notification = Notification(
+            user_id=user_id,
+            type=type_,
+            title=title,
+            message=message,
+            notification_metadata=metadata,
+        )
+        db.add(notification)
+        db.commit()
+        await send_fcm_notification(user_id, notification, db)
+    except Exception as e:
+        logger.error(f"Error sending notification '{title}' to user {user_id}: {e}")
+        try:
+            db.rollback()
+        except Exception:
+            pass
+
+
+async def notify_chat_event(
+    db: Session,
+    user_ids: Iterable,
+    event: ChatNotificationEvent,
+    title: str,
+    message: str,
+    metadata: Optional[dict] = None,
+) -> None:
+    """Notify every user in `user_ids` who hasn't muted `event`.
+
+    Recipients are resolved in a single query, so fanning out to a whole group
+    or org costs one lookup rather than one per user. Like notify_user, this
+    never raises — a notification must not fail the chat it reports on.
+    """
+    try:
+        recipients = UserNotificationSettingsRepository(db).filter_enabled(
+            user_ids, event.value
+        )
+        if not recipients:
+            return
+
+        payload = {**(metadata or {}), "event": event.value}
+        for user_id in recipients:
+            await notify_user(
+                db=db,
+                user_id=user_id,
+                type_=NotificationType.CHAT,
+                title=title,
+                message=message,
+                metadata=payload,
+            )
+    except Exception as e:
+        logger.error(f"Error sending '{event.value}' notification: {e}")

backend/app/services/shopify_auth_service.py

@@ -0,0 +1,82 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from sqlalchemy.orm import Session
+from typing import Optional, Tuple, List, Dict, Any
+from app.core.logger import get_logger
+from app.repositories.agent_shopify_config_repository import AgentShopifyConfigRepository
+from app.repositories.widget import WidgetRepository
+from app.services.shopify import ShopifyService
+from app.repositories.agent import AgentRepository
+from fastapi import Request
+from fastapi.responses import RedirectResponse, HTMLResponse
+from app.models.organization import Organization
+from app.core.config import settings
+from urllib.parse import quote
+import hmac
+import hashlib
+import base64
+
+logger = get_logger(__name__)
+
+
+class ShopifyAuthService:
+    """Service for handling Shopify authentication and HTML generation"""
+    
+    @staticmethod
+    async def get_shop_success_data(db: Session, shop_id: str) -> Tuple[int, Optional[str]]:
+        """
+        Get success page data for a shop (agent count and widget ID).
+        
+        Args:
+            db: Database session
+            shop_id: Shop ID
+            
+        Returns:
+            Tuple of (agents_connected, widget_id)
+        """
+        agent_config_repository = AgentShopifyConfigRepository(db)
+        configs = agent_config_repository.get_configs_by_shop(shop_id, enabled_only=True)
+        agents_connected = len(configs) if configs else 0
+        
+        # Get widget ID from the first connected agent
+        widget_id = None
+        if configs and len(configs) > 0:
+            widget_repo = WidgetRepository(db)
+            widgets = widget_repo.get_widgets_by_agent(configs[0].agent_id)
+            if widgets and len(widgets) > 0:
+                widget_id = str(widgets[0].id)
+        
+        return agents_connected, widget_id
+    
+    @staticmethod
+    async def verify_shop_token(db: Session, db_shop) -> bool:
+        """
+        Verify if a shop's access token is valid.
+        
+        Args:
+            db: Database session
+            db_shop: Shop database record
+            
+        Returns:
+            True if token is valid, False otherwise
+        """
+        shopify_service = ShopifyService(db)
+        validation_query = "query { shop { name } }"
+        validation_result = await shopify_service._execute_graphql_async(db_shop, validation_query)
+        return validation_result.get("success", False)
+    
+

backend/app/services/shopify_helper_service.py

@@ -0,0 +1,209 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from sqlalchemy.orm import Session
+from typing import Optional
+from fastapi import Request
+from fastapi.responses import RedirectResponse, HTMLResponse
+from app.core.logger import get_logger
+from app.core.config import settings
+from app.models.organization import Organization
+from app.repositories.widget import WidgetRepository
+from app.services.shopify_auth_service import ShopifyAuthService
+from app.services.shopify_urls import shopify_oauth_token_url
+from urllib.parse import quote
+import hmac
+import hashlib
+import base64
+
+logger = get_logger(__name__)
+
+
+class ShopifyHelperService:
+    """Service for Shopify helper functions and utilities"""
+    
+    @staticmethod
+    def get_shop_agents_list(db: Session, organization_id: str) -> list[dict]:
+        """
+        Get all agents for a shop's organization and convert to dict format.
+        
+        Args:
+            db: Database session
+            organization_id: Organization ID (string UUID)
+            
+        Returns:
+            List of agent dictionaries with id, name, display_name, description, is_active
+        """
+        from app.repositories.agent import AgentRepository
+        agent_repository = AgentRepository(db)
+        agents = agent_repository.get_active_agents(organization_id)
+        
+        # Convert agents to dict format
+        agents_list = [
+            {
+                "id": str(agent.id),
+                "name": agent.name,
+                "display_name": agent.display_name,
+                "description": agent.description,
+                "is_active": agent.is_active
+            }
+            for agent in agents
+        ]
+        
+        return agents_list
+
+    @staticmethod
+    def get_shop_widget_id(db: Session, configs: list, default: str = None) -> Optional[str]:
+        """
+        Get the widget ID for the first configured agent.
+        
+        Args:
+            db: Database session
+            configs: List of agent Shopify configs
+            default: Default value if no widget found
+            
+        Returns:
+            Widget ID as string or None
+        """
+        if configs and len(configs) > 0:
+            widget_repo = WidgetRepository(db)
+            widgets = widget_repo.get_widgets_by_agent(configs[0].agent_id)
+            if widgets and len(widgets) > 0:
+                return str(widgets[0].id)
+        return default
+
+    @staticmethod
+    def exchange_oauth_code_for_token(shop: str, code: str) -> tuple[str, str]:
+        """
+        Exchange OAuth authorization code for access token.
+        
+        Args:
+            shop: Shop domain
+            code: OAuth authorization code
+            
+        Returns:
+            Tuple of (access_token, scope)
+            
+        Raises:
+            HTTPException: If token exchange fails
+        """
+        import requests
+        from fastapi import HTTPException
+        
+        token_url = shopify_oauth_token_url(shop)
+        payload = {
+            "client_id": settings.SHOPIFY_API_KEY,
+            "client_secret": settings.SHOPIFY_API_SECRET,
+            "code": code
+        }
+        
+        # Check if we should verify SSL certificates (default to True for production)
+        verify_ssl = settings.VERIFY_SSL_CERTIFICATES
+        logger.info(f"Exchanging OAuth code for token - Shop: {shop}")
+        logger.info(f"Verify SSL Certificates: {verify_ssl}")
+        logger.info(f"Token URL: {token_url}")
+        
+        try:
+            response = requests.post(token_url, json=payload, verify=verify_ssl)
+            response.raise_for_status()
+            token_data = response.json()
+            
+            access_token = token_data.get("access_token")
+            scope = token_data.get("scope")
+            
+            if not access_token:
+                logger.error(f"Failed to get access token for shop: {shop}")
+                raise HTTPException(status_code=400, detail="Failed to obtain access token")
+            
+            logger.info(f"Successfully obtained access token for shop: {shop}")
+            return access_token, scope
+            
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Error exchanging OAuth code for token: {str(e)}")
+            if isinstance(e, requests.exceptions.SSLError):
+                logger.error("SSL Certificate verification failed. If this is a development environment, consider setting VERIFY_SSL_CERTIFICATES=False in settings.")
+            raise HTTPException(status_code=500, detail=f"Failed to exchange OAuth code: {str(e)}")
+
+    @staticmethod
+    def verify_shopify_webhook(request_headers, request_body) -> bool:
+        """
+        Validate Shopify HMAC signature for webhooks.
+        
+        Args:
+            request_headers: Request headers containing X-Shopify-Hmac-Sha256
+            request_body: Raw request body bytes
+            
+        Returns:
+            True if signature is valid, False otherwise
+        """
+        shopify_hmac = request_headers.get('X-Shopify-Hmac-Sha256')
+        if not shopify_hmac:
+            return False
+        
+        digest = hmac.new(
+            settings.SHOPIFY_API_SECRET.encode('utf-8'),
+            request_body,
+            hashlib.sha256
+        ).digest()
+        
+        computed_hmac = base64.b64encode(digest).decode('utf-8')
+        return hmac.compare_digest(computed_hmac, shopify_hmac)
+
+    @staticmethod
+    def validate_shop_request(request: Request, shop: str, hmac_param: str) -> bool:
+        """
+        Validate the OAuth request from Shopify using HMAC validation.
+        
+        Args:
+            request: FastAPI Request object
+            shop: Shop domain
+            hmac_param: HMAC parameter from query string
+            
+        Returns:
+            True if request is valid, False otherwise
+        """
+        # Check if the shop URL is a valid Shopify domain
+        if not shop.endswith('.myshopify.com'):
+            logger.warning(f"Invalid shop domain: {shop}")
+            return False
+        
+        # Get query parameters as dictionary
+        query_params = dict(request.query_params)
+        
+        # Remove hmac from params for validation
+        if 'hmac' in query_params:
+            query_params.pop('hmac')
+        
+        logger.info(f"Query params: {query_params}")
+        logger.info(f"HMAC param: {hmac_param}")
+        
+        # Sort and encode parameters
+        sorted_params = "&".join([f"{key}={quote(value)}" for key, value in sorted(query_params.items())])
+        
+        # Generate HMAC
+        digest = hmac.new(
+            settings.SHOPIFY_API_SECRET.encode('utf-8'),
+            sorted_params.encode('utf-8'),
+            hashlib.sha256
+        ).hexdigest()
+        
+        # Compare with provided HMAC
+        is_valid = hmac.compare_digest(digest, hmac_param)
+        if not is_valid:
+            logger.warning(f"HMAC validation failed for shop: {shop}")
+        
+        return is_valid
+

backend/app/services/shopify_session.py

@@ -0,0 +1,241 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import jwt
+import time
+from typing import Optional, Dict, Any
+from fastapi import Request, HTTPException, Depends
+from sqlalchemy.orm import Session
+from app.core.config import settings
+from app.core.logger import get_logger
+from app.database import get_db
+
+logger = get_logger(__name__)
+
+
+class ShopifySessionService:
+    """Service for handling Shopify session token validation"""
+    
+    @staticmethod
+    def get_session_token_from_request(request: Request) -> Optional[str]:
+        """
+        Get session token from request header or URL parameter.
+        Per Shopify docs, session token can be in:
+        1. Authorization header: Bearer <token>
+        2. URL query parameter: id_token
+        """
+        # Try to get from Authorization header first
+        auth_header = request.headers.get('authorization', '')
+        if auth_header.startswith('Bearer '):
+            token = auth_header.replace('Bearer ', '').strip()
+            if token:
+                return token
+        
+        # Try to get from URL query parameter
+        id_token = request.query_params.get('id_token')
+        if id_token:
+            return id_token
+        
+        return None
+    
+    @staticmethod
+    def validate_session_token(token: str) -> Dict[str, Any]:
+        """
+        Validate and decode Shopify session token.
+        Returns decoded token payload if valid.
+        Raises HTTPException if invalid.
+        """
+        try:
+            # Decode without verification first to get the shop
+            unverified = jwt.decode(token, options={"verify_signature": False})
+            
+            # Verify the token with Shopify's public key
+            # Note: In production, you should fetch and cache Shopify's public keys
+            # For now, we'll do basic validation
+            decoded = jwt.decode(
+                token,
+                settings.SHOPIFY_API_SECRET,
+                algorithms=["HS256"],
+                audience=settings.SHOPIFY_API_KEY
+            )
+            
+            # Check expiration
+            exp = decoded.get('exp')
+            if exp and exp < time.time():
+                raise HTTPException(status_code=401, detail="Session token expired")
+            
+            # Check nbf (not before)
+            nbf = decoded.get('nbf')
+            if nbf and nbf > time.time():
+                raise HTTPException(status_code=401, detail="Session token not yet valid")
+            
+            logger.info(f"Session token validated for shop: {decoded.get('dest')}")
+            return decoded
+            
+        except jwt.ExpiredSignatureError:
+            logger.warning("Session token expired")
+            raise HTTPException(
+                status_code=401,
+                detail="Session token expired",
+                headers={"X-Shopify-Retry-Invalid-Session-Request": "1"}
+            )
+        except jwt.InvalidTokenError as e:
+            logger.warning(f"Invalid session token: {str(e)}")
+            raise HTTPException(
+                status_code=401,
+                detail="Invalid session token",
+                headers={"X-Shopify-Retry-Invalid-Session-Request": "1"}
+            )
+    
+    @staticmethod
+    def is_document_request(request: Request) -> bool:
+        """
+        Check if this is a document request (initial page load) vs API request.
+        Document requests typically don't have Authorization header.
+        """
+        return not request.headers.get("authorization")
+    
+
+# FastAPI dependency for session token validation
+async def require_shopify_session(
+    request: Request,
+    db: Session = Depends(get_db)
+) -> Dict[str, Any]:
+    """
+    FastAPI dependency to validate Shopify session token.
+    Extracts shop from token and looks up organization.
+    Returns decoded token with db_shop and organization_id.
+    
+    Usage:
+        @router.get("/some-endpoint")
+        async def some_endpoint(
+            shopify_session: dict = Depends(require_shopify_session),
+            db: Session = Depends(get_db)
+        ):
+            shop_id = shopify_session['shop_id']
+            org_id = shopify_session['organization_id']
+            db_shop = shopify_session['db_shop']
+            # ... use the shop info
+    """
+    from app.repositories.shopify_shop_repository import ShopifyShopRepository
+    
+    session_token = ShopifySessionService.get_session_token_from_request(request)
+    
+    if not session_token:
+        raise HTTPException(
+            status_code=401,
+            detail="Session token required for embedded app",
+            headers={"X-Shopify-Retry-Invalid-Session-Request": "1"}
+        )
+    
+    # Validate and decode token
+    decoded = ShopifySessionService.validate_session_token(session_token)
+    
+    # Extract shop domain from token
+    shop_domain = decoded.get('dest', '').replace('https://', '').replace('http://', '')
+    
+    # Look up shop in database
+    shop_repository = ShopifyShopRepository(db)
+    db_shop = shop_repository.get_shop_by_domain(shop_domain)
+    
+    if not db_shop or not db_shop.is_installed:
+        raise HTTPException(status_code=403, detail="Shop not installed or not found")
+    
+    # Add shop info to decoded token for easy access
+    decoded['db_shop'] = db_shop
+    decoded['shop_id'] = str(db_shop.id)
+    decoded['shop_domain'] = shop_domain
+    decoded['organization_id'] = db_shop.organization_id if db_shop.organization_id else None
+
+    logger.info(f"Session token validated for shop: {shop_domain}, org: {decoded['organization_id']}")
+
+    return decoded
+
+
+async def require_shopify_or_jwt_auth(
+    request: Request,
+    db: Session = Depends(get_db)
+) -> Dict[str, Any]:
+    """
+    Hybrid authentication dependency that tries Shopify session token first,
+    then falls back to JWT authentication.
+    Returns authentication info with type indicator.
+    
+    Usage:
+        @router.get("/some-endpoint")
+        async def some_endpoint(
+            auth_info: dict = Depends(require_shopify_or_jwt_auth),
+            db: Session = Depends(get_db)
+        ):
+            if auth_info['auth_type'] == 'shopify_session':
+                org_id = auth_info['organization_id']
+                shop_id = auth_info['shop_id']
+                user_id = None  # No user for session token auth
+            else:  # jwt
+                org_id = auth_info['organization_id']
+                user_id = auth_info['user_id']
+                current_user = auth_info['current_user']
+    """
+    from app.core.auth import get_current_user
+    from app.repositories.shopify_shop_repository import ShopifyShopRepository
+    
+    # First try Shopify session token
+    session_token = ShopifySessionService.get_session_token_from_request(request)
+    
+    if session_token:
+        try:
+            # Validate and decode the session token
+            decoded = ShopifySessionService.validate_session_token(session_token)
+            
+            # Extract shop domain from token
+            shop_domain = decoded.get('dest', '').replace('https://', '').replace('http://', '')
+            
+            if shop_domain:
+                # Look up shop in database
+                shop_repository = ShopifyShopRepository(db)
+                db_shop = shop_repository.get_shop_by_domain(shop_domain)
+                
+                if db_shop and db_shop.is_installed and db_shop.organization_id:
+                    logger.info(f"Using Shopify session token auth for shop: {shop_domain}")
+                    return {
+                        'auth_type': 'shopify_session',
+                        'shop_domain': shop_domain,
+                        'shop_id': str(db_shop.id),
+                        'organization_id': db_shop.organization_id,  # Keep as UUID for consistency
+                        'db_shop': db_shop,
+                        'decoded_token': decoded,
+                        'user_id': None  # No user_id for session token auth
+                    }
+        except Exception as e:
+            logger.debug(f"Shopify session token validation failed, trying JWT: {str(e)}")
+    
+    # Fall back to JWT authentication
+    try:
+        current_user = await get_current_user(request=request, db=db)
+        logger.info(f"Using JWT auth for user: {current_user.id}")
+        return {
+            'auth_type': 'jwt',
+            'organization_id': current_user.organization_id,  # Keep as UUID for consistency
+            'user_id': current_user.id,
+            'current_user': current_user
+        }
+    except Exception as e:
+        logger.error(f"Both Shopify session token and JWT authentication failed: {str(e)}")
+        raise HTTPException(
+            status_code=401,
+            detail="Authentication required"
+        )
+

backend/app/services/shopify_urls.py

@@ -0,0 +1,42 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+# Shop-scoped Shopify URLs. Every Admin API and OAuth call goes to the merchant's
+# own host (`{shop}.myshopify.com`), so there is no single base to swap. When
+# SHOPIFY_ADMIN_BASE is set (only the e2e suite does) the shop domain becomes a
+# path segment under it instead: `{base}/{shop_domain}/admin/...`.
+
+from app.core.external_urls import external_base
+
+SHOPIFY_API_VERSION = "2025-10"
+SHOPIFY_ADMIN_BASE = external_base("SHOPIFY_ADMIN_BASE", "")
+
+
+def shopify_shop_url(shop_domain: str, path: str) -> str:
+    """URL of `path` (no leading slash) on the shop's own host."""
+    if SHOPIFY_ADMIN_BASE:
+        return f"{SHOPIFY_ADMIN_BASE}/{shop_domain}/{path}"
+    return f"https://{shop_domain}/{path}"
+
+
+def shopify_admin_url(shop_domain: str, path: str) -> str:
+    """Admin API URL, e.g. `shopify_admin_url(shop, "graphql.json")`."""
+    return shopify_shop_url(shop_domain, f"admin/api/{SHOPIFY_API_VERSION}/{path}")
+
+
+def shopify_oauth_token_url(shop_domain: str) -> str:
+    """The shop's OAuth access-token endpoint (code and token exchange)."""
+    return shopify_shop_url(shop_domain, "admin/oauth/access_token")

backend/app/services/slack_events.py

@@ -0,0 +1,170 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+"""Slack App Home / Assistant lifecycle handlers.
+
+These run as FastAPI background tasks (the webhook acks within 3s and hands off
+here), so each opens its own DB session. They cover the non-message surfaces:
+the Messages-tab welcome, the Home tab view, and the Assistant sidebar — all of
+which the Slack Marketplace requires an app with the Messages tab enabled to use.
+"""
+
+from uuid import UUID
+
+from app.channels.slack import (
+    SUGGESTED_PROMPTS_TITLE,
+    STARTER_PROMPTS,
+    WELCOME_NO_AGENT,
+    WELCOME_WITH_AGENT,
+    ASSISTANT_READY_STATUS,
+    SlackAdapter,
+    build_home_view,
+    publish_home_view,
+    set_assistant_status,
+    set_suggested_prompts,
+    slack_api,
+)
+from app.core.config import settings
+from app.core.logger import get_logger
+from app.core.redis import get_redis
+from app.core.s3 import get_s3_signed_url
+from app.database import SessionLocal
+from app.repositories.agent import AgentRepository
+from app.repositories.channels import ChannelAccountRepository
+from app.repositories.channels.agent_config import AgentChannelConfigRepository
+
+logger = get_logger(__name__)
+
+# One welcome per user, forever — this flag is never meant to expire (a re-welcome
+# on a Redis flush is harmless). SETNX; TTL only as a backstop against orphans.
+_WELCOME_FLAG_TTL_SECONDS = 60 * 60 * 24 * 365
+
+
+def _assigned_agent(db, account):
+    """The single agent answering this Slack workspace, or None if unassigned."""
+    agent_id = AgentChannelConfigRepository(db).get_active_agent_id(account.id)
+    if not agent_id:
+        return None
+    return AgentRepository(db).get_agent(agent_id)
+
+
+def _welcome_text(agent) -> str:
+    if agent is None:
+        return WELCOME_NO_AGENT
+    return WELCOME_WITH_AGENT.format(agent=agent.display_name or agent.name)
+
+
+def _already_welcomed(account_id, user_id: str) -> bool:
+    """Claim the first-welcome slot for (account, user). True if already welcomed.
+
+    Without Redis we can't dedupe, so we welcome (once per process start is the
+    worst case) rather than stay silent — the review requires a welcome to be sent.
+    """
+    redis_client = get_redis()
+    if redis_client is None:
+        return False
+    try:
+        key = f"slack_home_welcomed:{account_id}:{user_id}"
+        claimed = redis_client.set(key, "1", nx=True, ex=_WELCOME_FLAG_TTL_SECONDS)
+        return not claimed
+    except Exception as e:
+        logger.error(f"Slack welcome dedupe failed (welcoming anyway): {e}")
+        return False
+
+
+async def deliver_home_welcome(account_id: UUID, user_id: str, channel: str) -> None:
+    """Send the one-time welcome DM when a user first opens the Messages tab."""
+    if not user_id or not channel:
+        return
+    if _already_welcomed(account_id, user_id):
+        return
+    db = SessionLocal()
+    try:
+        account = ChannelAccountRepository(db).get_by_id(account_id)
+        if account is None or not account.is_active:
+            return
+        text = _welcome_text(_assigned_agent(db, account))
+        await slack_api("chat.postMessage", SlackAdapter._access_token(account),
+                        {"channel": channel, "text": text})
+    except Exception as e:
+        logger.error(f"Slack Messages-tab welcome failed: {e}")
+    finally:
+        db.close()
+
+
+async def handle_assistant_thread_started(account_id: UUID, channel_id: str, thread_ts: str) -> None:
+    """On assistant-sidebar open: show a status, offer starter prompts, welcome."""
+    if not channel_id or not thread_ts:
+        return
+    db = SessionLocal()
+    try:
+        account = ChannelAccountRepository(db).get_by_id(account_id)
+        if account is None or not account.is_active:
+            return
+        token = SlackAdapter._access_token(account)
+        await set_assistant_status(token, channel_id, thread_ts, ASSISTANT_READY_STATUS)
+        await set_suggested_prompts(token, channel_id, thread_ts, STARTER_PROMPTS, SUGGESTED_PROMPTS_TITLE)
+        text = _welcome_text(_assigned_agent(db, account))
+        await slack_api("chat.postMessage", token,
+                        {"channel": channel_id, "text": text, "thread_ts": thread_ts})
+    except Exception as e:
+        logger.error(f"Slack assistant welcome failed: {e}")
+    finally:
+        db.close()
+
+
+async def _agent_card(agent) -> dict:
+    """Flatten an Agent into the resolved shape build_home_view expects."""
+    photo_url = None
+    customization = getattr(agent, "customization", None)
+    if customization and customization.photo_url:
+        photo_url = customization.photo_url
+        if settings.S3_FILE_STORAGE:
+            try:
+                photo_url = await get_s3_signed_url(customization.photo_url)
+            except Exception as e:
+                logger.debug(f"Could not sign agent photo URL: {e}")
+    instructions = agent.instructions or []
+    return {
+        "name": agent.display_name or agent.name,
+        "is_active": agent.is_active,
+        "instruction": instructions[0] if instructions else "",
+        "photo_url": photo_url,
+    }
+
+
+def _dashboard_url() -> str:
+    return settings.FRONTEND_URL.rstrip("/")
+
+
+async def publish_agent_home(account_id: UUID, user_id: str) -> None:
+    """Publish the Home tab: what ReplyPier does, the agent connected to this
+    workspace, how to use it, and a link to the dashboard."""
+    if not user_id:
+        return
+    db = SessionLocal()
+    try:
+        account = ChannelAccountRepository(db).get_by_id(account_id)
+        if account is None or not account.is_active:
+            return
+        agent = _assigned_agent(db, account)
+        card = await _agent_card(agent) if agent is not None else None
+        view = build_home_view(card, _dashboard_url())
+        await publish_home_view(SlackAdapter._access_token(account), user_id, view)
+    except Exception as e:
+        logger.error(f"Slack Home tab publish failed: {e}")
+    finally:
+        db.close()

backend/app/services/sql_guardrails.py

@@ -0,0 +1,369 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+SQL guardrails for the AI ticket investigator's database connector.
+
+The model's SQL is untrusted input. Every guarantee here is enforced OUTSIDE
+the model, on the parsed AST (sqlglot):
+
+- exactly one statement, and it must be a plain SELECT (no INTO, no locking)
+- every referenced table — including inside CTEs, subqueries, joins and
+  unions — must be on the connector's explicit allowlist
+- dangerous functions (sleep/file/network/admin) are denied
+- a LIMIT is always enforced (added or clamped to the connector's max_rows)
+- row-scoped tables are rewritten into a subquery pre-filtered to the ticket
+  customer's own rows, so the agent cannot read across customers
+- the statement is re-rendered from the AST, so only canonical SQL executes
+
+The executor adds the second, independent barrier: a read-only transaction
+with a statement timeout. Column masking happens on the result rows.
+"""
+
+from dataclasses import dataclass
+from typing import Iterable, List, Optional, Sequence
+
+import sqlglot
+from sqlglot import exp
+
+MASK_VALUE = "***MASKED***"
+
+# Time-wasting, file, network and admin functions. The read-only transaction
+# already blocks writes — this denylist is defense-in-depth against DoS and
+# data exfiltration via server-side functions.
+BLOCKED_FUNCTIONS = {
+    # postgres
+    "pg_sleep", "pg_sleep_for", "pg_sleep_until",
+    "pg_read_file", "pg_read_binary_file", "pg_ls_dir", "pg_stat_file",
+    "pg_terminate_backend", "pg_cancel_backend", "pg_reload_conf",
+    "pg_rotate_logfile", "set_config", "pg_promote",
+    "dblink", "dblink_exec", "dblink_connect", "dblink_connect_u",
+    "lo_import", "lo_export",
+    "query_to_xml", "database_to_xml", "table_to_xml",
+    "pg_logical_slot_get_changes", "pg_logical_slot_peek_changes",
+    "pg_create_logical_replication_slot", "pg_drop_replication_slot",
+    # mysql
+    "sleep", "benchmark", "load_file", "get_lock", "release_lock",
+    "master_pos_wait", "sys_eval", "sys_exec",
+}
+
+# Statement/expression node types that must never appear anywhere in the tree.
+BLOCKED_NODE_TYPES = (
+    exp.Insert, exp.Update, exp.Delete, exp.Merge,
+    exp.Create, exp.Drop, exp.Alter, exp.TruncateTable,
+    exp.Grant, exp.Command, exp.Transaction, exp.Set,
+    exp.Into, exp.Lock,
+)
+
+# Default schema names: a bare table reference resolves to these, so an
+# allowlist entry "public.orders" also authorizes "orders".
+DEFAULT_SCHEMAS = {"public", "dbo"}
+
+_DIALECTS = {"postgresql": "postgres", "postgres": "postgres", "mysql": "mysql"}
+
+
+@dataclass
+class SqlValidationResult:
+    ok: bool
+    sql: Optional[str] = None       # canonical SQL, only when ok
+    reason: Optional[str] = None    # block reason, only when not ok
+
+
+def _normalize_allowlist(allowed_tables: Iterable[str]) -> set:
+    """Expand allowlist entries so both qualified and bare references match
+    when the schema is a default one."""
+    allowed = set()
+    for entry in allowed_tables or []:
+        entry = (entry or "").strip().lower()
+        if not entry:
+            continue
+        allowed.add(entry)
+        if "." in entry:
+            schema, _, name = entry.rpartition(".")
+            if schema in DEFAULT_SCHEMAS:
+                allowed.add(name)
+    return allowed
+
+
+def _table_reference(table: exp.Table) -> str:
+    name = (table.name or "").lower()
+    if table.db:
+        return f"{table.db.lower()}.{name}"
+    return name
+
+
+def _cte_names(statement: exp.Expression) -> set:
+    return {cte.alias_or_name.lower() for cte in statement.find_all(exp.CTE)}
+
+
+def _blocked_function(statement: exp.Expression) -> Optional[str]:
+    for func in statement.find_all(exp.Func):
+        name = (func.sql_name() or "").lower()
+        if name in BLOCKED_FUNCTIONS:
+            return name
+        if isinstance(func, exp.Anonymous):
+            anon = (func.name or "").lower()
+            if anon in BLOCKED_FUNCTIONS:
+                return anon
+    return None
+
+
+def normalize_scope_map(row_scope: Optional[dict]) -> dict:
+    """Expand row-scope entries the same way as the allowlist, so a rule on
+    "public.orders" also binds a bare "orders" reference."""
+    scoped = {}
+    for raw_table, raw_column in (row_scope or {}).items():
+        table = (raw_table or "").strip().lower()
+        column = (raw_column or "").strip()
+        if not table or not column:
+            continue
+        scoped[table] = column
+        if "." in table:
+            schema, _, name = table.rpartition(".")
+            if schema in DEFAULT_SCHEMAS:
+                scoped.setdefault(name, column)
+    return scoped
+
+
+def _apply_row_scope(
+    statement: exp.Select,
+    scoped: dict,
+    scope_value: Optional[str],
+    cte_names: set,
+    case_insensitive: bool = False,
+) -> tuple:
+    """Rewrite every scoped base table into a pre-filtered subquery:
+
+        FROM orders o  ->  FROM (SELECT * FROM orders WHERE email = 'x') AS o
+
+    Filtering the *relation* rather than checking the model's WHERE clause is
+    what makes this unbypassable. Whatever the model writes outside — OR 1=1,
+    a UNION arm, a correlated subquery — it can only ever see rows the wrapper
+    already returned. Validating a model-supplied predicate instead would mean
+    proving that no OR, no join and no outer reference widens it, which is not
+    a thing you can check reliably.
+
+    Returns (statement, error). Fails closed: a scoped table with no value to
+    scope by is refused, never silently unscoped.
+    """
+    targets = []
+    for table in statement.find_all(exp.Table):
+        reference = _table_reference(table)
+        if not reference:
+            continue
+        if not table.db and reference in cte_names:
+            continue
+        column = scoped.get(reference)
+        if column is None and table.name:
+            column = scoped.get(table.name.lower())
+        if column:
+            targets.append((table, column))
+
+    if not targets:
+        return statement, None
+
+    if not (scope_value or "").strip():
+        return None, (
+            "This table is restricted to the rows of the ticket's own customer, "
+            "and this ticket has no customer identity to scope by. Set the "
+            "ticket's customer, or query a table that is not customer-scoped."
+        )
+
+    value = scope_value.strip()
+    for table, column in targets:
+        alias_name = table.alias or table.name
+        inner_table = table.copy()
+        inner_table.set("alias", None)
+        column_ref = exp.Column(this=exp.to_identifier(column))
+        # Emails are compared case-insensitively — the same person may be
+        # stored as Bob@x.com here and bob@x.com there, and an exact match
+        # would silently return nothing rather than their rows. Costs an index
+        # on the raw column unless one exists on LOWER(col).
+        predicate = exp.EQ(
+            this=exp.Lower(this=column_ref) if case_insensitive else column_ref,
+            # Literal.string escapes on render — never string-format a value
+            # into SQL.
+            expression=exp.Literal.string(value.lower() if case_insensitive else value),
+        )
+        inner = (
+            exp.Select().select(exp.Star()).from_(inner_table).where(predicate)
+        )
+        table.replace(
+            exp.Subquery(this=inner, alias=exp.TableAlias(this=exp.to_identifier(alias_name)))
+        )
+    return statement, None
+
+
+def _force_limit(statement: exp.Select, max_rows: int) -> exp.Select:
+    """Add a LIMIT, or clamp an existing numeric one, to max_rows."""
+    limit_node = statement.args.get("limit")
+    if limit_node is not None:
+        literal = limit_node.expression
+        if isinstance(literal, exp.Literal) and literal.is_int:
+            if int(literal.this) <= max_rows:
+                return statement
+    return statement.limit(max_rows, copy=False)
+
+
+def validate_sql(
+    raw_sql: str,
+    allowed_tables: Sequence[str],
+    max_rows: int,
+    engine: str = "postgresql",
+    masked_columns: Optional[Iterable[str]] = None,
+    row_scope: Optional[dict] = None,
+    scope_value: Optional[str] = None,
+    scope_case_insensitive: bool = False,
+) -> SqlValidationResult:
+    """Validate untrusted SQL against the connector policy. Returns canonical
+    SQL to execute, or the reason it was blocked.
+
+    row_scope maps a table to the column identifying the customer who owns the
+    row ({"orders": "customer_email"}); scope_value is that customer's value
+    for the ticket being investigated. Listed tables are rewritten so only
+    their rows are visible."""
+    dialect = _DIALECTS.get((engine or "").lower(), "postgres")
+    if not (raw_sql or "").strip():
+        return SqlValidationResult(ok=False, reason="Empty query")
+
+    try:
+        statements = sqlglot.parse(raw_sql, read=dialect)
+    except sqlglot.errors.ParseError as e:
+        return SqlValidationResult(ok=False, reason=f"SQL could not be parsed: {e}")
+
+    statements = [s for s in statements if s is not None]
+    if len(statements) != 1:
+        return SqlValidationResult(
+            ok=False, reason="Exactly one SQL statement is allowed"
+        )
+    statement = statements[0]
+
+    if not isinstance(statement, exp.Select):
+        return SqlValidationResult(
+            ok=False,
+            reason="Only plain SELECT statements are allowed "
+            "(wrap set operations like UNION in a subquery)",
+        )
+
+    for node_type in BLOCKED_NODE_TYPES:
+        found = statement.find(node_type)
+        if found is not None:
+            return SqlValidationResult(
+                ok=False, reason=f"Disallowed SQL construct: {found.key.upper()}"
+            )
+
+    blocked_func = _blocked_function(statement)
+    if blocked_func is not None:
+        return SqlValidationResult(ok=False, reason=f"Disallowed function: {blocked_func}")
+
+    allowed = _normalize_allowlist(allowed_tables)
+    if not allowed:
+        return SqlValidationResult(ok=False, reason="No tables are allowlisted for this connector")
+    cte_names = _cte_names(statement)
+    for table in statement.find_all(exp.Table):
+        reference = _table_reference(table)
+        if not reference:
+            continue
+        # References to CTE names defined in this query are fine — the CTE
+        # bodies themselves are walked by this same loop.
+        if not table.db and reference in cte_names:
+            continue
+        if reference not in allowed:
+            return SqlValidationResult(
+                ok=False, reason=f"Table not on the allowlist: {reference}"
+            )
+
+    # Masked columns may not be referenced at all — output-name masking alone
+    # is alias-bypassable (SELECT email AS x), and blocking references also
+    # prevents value probing via WHERE clauses. SELECT * and t.* stay allowed:
+    # they expand to real column names that mask_rows redacts by name.
+    masked = {c.strip().lower() for c in masked_columns or [] if c and c.strip()}
+    if masked:
+        # Names that denote a whole row when used as a bare value.
+        scope_names = {t.name.lower() for t in statement.find_all(exp.Table) if t.name}
+        scope_names |= {
+            t.alias.lower() for t in statement.find_all(exp.Table) if t.alias
+        }
+        scope_names |= cte_names
+
+        for column in statement.find_all(exp.Column):
+            # A qualified/bare star (t.* or *) nested inside a function or cast
+            # collapses the row into one value (to_jsonb(c.*), (c.*)::text),
+            # defeating name-based masking — only a top-level projection star
+            # is safe. Same for a bare Star node.
+            if isinstance(column.this, exp.Star) and not isinstance(column.parent, exp.Select):
+                return SqlValidationResult(
+                    ok=False,
+                    reason="Row expansion (t.*) inside an expression is not allowed with masked columns",
+                )
+            if (column.name or "").lower() in masked and not isinstance(column.this, exp.Star):
+                return SqlValidationResult(
+                    ok=False,
+                    reason=f"Column '{column.name}' is masked and cannot be referenced",
+                )
+            # A bare (unqualified) reference to a table/alias/CTE name is a
+            # whole-row value — SELECT customers, to_jsonb(c), c::text, row(c).
+            # These serialize masked fields past name-based masking.
+            if not column.table and (column.name or "").lower() in scope_names:
+                return SqlValidationResult(
+                    ok=False,
+                    reason=(
+                        f"Whole-row reference to '{column.name}' is not allowed with "
+                        "masked columns — select individual columns instead"
+                    ),
+                )
+        for star in statement.find_all(exp.Star):
+            # Safe: top-level projection (SELECT *), the star of a t.* Column
+            # node (handled above), and count(*) — a cardinality marker that
+            # never exposes row values.
+            if not isinstance(star.parent, (exp.Select, exp.Column, exp.Count)):
+                return SqlValidationResult(
+                    ok=False,
+                    reason="Row expansion (*) inside an expression is not allowed with masked columns",
+                )
+
+    # Last, so the rewrite can't perturb the checks above: they inspect what
+    # the model actually wrote, not the wrappers added on its behalf.
+    scoped = normalize_scope_map(row_scope)
+    if scoped:
+        statement, scope_error = _apply_row_scope(
+            statement, scoped, scope_value, cte_names,
+            case_insensitive=scope_case_insensitive,
+        )
+        if scope_error is not None:
+            return SqlValidationResult(ok=False, reason=scope_error)
+
+    statement = _force_limit(statement, max_rows)
+    # comments=False: comments are dropped from the canonical SQL — MySQL
+    # executable comments (/*! ... */) must never reach the server.
+    return SqlValidationResult(
+        ok=True, sql=statement.sql(dialect=dialect, comments=False)
+    )
+
+
+def mask_rows(
+    columns: List[str], rows: List[tuple], masked_columns: Iterable[str]
+) -> List[tuple]:
+    """Replace values of masked columns (matched by column name, case
+    insensitive) so PII never reaches the model."""
+    masked = {c.strip().lower() for c in masked_columns or [] if c and c.strip()}
+    if not masked:
+        return rows
+    mask_indexes = {i for i, col in enumerate(columns) if col.lower() in masked}
+    if not mask_indexes:
+        return rows
+    return [
+        tuple(MASK_VALUE if i in mask_indexes else value for i, value in enumerate(row))
+        for row in rows
+    ]

backend/app/services/telegram_webhooks.py

@@ -0,0 +1,83 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Registering ReplyPier's webhook with a Telegram bot, with the updates it
+must receive. Done on connect, and again for an agent's bots when its reply
+buttons are turned on: a bot connected before buttons existed was registered
+without button taps, and Telegram keeps a bot's last registration.
+"""
+
+import asyncio
+import json
+from typing import Optional, Tuple
+from uuid import UUID
+
+from sqlalchemy.orm import Session
+
+from app.channels import telegram as telegram_api
+from app.core.config import settings
+from app.core.security import decrypt_api_key
+from app.core.logger import get_logger
+from app.models.channels import ChannelAccount, ChannelType
+from app.repositories.channels import AgentChannelConfigRepository, ChannelAccountRepository
+
+logger = get_logger(__name__)
+
+
+def webhook_url(account_id) -> str:
+    return f"{settings.BACKEND_URL.rstrip('/')}{settings.API_V1_STR}/webhooks/telegram/{account_id}"
+
+
+async def register(account: ChannelAccount, bot_token: str) -> Tuple[bool, str]:
+    return await telegram_api.set_webhook(bot_token, webhook_url(account.id), account.webhook_secret)
+
+
+async def reregister_for_agent(db: Session, agent_id: UUID) -> None:
+    """Register again, at once, every active Telegram bot routed to this agent
+    — its buttons were just turned on."""
+    try:
+        accounts = ChannelAccountRepository(db)
+        bots = [account for account in (accounts.get_by_id(config.channel_account_id)
+                                        for config in AgentChannelConfigRepository(db).list_by_agent(agent_id))
+                if _is_active_bot(account)]
+    except Exception:
+        # The agent is already saved; only the re-registration is lost
+        logger.exception(f"Could not list agent {agent_id}'s Telegram bots to re-register")
+        return
+    await asyncio.gather(*(_reregister(account) for account in bots))
+
+
+async def reregister_if_buttons_on(account: ChannelAccount, agent) -> None:
+    """A bot just routed to an agent whose buttons are already on."""
+    if _is_active_bot(account) and agent.channel_quick_replies:
+        await _reregister(account)
+
+
+async def _reregister(account: ChannelAccount) -> None:
+    """Best effort: a bot that can't be registered keeps working — only its
+    button taps don't arrive until it is (turn the agent's buttons off and on
+    again, or reconnect the bot)."""
+    try:
+        bot_token = json.loads(decrypt_api_key(account.encrypted_credentials))["bot_token"]
+        ok, error = await register(account, bot_token)
+    except Exception as e:
+        ok, error = False, f"{type(e).__name__}: {e}"
+    if not ok:
+        logger.warning(f"Could not re-register Telegram bot {account.id} for button taps: {error}")
+
+
+def _is_active_bot(account: Optional[ChannelAccount]) -> bool:
+    return (account is not None and account.is_active
+            and account.channel_type == ChannelType.TELEGRAM.value)

backend/app/services/ticket_access.py

@@ -0,0 +1,43 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+AI-ticketing plan gating: paid-plan feature on cloud, unrestricted for
+OSS/self-hosted. Thin wrappers over the generic feature gate (the
+help-center pattern).
+"""
+
+from uuid import UUID
+
+from sqlalchemy.orm import Session
+
+from app.services.feature_gate import check_feature_access, feature_allowed
+
+AI_TICKETING_FEATURE = "ai_ticketing"
+
+UPGRADE_MESSAGE = (
+    "AI Ticketing is not available in your current plan. "
+    "Please upgrade to access this feature."
+)
+
+
+def ticketing_allowed(db: Session, organization_id: UUID) -> bool:
+    """Non-raising check — used by the chat-agent tool assembly and workers.
+    Fails closed on gating errors."""
+    return feature_allowed(db, organization_id, AI_TICKETING_FEATURE)
+
+
+def check_ticketing_access(db: Session, organization_id: UUID) -> None:
+    """Raising check for admin endpoints (403 with the upgrade message)."""
+    check_feature_access(db, organization_id, AI_TICKETING_FEATURE, UPGRADE_MESSAGE)

backend/app/services/ticket_csat.py

@@ -0,0 +1,67 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Capture side of the ticket CSAT loop.
+
+TicketService.close() asks the customer for a rating through the widget's
+existing conversation rating prompt. This module carries the answer back: every
+path that creates a Rating calls record_conversation_rating(), which attaches
+the score to the ticket linked to that session, if there is one.
+
+Deliberately conversation-only. ratings.session_id is NOT NULL, so a rating
+cannot exist without a chat session — a manual or email-only ticket has nothing
+to rate. Supporting those would mean a public tokenised rating page and a
+session-less Rating; until that exists, such tickets are never asked and their
+CSAT card reads "not requested".
+
+Best-effort throughout: a rating is a customer-facing success the moment it is
+stored, so a failure to link it must never fail the submission.
+"""
+
+from typing import Optional
+from uuid import UUID
+
+from sqlalchemy.orm import Session
+
+from app.core.logger import get_logger
+from app.models.rating import Rating
+from app.services.ticket import TicketService
+
+logger = get_logger(__name__)
+
+
+def record_conversation_rating(db: Session, rating: Rating) -> Optional[UUID]:
+    """Link a just-submitted conversation rating to its ticket.
+
+    Returns the ticket id it was attached to, or None when the session has no
+    native ticket (the common case for plain chats) or the linkage failed.
+    """
+    try:
+        service = TicketService(db)
+        ticket = service.repo.get_by_session(rating.session_id)
+        if ticket is None:
+            return None
+        service.record_csat(ticket, rating.id, rating.rating, rating.feedback)
+        db.commit()
+        return ticket.id
+    except Exception as e:
+        # The rating row itself is already committed by the caller, so rolling
+        # back here only discards the half-applied ticket update.
+        logger.error(f"Failed to link rating {rating.id} to its ticket: {e}")
+        try:
+            db.rollback()
+        except Exception:
+            pass
+        return None

backend/app/services/ticket_email.py

@@ -0,0 +1,181 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Direct email delivery for ticket notifications — the fallback when a ticket
+has no linked chat conversation to deliver through (manual tickets created
+from the dashboard). Reuses the email channel's SMTP plumbing: the org's
+connected email inbox when one exists (correct SPF/DKIM from their own
+domain), platform SMTP otherwise.
+"""
+
+import asyncio
+import re
+import secrets
+from email.message import EmailMessage
+from email.utils import make_msgid
+from typing import List, Optional
+from uuid import UUID
+
+from sqlalchemy.orm import Session
+
+from app.channels.email import open_smtp, platform_smtp_config, smtp_config
+from app.core.config import settings
+from app.core.logger import get_logger
+from app.models.channels import ChannelAccount
+from app.models.ticket import TICKET_NUMBER_PREFIX
+from app.repositories.channels.accounts import ChannelAccountRepository
+
+logger = get_logger(__name__)
+
+EMAIL_CHANNEL_TYPE = "email"
+
+# Message-ID local part for ticket mail: 'ticket-<uuid hex>.<unique>'. The
+# ticket id lives in the address itself, so a reply that quotes any of our
+# Message-IDs back in References identifies its ticket with no lookup table.
+# Every sent message still gets its own unique id — reusing one id across
+# messages makes Gmail collapse them into a single copy.
+_TICKET_MSGID_RE = re.compile(r"ticket-([0-9a-f]{32})(?:\.[^@\s>]+)?@", re.I)
+# The phantom id every ticket mail references. Never sent as a message itself:
+# it is the shared thread root that ties separate notifications together.
+_ROOT_SUFFIX = "root"
+_SUBJECT_TOKEN_RE = re.compile(rf"\[{TICKET_NUMBER_PREFIX}-(\d+)\]", re.I)
+
+DEFAULT_MSGID_DOMAIN = "replypier.site"
+
+
+def _msgid_domain(from_email: Optional[str] = None) -> str:
+    """Right-hand side of generated Message-IDs. Cosmetic only — the parser
+    matches on the local part, so mail sent from a customer's own inbox domain
+    still threads back to its ticket."""
+    candidate = from_email or settings.FROM_EMAIL or ""
+    _, _, domain = candidate.rpartition("@")
+    return domain.strip("> ").lower() or DEFAULT_MSGID_DOMAIN
+
+
+def ticket_root_message_id(ticket_id, from_email: Optional[str] = None) -> str:
+    """The stable per-ticket thread root, carried in References on every mail."""
+    return f"<ticket-{UUID(str(ticket_id)).hex}.{_ROOT_SUFFIX}@{_msgid_domain(from_email)}>"
+
+
+def new_ticket_message_id(ticket_id, from_email: Optional[str] = None) -> str:
+    """A fresh Message-ID for one outbound ticket mail."""
+    return (
+        f"<ticket-{UUID(str(ticket_id)).hex}.{secrets.token_hex(8)}"
+        f"@{_msgid_domain(from_email)}>"
+    )
+
+
+def build_thread_headers(
+    ticket_id, in_reply_to: Optional[str] = None, from_email: Optional[str] = None
+) -> dict:
+    """Threading headers for one ticket notification.
+
+    References always starts at the ticket root so every notification for a
+    ticket lands in one client-side thread; In-Reply-To points at the
+    customer's most recent reply when we have seen one, so their client keeps
+    the conversation in order.
+    """
+    root = ticket_root_message_id(ticket_id, from_email)
+    references: List[str] = [root]
+    if in_reply_to and in_reply_to != root:
+        references.append(in_reply_to)
+    return {
+        "Message-ID": new_ticket_message_id(ticket_id, from_email),
+        "In-Reply-To": in_reply_to or root,
+        "References": " ".join(references),
+    }
+
+
+def ticket_ids_from_references(*values: Optional[str]) -> List[UUID]:
+    """Ticket ids encoded in In-Reply-To/References/Message-ID headers, most
+    recently referenced first — References lists ancestors oldest-first, and
+    the nearest ancestor is the better match."""
+    found: List[UUID] = []
+    for value in values:
+        if not value:
+            continue
+        for hex_id in _TICKET_MSGID_RE.findall(str(value)):
+            try:
+                ticket_id = UUID(hex_id)
+            except ValueError:
+                continue
+            if ticket_id not in found:
+                found.append(ticket_id)
+    found.reverse()
+    return found
+
+
+def ticket_number_from_subject(subject: Optional[str]) -> Optional[int]:
+    """The [TKT-12] token a customer's client keeps in the reply subject —
+    the fallback when a mail client drops the threading headers."""
+    if not subject:
+        return None
+    match = _SUBJECT_TOKEN_RE.search(str(subject))
+    return int(match.group(1)) if match else None
+
+
+def _org_email_account(db: Session, organization_id) -> Optional[ChannelAccount]:
+    accounts = ChannelAccountRepository(db).list_by_org(organization_id, EMAIL_CHANNEL_TYPE)
+    for account in accounts:
+        if account.is_active:
+            return account
+    return None
+
+
+def _resolve_smtp(db: Session, organization_id) -> dict:
+    account = _org_email_account(db, organization_id)
+    if account is not None:
+        return smtp_config(account)
+    return platform_smtp_config(settings.FROM_EMAIL)
+
+
+def _smtp_send(cfg: dict, message: EmailMessage) -> None:
+    with open_smtp(cfg) as smtp:
+        smtp.send_message(message)
+
+
+async def send_ticket_email(
+    db: Session,
+    organization_id,
+    to_email: str,
+    subject: str,
+    body: str,
+    ticket_id=None,
+    in_reply_to: Optional[str] = None,
+) -> bool:
+    """Send a plain-text ticket notification email. Returns False (and logs)
+    on failure — ticket mutations must never fail because SMTP hiccuped.
+
+    With a ticket_id the mail carries ticket-scoped threading headers, so the
+    customer's reply comes back identifiable as a reply to that ticket."""
+    try:
+        cfg = _resolve_smtp(db, organization_id)
+        message = EmailMessage()
+        message["From"] = cfg["from_email"]
+        message["To"] = to_email
+        message["Subject"] = subject
+        if ticket_id is not None:
+            for header, value in build_thread_headers(
+                ticket_id, in_reply_to, cfg["from_email"]
+            ).items():
+                message[header] = value
+        else:
+            message["Message-ID"] = make_msgid()
+        message.set_content(body)
+        await asyncio.to_thread(_smtp_send, cfg, message)
+        return True
+    except Exception as e:
+        logger.error(f"Ticket email to {to_email} failed: {e}")
+        return False

backend/app/services/ticket_email_reply.py

@@ -0,0 +1,114 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Inbound side of ticket email threading: decide whether a mail arriving on the
+email channel is a reply to one of our ticket notifications, and append it to
+that ticket instead of feeding it to the chat agent.
+"""
+
+from typing import Optional
+from uuid import UUID
+
+from sqlalchemy.orm import Session
+
+from app.channels.base import InboundMessage
+from app.core.logger import get_logger
+from app.database import SessionLocal
+from app.models.ticket import Ticket
+from app.repositories.ticket import TicketRepository
+from app.services.ticket_email import (
+    ticket_ids_from_references,
+    ticket_number_from_subject,
+)
+
+logger = get_logger(__name__)
+
+
+def find_ticket_for_reply(
+    db: Session, organization_id: UUID, profile: Optional[dict]
+) -> Optional[Ticket]:
+    """The ticket an inbound email is replying to, or None.
+
+    Headers first: In-Reply-To/References carry a Message-ID we minted, and
+    the ticket id is inside it — unguessable, so a match is proof the mail
+    descends from a notification we sent.
+
+    The [TKT-n] subject token is the fallback for clients that drop threading
+    headers. It is guessable, and a ticket's feed is read by agents and by the
+    investigator AI, so that path additionally requires the sender to be the
+    ticket's own customer; anything else falls through to the chat pipeline.
+    """
+    profile = profile or {}
+    repo = TicketRepository(db)
+
+    for ticket_id in ticket_ids_from_references(
+        profile.get("in_reply_to"),
+        profile.get("references"),
+    ):
+        ticket = repo.get_by_id(ticket_id, organization_id)
+        if ticket is not None:
+            return ticket
+
+    number = ticket_number_from_subject(profile.get("subject"))
+    if number is None:
+        return None
+    ticket = repo.get_by_number(organization_id, number)
+    if ticket is None:
+        return None
+    sender = (profile.get("email") or "").strip().lower()
+    customer_email = (getattr(ticket.customer, "email", "") or "").strip().lower()
+    if sender and sender == customer_email:
+        return ticket
+    logger.info(
+        f"Subject token named {ticket.display_number} but {sender or 'an unknown sender'} "
+        "is not its customer — handling as a normal message"
+    )
+    return None
+
+
+async def record_ticket_email_reply(
+    organization_id: UUID, ticket_id: UUID, inbound: InboundMessage
+) -> None:
+    """Append an inbound email to its ticket. Runs from a BackgroundTask with
+    its own DB session, mirroring process_channel_message."""
+    db = SessionLocal()
+    try:
+        ticket = TicketRepository(db).get_by_id(ticket_id, organization_id)
+        if ticket is None:
+            logger.warning(f"Email reply for unknown ticket {ticket_id}")
+            return
+
+        from app.services.ticket import TicketService
+        service = TicketService(db)
+        profile = inbound.profile or {}
+        service.record_customer_email_reply(
+            ticket,
+            body=inbound.text or "",
+            from_email=inbound.external_user_id,
+            message_id=profile.get("inbound_message_id") or None,
+        )
+        db.commit()
+        db.refresh(ticket)
+
+        from app.services.ticket_events import emit_ticket_update
+        await emit_ticket_update(
+            organization_id, ticket.id, "comment", {"status": str(ticket.status)}
+        )
+        logger.info(f"Email reply appended to {ticket.display_number}")
+    except Exception as e:
+        db.rollback()
+        logger.error(f"Failed to record email reply on ticket {ticket_id}: {e}")
+    finally:
+        db.close()

backend/app/services/ticket_events.py

@@ -0,0 +1,55 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Live ticket updates to the dashboard. Emitted into the per-org ticket room on
+the /agent namespace after a mutation commits; the frontend keeps a poll
+fallback, so a missed frame is never fatal.
+"""
+
+from typing import Optional
+from uuid import UUID
+
+from app.core.logger import get_logger
+from app.core.socketio import sio
+
+logger = get_logger(__name__)
+
+ORG_TICKET_ROOM_PREFIX = "org_tickets_"
+
+
+def org_ticket_room(organization_id) -> str:
+    return f"{ORG_TICKET_ROOM_PREFIX}{organization_id}"
+
+
+async def emit_ticket_update(
+    organization_id: UUID,
+    ticket_id: UUID,
+    kind: str,
+    payload: Optional[dict] = None,
+) -> None:
+    """kind: 'created' | 'status' | 'triage' | 'comment' | 'run' | 'updated'."""
+    try:
+        await sio.emit(
+            "ticket_update",
+            {
+                "ticket_id": str(ticket_id),
+                "kind": kind,
+                "payload": payload or {},
+            },
+            room=org_ticket_room(organization_id),
+            namespace="/agent",
+        )
+    except Exception as e:
+        logger.error(f"ticket_update emit failed for {ticket_id}: {e}")

backend/app/services/ticket_investigation.py

@@ -0,0 +1,349 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Phase 3 investigation engine: hypothesis-driven root-cause analysis with every
+tool call captured as a first-class InvestigationEvent row (the glass box).
+Orchestrated here, executed by the ticket_investigator worker.
+"""
+
+import asyncio
+import json
+import time
+from typing import List, Optional, Tuple
+from uuid import UUID
+
+from app.core.logger import get_logger
+from app.database import SessionLocal
+from app.models.investigation import (
+    HypothesisStatus,
+    InvestigationEvent,
+    InvestigationEventType,
+    InvestigationHypothesis,
+    InvestigationRun,
+    InvestigationRunStatus,
+    RCADocument,
+)
+from app.models.ticket import Ticket
+from app.utils.guardrails import GuardrailAction, PIIDetector
+
+logger = get_logger(__name__)
+
+# Per-hypothesis tool budget; the run-level max_tool_calls caps the total.
+HYPOTHESIS_TOOL_CALL_LIMIT = 6
+# Truncation applied at capture time — raw payloads never land in the DB.
+MAX_EVENT_INPUT_CHARS = 2000
+MAX_EVENT_RESULT_CHARS = 4000
+# Bounded time for the final RCA synthesis (outside the main wall clock so a
+# timed-out investigation can still produce a partial RCA).
+RCA_SYNTHESIS_TIMEOUT_SECONDS = 180
+
+VALID_VERDICT_STATUSES = {
+    HypothesisStatus.VALIDATED.value,
+    HypothesisStatus.INVALIDATED.value,
+    HypothesisStatus.INCONCLUSIVE.value,
+}
+
+
+def redact_snippet(text: str, max_chars: int) -> str:
+    """PII-redact and truncate a tool payload before it is persisted."""
+    if not text:
+        return ""
+    result = PIIDetector.detect(text[: max_chars * 2], action=GuardrailAction.REDACT)
+    redacted = result.redacted_text if result.redacted_text is not None else text
+    redacted = redacted[:max_chars]
+    return redacted
+
+
+class EvidenceRecorder:
+    """Writes InvestigationEvent rows as the investigator works.
+
+    Each write uses its own short-lived session so evidence is visible live
+    (the glass-box UI polls during active runs) and survives a failed run.
+    Also the run's tool-call meter: the budget check reads self.tool_calls.
+    """
+
+    def __init__(self, run_id: UUID, ticket_id: UUID):
+        self.run_id = run_id
+        self.ticket_id = ticket_id
+        self.seq = 0
+        self.tool_calls = 0
+        self.hypothesis_id: Optional[UUID] = None
+        # Per-hypothesis tool-call outcomes: {hypothesis_id: [total, errored]}.
+        # A verdict is only worth as much as the evidence under it, and a run
+        # where every query failed must not be able to assert a fact (#318).
+        self.calls_by_hypothesis: dict = {}
+        # Run-level totals, for the dashboard's connector banner.
+        self.failed_tool_calls = 0
+        # tool function name -> configured connector (MCP server) name.
+        self.connector_by_function: dict = {}
+        # In-memory digest for RCA synthesis, mirroring the persisted rows.
+        self.evidence_lines: List[str] = []
+
+    def map_connectors(self, mcp_tools: list) -> None:
+        for tool in mcp_tools or []:
+            connector = getattr(tool, "_connector_name", None) or "MCP"
+            for function_name in getattr(tool, "functions", {}) or {}:
+                self.connector_by_function[function_name] = connector
+
+    def _write(self, **fields) -> None:
+        self.seq += 1
+        try:
+            with SessionLocal() as db:
+                db.add(
+                    InvestigationEvent(
+                        run_id=self.run_id,
+                        ticket_id=self.ticket_id,
+                        hypothesis_id=self.hypothesis_id,
+                        seq=self.seq,
+                        **fields,
+                    )
+                )
+                db.commit()
+        except Exception as e:
+            logger.error(f"Failed to record investigation event: {e}")
+
+    def record_phase(self, label: str) -> None:
+        """Phase marker — powers the live ticker line in the UI."""
+        self._write(event_type=InvestigationEventType.PHASE, label=label[:300])
+
+    async def tool_hook(self, function_name: str, function, arguments: dict):
+        """agno tool hook: times the call and captures redacted input/result.
+        Registered on the hypothesis-testing agents only."""
+        # The Groq structured-output capture tool is plumbing, not evidence.
+        if function_name == "json":
+            return await function(**arguments)
+        started = time.monotonic()
+        error: Optional[str] = None
+        result = None
+        try:
+            result = await function(**arguments)
+            return result
+        except Exception as e:
+            error = str(e)
+            raise
+        finally:
+            self.tool_calls += 1
+            if error:
+                self.failed_tool_calls += 1
+            if self.hypothesis_id is not None:
+                tally = self.calls_by_hypothesis.setdefault(self.hypothesis_id, [0, 0])
+                tally[0] += 1
+                if error:
+                    tally[1] += 1
+            duration_ms = int((time.monotonic() - started) * 1000)
+            try:
+                input_text = json.dumps(arguments, default=str)
+            except Exception:
+                input_text = str(arguments)
+            input_snippet = redact_snippet(input_text, MAX_EVENT_INPUT_CHARS)
+            result_snippet = redact_snippet(str(result), MAX_EVENT_RESULT_CHARS) if result is not None else None
+            self._write(
+                event_type=InvestigationEventType.TOOL_CALL,
+                tool_name=function_name[:200],
+                connector_name=self.connector_by_function.get(function_name, "MCP")[:200],
+                tool_input=input_snippet,
+                tool_result=result_snippet,
+                duration_ms=duration_ms,
+                error=error[:2000] if error else None,
+            )
+            self.evidence_lines.append(
+                f"[E{self.seq}] {function_name}({input_snippet[:300]}) -> "
+                + (f"ERROR: {error[:200]}" if error else (result_snippet or "")[:500])
+            )
+
+
+    def tool_outcomes(self, hypothesis_id: UUID) -> tuple:
+        """(total, errored) tool calls made while testing this hypothesis."""
+        total, errored = self.calls_by_hypothesis.get(hypothesis_id, [0, 0])
+        return total, errored
+
+
+def enforce_evidence_backed_verdict(
+    hypothesis: InvestigationHypothesis, recorder: "EvidenceRecorder"
+) -> bool:
+    """Downgrade a definite verdict that no successful query supports.
+
+    The model returns status and confidence as free text, and nothing else
+    checks them against what actually happened. A run whose every search
+    errored — a missing index, a rejected client version — can still come back
+    "the record does not exist, validated, 0.90", which a support agent would
+    reasonably act on. Tool errors are the absence of evidence, never evidence
+    of absence.
+
+    Only fires when the phase made tool calls and every one of them failed;
+    a hypothesis reasoned from ticket context alone is left as the model
+    returned it. Returns True when it changed the verdict.
+    """
+    total, errored = recorder.tool_outcomes(hypothesis.id)
+    if not total or errored < total:
+        return False
+    if str(hypothesis.status) == HypothesisStatus.INCONCLUSIVE.value:
+        return False
+
+    original = str(hypothesis.status)
+    confidence = hypothesis.confidence
+    hypothesis.status = HypothesisStatus.INCONCLUSIVE
+    hypothesis.confidence = 0.0
+    # The model's own conclusion is dropped rather than kept: it reads as a
+    # finding, and the RCA writer would quote it as one.
+    hypothesis.conclusion = (
+        f"Not tested — all {total} tool call{'' if total == 1 else 's'} in this phase "
+        f"failed, so no evidence was gathered. The tester reported "
+        f"\"{original}\"{f' at {confidence:.2f} confidence' if confidence is not None else ''}, "
+        "which nothing supports. Check the connector errors in the evidence log."
+    )
+    return True
+
+
+def hypotheses_digest(hypotheses: List[InvestigationHypothesis]) -> str:
+    lines = []
+    for h in hypotheses:
+        lines.append(
+            f"H{h.idx}: {h.title} — {h.status} (confidence {h.confidence if h.confidence is not None else 'n/a'})\n"
+            f"    {h.conclusion or h.rationale or ''}"
+        )
+    return "\n".join(lines)
+
+
+async def run_investigation_phases(
+    db,
+    run: InvestigationRun,
+    ticket: Ticket,
+    agent,
+    context_message: str,
+    mcp_tools: list,
+    recorder: EvidenceRecorder,
+) -> Tuple[List[InvestigationHypothesis], bool]:
+    """Generate hypotheses, then test each within the run's tool budget.
+    Mutates hypothesis rows (committed per step so the UI streams progress).
+    Returns (hypotheses, budget_exhausted)."""
+    recorder.record_phase("Generating hypotheses")
+    plan = await agent.generate_hypotheses(context_message)
+    if plan is None or not plan.hypotheses:
+        raise RuntimeError("Hypothesis generation produced no result")
+
+    hypotheses: List[InvestigationHypothesis] = []
+    for i, spec in enumerate(plan.hypotheses, start=1):
+        hypothesis = InvestigationHypothesis(
+            run_id=run.id,
+            ticket_id=ticket.id,
+            idx=i,
+            title=spec.title[:300],
+            rationale=spec.rationale,
+            status=HypothesisStatus.PENDING,
+        )
+        db.add(hypothesis)
+        hypotheses.append(hypothesis)
+    db.commit()
+
+    budget_exhausted = False
+    max_tool_calls = run.max_tool_calls if run.max_tool_calls is not None else 25
+    for hypothesis in hypotheses:
+        remaining_budget = max(0, max_tool_calls - recorder.tool_calls)
+        if remaining_budget <= 0 and mcp_tools:
+            budget_exhausted = True
+            hypothesis.status = HypothesisStatus.INCONCLUSIVE
+            hypothesis.confidence = 0.0
+            hypothesis.conclusion = "Not tested — the run's tool-call budget was exhausted."
+            db.commit()
+            continue
+
+        recorder.hypothesis_id = hypothesis.id
+        recorder.record_phase(f"Testing H{hypothesis.idx}: {hypothesis.title}")
+        hypothesis.status = HypothesisStatus.TESTING
+        db.commit()
+
+        verdict = await agent.test_hypothesis(
+            context_message,
+            hypothesis.title,
+            hypothesis.rationale,
+            mcp_tools=mcp_tools,
+            tool_call_limit=min(HYPOTHESIS_TOOL_CALL_LIMIT, remaining_budget) if mcp_tools else None,
+            tool_hooks=[recorder.tool_hook],
+        )
+        if verdict is not None and verdict.status in VALID_VERDICT_STATUSES:
+            hypothesis.status = verdict.status
+            hypothesis.confidence = max(0.0, min(1.0, verdict.confidence))
+            hypothesis.conclusion = verdict.conclusion
+            if enforce_evidence_backed_verdict(hypothesis, recorder):
+                logger.warning(
+                    f"H{hypothesis.idx} claimed '{verdict.status}' with every tool "
+                    f"call failing; forced to inconclusive"
+                )
+        else:
+            hypothesis.status = HypothesisStatus.INCONCLUSIVE
+            hypothesis.confidence = 0.0
+            # A provider that refused the request is a hard, deterministic
+            # failure — saying "no parseable verdict" for it reads as a model
+            # quality problem and hides a fixable connector issue (#303).
+            hypothesis.conclusion = (
+                getattr(agent, "last_provider_error", None)
+                or "The tester produced no parseable verdict."
+            )
+        db.commit()
+    recorder.hypothesis_id = None
+    return hypotheses, budget_exhausted
+
+
+async def synthesize_and_store_rca(
+    db,
+    run: InvestigationRun,
+    ticket: Ticket,
+    agent,
+    context_message: str,
+    hypotheses: List[InvestigationHypothesis],
+    recorder: EvidenceRecorder,
+    partial: bool,
+) -> Optional[RCADocument]:
+    """Write the versioned RCA document from whatever evidence exists."""
+    from app.repositories.ticket import InvestigationRepository
+
+    recorder.record_phase("Writing root-cause analysis")
+    try:
+        result = await asyncio.wait_for(
+            agent.synthesize_rca(
+                context_message,
+                hypotheses_digest(hypotheses),
+                "\n".join(recorder.evidence_lines[-60:]),
+                partial=partial,
+            ),
+            timeout=RCA_SYNTHESIS_TIMEOUT_SECONDS,
+        )
+    except asyncio.TimeoutError:
+        logger.error(f"RCA synthesis timed out for {ticket.display_number}")
+        return None
+    if result is None:
+        return None
+
+    rca = RCADocument(
+        ticket_id=ticket.id,
+        run_id=run.id,
+        version=InvestigationRepository(db).next_rca_version(ticket.id),
+        summary=result.summary,
+        impact=result.impact,
+        timeline=[entry.model_dump() for entry in result.timeline],
+        investigation_log=result.investigation_log,
+        contributing_factors=result.contributing_factors,
+        conclusion=result.conclusion,
+        remediation=result.remediation,
+        prevention=result.prevention,
+        customer_summary=result.customer_summary,
+        confidence=max(0.0, min(1.0, result.confidence)),
+        is_partial=partial,
+        generated_by="ai",
+    )
+    db.add(rca)
+    db.commit()
+    return rca

backend/app/services/ticket_jira.py

@@ -0,0 +1,134 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+One-way Jira escalation for native tickets: when enabled, tickets at or
+above the configured priority get a mirrored Jira issue (external_ref_*).
+Native ReplyPier stays the source of truth — nothing syncs back.
+"""
+
+from typing import Optional
+
+from app.core.logger import get_logger
+from app.models.jira import AgentJiraConfig, JiraToken
+from app.models.organization import Organization
+from app.models.ticket import Ticket
+from app.models.ticket_activity import TicketActivityType, TicketActorType
+
+logger = get_logger(__name__)
+
+_PRIORITY_ORDER = {"urgent": 0, "high": 1, "medium": 2, "low": 3}
+
+
+def _find_project_config(db, ticket: Ticket) -> Optional[AgentJiraConfig]:
+    """Project/issue-type to escalate into: the ticket's agent's Jira config
+    when enabled, else any enabled agent config in the organization."""
+    from app.models.agent import Agent
+
+    if ticket.agent_id is not None:
+        config = (
+            db.query(AgentJiraConfig)
+            .filter(
+                AgentJiraConfig.agent_id == str(ticket.agent_id),
+                AgentJiraConfig.enabled == True,  # noqa: E712
+                AgentJiraConfig.project_key.isnot(None),
+            )
+            .first()
+        )
+        if config is not None:
+            return config
+    return (
+        db.query(AgentJiraConfig)
+        .join(Agent, Agent.id == AgentJiraConfig.agent_id.cast(Agent.id.type))
+        .filter(
+            Agent.organization_id == ticket.organization_id,
+            AgentJiraConfig.enabled == True,  # noqa: E712
+            AgentJiraConfig.project_key.isnot(None),
+        )
+        .first()
+    )
+
+
+def should_escalate(ticket: Ticket, settings_row) -> bool:
+    if not settings_row.jira_escalation_enabled:
+        return False
+    if ticket.external_ref_id:  # already escalated — one-way, never re-synced
+        return False
+    minimum = str(settings_row.jira_escalation_priority or "urgent")
+    return _PRIORITY_ORDER.get(str(ticket.priority), 99) <= _PRIORITY_ORDER.get(minimum, 0)
+
+
+async def maybe_escalate_to_jira(db, service, ticket: Ticket, settings_row) -> bool:
+    """Escalate when policy matches and the org has Jira connected. Failures
+    are logged, never raised — escalation must not break ticket flows."""
+    if not should_escalate(ticket, settings_row):
+        return False
+    try:
+        token = (
+            db.query(JiraToken)
+            .filter(JiraToken.organization_id == ticket.organization_id)
+            .first()
+        )
+        if token is None:
+            return False
+        config = _find_project_config(db, ticket)
+        if config is None:
+            logger.info(
+                f"{ticket.display_number}: Jira escalation enabled but no agent "
+                "has a Jira project configured"
+            )
+            return False
+
+        from app.api.jira import CreateJiraIssueModel
+        from app.services.jira import JiraService
+
+        description_parts = [
+            f"Escalated from ReplyPier ticket {ticket.display_number} "
+            f"(priority: {ticket.priority}, severity: {ticket.severity or 'n/a'}).",
+        ]
+        if ticket.ai_summary:
+            description_parts.append(f"AI summary: {ticket.ai_summary}")
+        if ticket.description:
+            description_parts.append(ticket.description[:4000])
+
+        organization = db.query(Organization).filter(
+            Organization.id == ticket.organization_id
+        ).first()
+        result = await JiraService().create_issue(
+            organization,
+            db,
+            CreateJiraIssueModel(
+                projectKey=config.project_key,
+                issueTypeId=config.issue_type_id,
+                summary=f"[{ticket.display_number}] {ticket.title}"[:250],
+                description="\n\n".join(description_parts),
+            ),
+        )
+        issue_key = result.get("key")
+        if not issue_key:
+            return False
+        ticket.external_ref_type = "jira"
+        ticket.external_ref_id = issue_key
+        ticket.external_ref_url = f"{token.site_url}/browse/{issue_key}"
+        service._add_activity(
+            ticket,
+            TicketActivityType.JIRA_ESCALATED,
+            actor_type=TicketActorType.SYSTEM,
+            body=f"Escalated to Jira as {issue_key}",
+            metadata={"issue_key": issue_key, "url": ticket.external_ref_url},
+        )
+        return True
+    except Exception as e:
+        logger.error(f"Jira escalation failed for {ticket.display_number}: {e}")
+        return False

backend/app/services/ticket_privacy.py

@@ -0,0 +1,117 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Cross-customer privacy guards for AI ticketing.
+
+The investigator reasons over material that belongs to people other than the
+person it is writing to: similar past tickets from the same org, and rows from
+a connected database that has no row-level scoping. The same LLM call that
+reads all of that also writes `customer_summary`, which is delivered to the
+customer — at autonomy L3 with no human in the loop. Prompt wording alone can't
+be the control on that, so these are deterministic, applied at the boundary.
+"""
+
+import re
+from typing import Iterable, Optional, Set
+
+from app.core.logger import get_logger
+from app.utils.guardrails import GuardrailAction, PIIDetector
+
+logger = get_logger(__name__)
+
+# Identifiers that must never appear in a message we send out, no matter whose
+# they are — a support summary has no legitimate reason to restate them.
+ALWAYS_REDACT = ("ssn", "credit_card", "date_of_birth")
+
+# Identifiers that are fine to echo back to their *owner* but not to anyone
+# else. Kept when they match the recipient, redacted otherwise.
+OWNER_ONLY = ("email", "phone")
+
+THIRD_PARTY_MARKER = "[redacted]"
+
+
+def _digits(value: str) -> str:
+    return re.sub(r"\D", "", value or "")
+
+
+def _owner_keys(identifiers: Iterable[Optional[str]]) -> Set[str]:
+    """Normalized forms of the recipient's own identifiers, so a match is not
+    defeated by case or phone formatting."""
+    keys: Set[str] = set()
+    for raw in identifiers:
+        value = (raw or "").strip()
+        if not value:
+            continue
+        keys.add(value.lower())
+        digits = _digits(value)
+        # 7 digits filters out short numerics (order counts, amounts) that
+        # would otherwise whitelist themselves as if they were the phone.
+        if len(digits) >= 7:
+            keys.add(digits)
+    return keys
+
+
+def scrub_outbound(text: Optional[str], owner_identifiers: Iterable[Optional[str]] = ()) -> Optional[str]:
+    """Strip third-party identifiers from a message bound for the customer.
+
+    The recipient's own email/phone survive — telling someone their own address
+    is not a leak, and blanking it makes legitimate summaries read as broken.
+    Everything else that looks like an identifier is redacted, because in a
+    ticket summary it can only have come from another customer's record.
+    """
+    if not text:
+        return text
+
+    owners = _owner_keys(owner_identifiers)
+    scrubbed = text
+
+    for pii_type in ALWAYS_REDACT:
+        pattern = PIIDetector.PATTERNS.get(pii_type)
+        if pattern:
+            scrubbed = re.sub(pattern, THIRD_PARTY_MARKER, scrubbed)
+
+    for pii_type in OWNER_ONLY:
+        pattern = PIIDetector.PATTERNS.get(pii_type)
+        if not pattern:
+            continue
+
+        def _replace(match: re.Match) -> str:
+            value = match.group()
+            if value.lower() in owners or (_digits(value) and _digits(value) in owners):
+                return value
+            return THIRD_PARTY_MARKER
+
+        scrubbed = re.sub(pattern, _replace, scrubbed)
+
+    if scrubbed != text:
+        logger.warning(
+            "Redacted third-party identifiers from an outbound ticket message"
+        )
+    return scrubbed
+
+
+def redact_reference_text(text: Optional[str]) -> Optional[str]:
+    """Strip identifiers from another customer's material before it becomes
+    model context.
+
+    Similar past tickets are included to convey *what broke and how it was
+    fixed*; the names, addresses and numbers in their titles carry none of that
+    signal and are the most likely thing to be echoed into a summary written
+    for a different person.
+    """
+    if not text:
+        return text
+    result = PIIDetector.detect(text, action=GuardrailAction.REDACT)
+    return result.redacted_text or text

backend/app/services/whatsapp_outbound.py

@@ -0,0 +1,319 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import re
+import uuid
+from dataclasses import dataclass
+from typing import Optional
+from uuid import UUID
+
+from sqlalchemy.orm import Session
+
+from app.channels import get_adapter
+from app.channels.app_hooks import OutboundRecord, emit_channel_event, outbound_event
+from app.channels.constants import DEFAULT_TEMPLATE_LANGUAGE, MessageSource
+from app.services.channel_chat import placeholder_name
+from app.services.business_message import remember_business_message
+from app.services.channel_status import record_sent
+from app.services.chat_notifications import team_not_yet_notified
+from app.channels.meta_base import fetch_message_templates, graph_auth_refused, graph_detail
+from app.core.logger import get_logger
+from app.models.channels import ChannelAccount
+from app.repositories.channels import (
+    AgentChannelConfigRepository,
+    ChannelAccountRepository,
+    ChannelConversationRepository,
+)
+from app.repositories.chat import ChatRepository
+from app.repositories.customer import CustomerRepository
+from app.repositories.session_to_agent import SessionToAgentRepository
+from app.utils.phone import normalize_phone, to_wa_id
+
+logger = get_logger(__name__)
+
+# Business-initiated conversations are Utility/Authentication only. Marketing
+# is deliberately unsupported outbound: it carries opt-in obligations and
+# per-user caps (Graph error 131049) that a support product should not walk
+# its customers into. The reopen flow (an existing conversation) is unaffected.
+OUTBOUND_CATEGORIES = {"UTILITY", "AUTHENTICATION"}
+
+_PLACEHOLDER = re.compile(r"\{\{(\d+)\}\}")
+
+
+@dataclass
+class OutboundSent:
+    """A started (or rejoined) conversation and the template that opened it."""
+    session_id: UUID
+    message_id: int
+    external_message_id: Optional[str]
+
+
+class OutboundError(Exception):
+    """A refused or failed outbound send. Carries the HTTP status the API
+    route should answer with, so the Phase-2 scheduler can reuse the service
+    without importing anything web-shaped."""
+
+    def __init__(self, status_code: int, detail: str):
+        super().__init__(detail)
+        self.status_code = status_code
+        self.detail = detail
+
+
+def _render_body(template: dict, components: Optional[list]) -> str:
+    """The message text as the customer will read it.
+
+    Graph matches body parameters to {{n}} placeholders positionally in
+    ascending order (the same rule the frontend's previewTemplate applies —
+    keep the two honest with each other). Falls back to the raw body when a
+    parameter is missing rather than failing the send: Meta is the authority
+    on completeness and will reject a genuinely short send itself.
+    """
+    body = next((c.get("text", "") for c in template.get("components") or []
+                 if str(c.get("type", "")).upper() == "BODY"), "")
+    values = [
+        str(parameter.get("text", ""))
+        for component in components or []
+        if str(component.get("type", "")).lower() == "body"
+        for parameter in component.get("parameters") or []
+    ]
+    return _PLACEHOLDER.sub(
+        lambda match: values[int(match.group(1)) - 1]
+        if 0 < int(match.group(1)) <= len(values) else match.group(0),
+        body,
+    )
+
+
+async def _approved_outbound_template(db: Session, account: ChannelAccount,
+                                      name: str, language: str,
+                                      *, starts_conversation: bool) -> dict:
+    """The template this send is allowed to use, or the reason it isn't.
+
+    `starts_conversation` decides whether the Utility/Authentication rule
+    applies. It is a rule about business-INITIATED contact, so it belongs only
+    to a send that opens a thread with someone who never wrote to us. Reaching
+    an existing conversation is the reopen case: the customer did message us,
+    which is why Meta accepts Marketing there — and why the inbox picker has
+    never been category-gated.
+    """
+    repo = ChannelAccountRepository(db)
+    credentials = repo.get_credentials(account)
+    waba_id = credentials.get("waba_id")
+    if not waba_id:
+        raise OutboundError(400, "Reconnect this number with its WhatsApp "
+                                 "Business Account ID to send templates")
+
+    ok, data = await fetch_message_templates(waba_id, credentials["access_token"])
+    if not ok:
+        if graph_auth_refused(data):
+            repo.record_auth_failure(account, graph_detail(data, "Authentication Error"))
+        raise OutboundError(502, "Could not read the account's templates from Meta")
+
+    template = next(
+        (t for t in data if t.get("name") == name and t.get("language") == language),
+        None,
+    )
+    if template is None:
+        raise OutboundError(404, f"No template named {name} in {language} on this account")
+    if str(template.get("status", "")).upper() != "APPROVED":
+        raise OutboundError(400, f"Template {name} is not approved yet")
+    if starts_conversation and \
+            str(template.get("category", "")).upper() not in OUTBOUND_CATEGORIES:
+        raise OutboundError(
+            400,
+            "Only Utility and Authentication templates can start a conversation. "
+            "Marketing templates need the customer to have messaged you first.",
+        )
+    return template
+
+
+def _resolve_customer(db: Session, account: ChannelAccount, phone: str,
+                      customer_id: Optional[UUID], customer_name: Optional[str]):
+    """The person this conversation belongs to.
+
+    A picked person must actually OWN the number being messaged — the two
+    arrive as independent fields, and binding a conversation to one person at
+    another person's number is unrecoverable: the reply resolves by phone to
+    somebody else, so history splits across two customers and the thread never
+    appears on the profile of the person actually in it.
+
+    set_phone_if_absent's skip-and-log is right for the automatic inbound
+    paths, where there is nobody to tell. Here there is a human waiting on a
+    response, so a conflict is refused instead of silently ignored.
+    """
+    repo = CustomerRepository(db)
+    if customer_id is not None:
+        customer = repo.get_by_id(customer_id)
+        if customer is None or customer.organization_id != account.organization_id:
+            raise OutboundError(404, "Customer not found")
+        if customer.phone and customer.phone != phone:
+            raise OutboundError(
+                400,
+                f"{customer.full_name or 'That person'} has a different number on "
+                f"file. Send to their number, or update it on their profile first.",
+            )
+        owner = repo.get_customer_by_phone(phone, account.organization_id)
+        if owner is not None and owner.id != customer.id:
+            raise OutboundError(
+                400,
+                f"{phone} already belongs to {owner.full_name or 'another person'}. "
+                f"Pick them instead, or clear the number from their profile.",
+            )
+        repo.set_phone_if_absent(customer, phone)
+        return customer
+
+    wa_id = to_wa_id(phone)
+    customer = repo.get_or_create_customer(
+        email=f"{wa_id}@whatsapp.channel",
+        organization_id=account.organization_id,
+        full_name=customer_name or placeholder_name(account.channel_type, wa_id),
+        phone=phone,
+    )
+    # First-touch source for People's Source column; setdefault semantics so a
+    # person who arrived some other way keeps their original story.
+    if customer.lead_source is None:
+        customer.lead_source = {"channel": "whatsapp", "via": "outbound"}
+    return customer
+
+
+async def start_outbound_conversation(
+    db: Session,
+    account: ChannelAccount,
+    *,
+    to: str,
+    template_name: str,
+    language: str = DEFAULT_TEMPLATE_LANGUAGE,
+    components: Optional[list] = None,
+    customer_id: Optional[UUID] = None,
+    customer_name: Optional[str] = None,
+    source: MessageSource,
+) -> OutboundSent:
+    """Start (or rejoin) a WhatsApp conversation by phone number.
+
+    Returns the session — everything downstream (the inbox thread, template
+    resends, the AI answering the reply) already keys off one, which is why
+    this hands one back instead of growing a parallel outbound code path —
+    and the stored template message. `source` is who sent it (an inbox user,
+    or an external app). The team is told about the thread only once the
+    customer's reply needs them (notify_new_channel_chat_once), so a bulk
+    send doesn't page anyone.
+
+    The conversation is created with last_inbound_at=None: no customer message
+    means no 24-hour window, and pretending otherwise would let the AI send
+    free-form text that Meta rejects.
+    """
+    phone = normalize_phone(to)
+    if phone is None:
+        raise OutboundError(400, "Enter the number in international format, e.g. +91 12345 67890")
+    wa_id = to_wa_id(phone)
+
+    agent_id = AgentChannelConfigRepository(db).get_active_agent_id(account.id)
+    if agent_id is None:
+        raise OutboundError(400, "Route an agent to this number first — "
+                                 "otherwise nobody would answer the reply")
+
+    # Resolved BEFORE the template is validated: whether this opens a thread or
+    # rejoins one decides which rules the template must satisfy.
+    conv_repo = ChannelConversationRepository(db)
+    existing = conv_repo.get_active(account.id, wa_id)
+
+    template = await _approved_outbound_template(
+        db, account, template_name, language, starts_conversation=existing is None)
+
+    if existing is not None:
+        # An open conversation already carries this number; send there instead
+        # of splitting the thread. The window may even be open, but a template
+        # send inside the window is legal and simpler than branching.
+        conversation, session_id = existing, existing.session_id
+        customer = None
+    else:
+        customer = _resolve_customer(db, account, phone, customer_id, customer_name)
+        session_id = uuid.uuid4()
+        SessionToAgentRepository(db).create_session(
+            session_id=session_id,
+            agent_id=str(agent_id),
+            customer_id=str(customer.id),
+            organization_id=str(account.organization_id),
+            channel=account.channel_type,
+        )
+        conversation = conv_repo.create(
+            channel_account_id=account.id,
+            channel_type=account.channel_type,
+            external_conversation_id=wa_id,
+            external_user_id=wa_id,
+            session_id=session_id,
+            organization_id=account.organization_id,
+            agent_id=agent_id,
+            customer_id=customer.id,
+            last_inbound_at=None,
+            extra=team_not_yet_notified(),
+        )
+
+    adapter = get_adapter("whatsapp")
+    result = await adapter.send_template(account, conversation, template_name,
+                                         language, components)
+    ChannelAccountRepository(db).track_send_result(account, result)
+    if not result.ok:
+        if existing is None:
+            # A failed send must not leave an empty thread in the inbox: drop
+            # the conversation, then the session it points at. The customer
+            # row stays — harmless, and reused on the next attempt.
+            #
+            # Guarded, because an exception escaping here would replace the 502
+            # (which names Meta's actual reason) with an opaque 500 AND leave
+            # the empty thread this block exists to remove. Failing to clean up
+            # is worth a log; losing the reason the send failed is not.
+            from app.models.session_to_agent import SessionToAgent
+            try:
+                db.delete(conversation)
+                db.query(SessionToAgent).filter(
+                    SessionToAgent.session_id == session_id).delete()
+                db.commit()
+            except Exception as e:
+                db.rollback()
+                logger.error(f"Could not roll back the empty outbound thread for "
+                             f"session {session_id}: {e}")
+        raise OutboundError(502, result.error or "WhatsApp did not accept the message")
+
+    rendered = _render_body(template, components)
+    stored = ChatRepository(db).create_message({
+        "message": rendered,
+        "message_type": "bot",
+        "session_id": str(session_id),
+        "organization_id": str(account.organization_id),
+        "agent_id": str(agent_id),
+        # A reused conversation row predating this feature can lack a customer.
+        "customer_id": str(conversation.customer_id) if conversation.customer_id else None,
+        "attributes": {
+            "channel": account.channel_type,
+            "external_message_id": result.external_message_id,
+            "outbound_template": template_name,
+            "source": source.value,
+        },
+    })
+    if result.external_message_id:
+        await record_sent(db, stored.id, account.id, result.external_message_id)
+    # The AI reads this when the customer replies: a bare "yes" is meaningless
+    # without knowing what we asked.
+    remember_business_message(db, conversation, rendered, result.external_message_id)
+    await emit_channel_event(outbound_event(
+        organization_id=account.organization_id, channel_account_id=account.id,
+        session_id=str(session_id),
+        customer_id=str(conversation.customer_id) if conversation.customer_id else None,
+        record=OutboundRecord(message_id=stored.id, external_message_id=result.external_message_id,
+                              source=source, text=rendered),
+    ))
+    return OutboundSent(session_id=session_id, message_id=stored.id,
+                        external_message_id=result.external_message_id)

backend/app/services/workflow.py

@@ -0,0 +1,185 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from typing import Optional
+from uuid import UUID
+from sqlalchemy.orm import Session
+from app.repositories.workflow import WorkflowRepository
+from app.repositories.agent import AgentRepository
+from app.models.workflow import Workflow
+from app.models.schemas.workflow import WorkflowCreate
+from app.core.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class WorkflowService:
+    def __init__(self, db: Session):
+        self.db = db
+        self.workflow_repo = WorkflowRepository(db)
+        self.agent_repo = AgentRepository(db)
+
+    def create_workflow(self, workflow_data: WorkflowCreate, created_by: UUID, organization_id: UUID) -> Workflow:
+        """Create a new workflow with validation"""
+        
+        # Validate agent exists and belongs to organization
+        agent = self.agent_repo.get_by_id(workflow_data.agent_id)
+        if not agent:
+            raise ValueError("Agent not found")
+        
+        if agent.organization_id != organization_id:
+            raise ValueError("Agent does not belong to your organization")
+        
+        # Check if workflow already exists for this agent
+        existing_workflow = self.workflow_repo.get_workflows_by_agent(workflow_data.agent_id)
+        if existing_workflow:
+            raise ValueError("Agent already has a workflow. Only one workflow per agent is allowed.")
+        
+        # Check if workflow name already exists in organization
+        existing_workflow_name = self.workflow_repo.get_by_name(
+            workflow_data.name, 
+            organization_id
+        )
+        if existing_workflow_name:
+            raise ValueError(f"Workflow with name '{workflow_data.name}' already exists")
+        
+        # Create workflow data
+        workflow_dict = workflow_data.model_dump()
+        workflow_dict.update({
+            "created_by": created_by,
+            "organization_id": organization_id
+        })
+        
+        # Create workflow
+        workflow = self.workflow_repo.create_workflow(**workflow_dict)
+        
+        logger.info(f"Created workflow {workflow.id} for agent {workflow_data.agent_id}")
+        return workflow
+
+    def get_workflow_by_agent_id(self, agent_id: UUID, organization_id: UUID) -> Optional[Workflow]:
+        """Get workflow by agent ID with organization validation"""
+        
+        # Validate agent exists and belongs to organization
+        agent = self.agent_repo.get_by_id(agent_id)
+        if not agent:
+            raise ValueError("Agent not found")
+        
+        if agent.organization_id != organization_id:
+            raise ValueError("Agent does not belong to your organization")
+        
+        # Get workflows for this agent
+        workflows = self.workflow_repo.get_workflows_by_agent(agent_id)
+        
+        # Return the first workflow (should be only one due to our constraint)
+        return workflows[0] if workflows else None
+
+    def update_workflow(self, workflow_id: UUID, workflow_data: dict, organization_id: UUID) -> Optional[Workflow]:
+        """Update workflow with validation"""
+        
+        # Get existing workflow
+        workflow = self.workflow_repo.get_by_id(workflow_id)
+        if not workflow:
+            raise ValueError("Workflow not found")
+        
+        # Verify organization access
+        if workflow.organization_id != organization_id:
+            raise ValueError("Workflow does not belong to your organization")
+        
+        # If name is being updated, check for duplicates
+        if 'name' in workflow_data and workflow_data['name'] != workflow.name:
+            existing_workflow = self.workflow_repo.get_by_name(
+                workflow_data['name'], 
+                organization_id
+            )
+            if existing_workflow and existing_workflow.id != workflow_id:
+                raise ValueError(f"Workflow with name '{workflow_data['name']}' already exists")
+        
+        # If agent_id is being updated, validate the new agent and check constraint
+        if 'agent_id' in workflow_data and workflow_data['agent_id'] != workflow.agent_id:
+            # Validate new agent exists and belongs to organization
+            agent = self.agent_repo.get_by_id(workflow_data['agent_id'])
+            if not agent:
+                raise ValueError("Agent not found")
+            
+            if agent.organization_id != organization_id:
+                raise ValueError("Agent does not belong to your organization")
+            
+            # Check if new agent already has a workflow
+            existing_workflows = self.workflow_repo.get_workflows_by_agent(workflow_data['agent_id'])
+            if existing_workflows:
+                raise ValueError("Agent already has a workflow. Only one workflow per agent is allowed.")
+        
+        # Check if status is being updated to published
+        if 'status' in workflow_data and workflow_data['status'] == 'published':
+            # When publishing, update the agent's active_workflow_id and use_workflow
+            agent = self.agent_repo.get_by_id(workflow.agent_id)
+            if agent:
+                self.agent_repo.update_agent(
+                    workflow.agent_id, 
+                    active_workflow_id=workflow_id,
+                    use_workflow=True
+                )
+                logger.info(f"Updated agent {workflow.agent_id} active_workflow_id to {workflow_id}")
+        
+        # Check if status is being updated to draft (unpublished)
+        if 'status' in workflow_data and workflow_data['status'] == 'draft':
+            # When unpublishing, clear the agent's active_workflow_id and disable workflow
+            agent = self.agent_repo.get_by_id(workflow.agent_id)
+            if agent and agent.active_workflow_id == workflow_id:
+                self.agent_repo.update_agent(
+                    workflow.agent_id, 
+                    active_workflow_id=None,
+                    use_workflow=False
+                )
+                logger.info(f"Cleared agent {workflow.agent_id} active_workflow_id")
+        
+        # Update workflow
+        updated_workflow = self.workflow_repo.update_workflow(workflow_id, **workflow_data)
+        
+        logger.info(f"Updated workflow {workflow_id}")
+        return updated_workflow
+
+    def delete_workflow(self, workflow_id: UUID, organization_id: UUID) -> bool:
+        """Delete workflow with validation"""
+        
+        # Get existing workflow
+        workflow = self.workflow_repo.get_by_id(workflow_id)
+        if not workflow:
+            raise ValueError("Workflow not found")
+        
+        # Verify organization access
+        if workflow.organization_id != organization_id:
+            raise ValueError("Workflow does not belong to your organization")
+        
+        # Clear agent's active_workflow_id if this workflow is active
+        agent = self.agent_repo.get_by_id(workflow.agent_id)
+        if agent and agent.active_workflow_id == workflow_id:
+            self.agent_repo.update_agent(
+                workflow.agent_id, 
+                active_workflow_id=None,
+                use_workflow=False
+            )
+            logger.info(f"Cleared agent {workflow.agent_id} active_workflow_id before deleting workflow")
+        
+        # Delete workflow
+        success = self.workflow_repo.delete_workflow(workflow_id)
+        
+        if success:
+            logger.info(f"Deleted workflow {workflow_id}")
+        
+        return success
+
+ 

backend/app/services/workflow_chat.py

@@ -0,0 +1,393 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import asyncio
+from app.core.logger import get_logger
+from app.agents.chat_agent import ChatAgent, ChatResponse
+from app.repositories.chat import ChatRepository
+from app.repositories.session_to_agent import SessionToAgentRepository
+from app.services.workflow_execution import WorkflowExecutionService
+from app.core.security import decrypt_api_key
+from app.models.schemas.chat import TransferReasonType
+
+logger = get_logger(__name__)
+
+
+class WorkflowChatService:
+    """Service for handling workflow-based chat processing"""
+    
+    def __init__(self, db):
+        self.db = db
+        self.chat_repo = ChatRepository(db)
+        self.session_repo = SessionToAgentRepository(db)
+        self.workflow_service = WorkflowExecutionService(db)
+    
+    async def handle_workflow_chat(
+        self,
+        active_session,
+        message: str,
+        session_id: str,
+        org_id: str,
+        customer_id: str,
+        session: dict,
+        sio,
+        namespace: str = '/widget',
+        source: str = None
+    ) -> ChatResponse:
+        """
+        Handle chat processing for workflow-enabled agents
+        
+        Args:
+            active_session: The active session object
+            message: User message
+            session_id: Session ID
+            org_id: Organization ID
+            customer_id: Customer ID
+            session: Session data containing agent and AI config
+            sio: SocketIO instance for emitting events
+            namespace: SocketIO namespace
+            
+        Returns:
+            ChatResponse: The response from workflow execution
+        """
+        
+        # Store user message in chat history
+        self.chat_repo.create_message({
+            "message": message,
+            "message_type": "user",
+            "session_id": session_id,
+            "organization_id": org_id,
+            "agent_id": session['agent_id'],
+            "customer_id": customer_id,
+        })
+        
+        # Execute workflow
+        workflow_result = await self.workflow_service.execute_workflow(
+            session_id=session_id,
+            user_message=message,
+            workflow_id=active_session.workflow_id,
+            current_node_id=active_session.current_node_id,
+            workflow_state=active_session.workflow_state,
+            api_key=decrypt_api_key(session['ai_config'].encrypted_api_key),
+            model_name=session['ai_config'].model_name,
+            model_type=session['ai_config'].model_type,
+            base_url=(session['ai_config'].settings or {}).get('base_url'),
+            org_id=org_id,
+            agent_id=session['agent_id'],
+            customer_id=customer_id,
+            source=source
+        )
+        
+        if workflow_result.success:
+            return await self._handle_successful_workflow(
+                workflow_result, active_session, session_id, org_id, 
+                customer_id, session, sio, namespace
+            )
+        else:
+            return await self._handle_failed_workflow(
+                workflow_result, session_id, org_id, session, customer_id
+            )
+    
+    async def _handle_successful_workflow(
+        self, 
+        workflow_result, 
+        active_session, 
+        session_id: str, 
+        org_id: str, 
+        customer_id: str, 
+        session: dict, 
+        sio, 
+        namespace: str
+    ) -> ChatResponse:
+        """Handle successful workflow execution"""
+        
+
+        
+        # Handle intermediate messages from MESSAGE nodes during automatic execution
+        if workflow_result.intermediate_messages:
+            await self._handle_intermediate_messages(
+                workflow_result.intermediate_messages, session_id, org_id, 
+                session, customer_id, active_session, sio, namespace
+            )
+        
+        # Check if this is a form node that needs to display a form
+        if workflow_result.form_data:
+            await self._handle_form_display(workflow_result, session_id, sio, namespace)
+            return None  # Don't send a regular chat response for form display
+        
+        # Create ChatResponse object from workflow result
+        response = ChatResponse(
+            message=workflow_result.message,
+            transfer_to_human=workflow_result.transfer_to_human,
+            transfer_reason=None,
+            transfer_description=None,
+            end_chat=workflow_result.end_chat,
+            request_rating=workflow_result.request_rating,
+            create_ticket=False
+        )
+        
+        # Handle transfer to human if requested by workflow LLM
+        if workflow_result.transfer_to_human:
+            transfer_response = await self._handle_workflow_transfer(
+                workflow_result, session_id, org_id, session, customer_id, response
+            )
+            if transfer_response:
+                return transfer_response
+        
+        # Handle end chat if requested by workflow
+        if workflow_result.end_chat:
+            response = await self._handle_workflow_end_chat(
+                response, session_id, org_id, session, customer_id
+            )
+        
+        # Only store message if there's actual content
+        if workflow_result.message:
+            self._store_workflow_response(
+                response, session_id, org_id, session, customer_id, 
+                active_session, workflow_result
+            )
+        
+        return response
+    
+    async def _handle_intermediate_messages(
+        self, 
+        intermediate_messages, 
+        session_id: str, 
+        org_id: str, 
+        session: dict, 
+        customer_id: str, 
+        active_session, 
+        sio, 
+        namespace: str
+    ):
+        """Handle intermediate messages from MESSAGE nodes"""
+        
+        logger.info(f"Found {len(intermediate_messages)} intermediate messages from MESSAGE nodes")
+        
+        for intermediate_message in intermediate_messages:
+            logger.debug(f"Emitting intermediate message: {intermediate_message}")
+            
+            # Store intermediate message in chat history
+            self.chat_repo.create_message({
+                "message": intermediate_message,
+                "message_type": "bot",
+                "session_id": session_id,
+                "organization_id": org_id,
+                "agent_id": session['agent_id'],
+                "customer_id": customer_id,
+                "attributes": {
+                    "workflow_execution": True,
+                    "workflow_id": str(active_session.workflow_id),
+                    "intermediate_message": True,
+                    "message_node": True
+                }
+            })
+            
+            # Emit intermediate message to client immediately
+            await sio.emit('chat_response', {
+                'message': intermediate_message,
+                'type': 'chat_response',
+                'transfer_to_human': False,
+                'end_chat': False,
+                'request_rating': False,
+                'shopify_output': None
+            }, room=session_id, namespace=namespace)
+    
+    async def _handle_form_display(self, workflow_result, session_id: str, sio, namespace: str):
+        """Handle form display for form nodes"""
+        
+        logger.info(f"Form data detected: {workflow_result.form_data}")
+        logger.info("Emitting display_form event")
+        
+        # Emit form display to client
+        await sio.emit('display_form', {
+            'form_data': workflow_result.form_data,
+            'session_id': session_id
+        }, room=session_id, namespace=namespace)
+        
+        logger.info("Form display event emitted successfully")
+    
+    async def _handle_workflow_transfer(
+        self, 
+        workflow_result, 
+        session_id: str, 
+        org_id: str, 
+        session: dict, 
+        customer_id: str, 
+        response: ChatResponse
+    ) -> ChatResponse:
+        """Handle transfer to human requested by workflow"""
+        
+        logger.info(f"Workflow LLM requested transfer to human for session {session_id}")
+
+        if not workflow_result.transfer_group_id:
+            logger.warning(
+                f"No transfer_group_id provided for workflow transfer in session {session_id}; "
+                "falling back to the agent's default group")
+
+        chat_agent = await ChatAgent.create_async(
+            api_key=decrypt_api_key(session['ai_config'].encrypted_api_key),
+            model_name=session['ai_config'].model_name,
+            model_type=session['ai_config'].model_type,
+            base_url=(session['ai_config'].settings or {}).get('base_url'),
+            org_id=org_id,
+            agent_id=session['agent_id'],
+            customer_id=customer_id,
+            session_id=session_id
+        )
+
+        try:
+            # Create ChatResponse object with transfer details from workflow
+            llm_response = ChatResponse(
+                message=workflow_result.message,
+                transfer_to_human=True,
+                transfer_reason=TransferReasonType(workflow_result.transfer_reason) if workflow_result.transfer_reason else None,
+                transfer_description=workflow_result.transfer_description,
+                end_chat=False,
+                request_rating=False,
+                create_ticket=False
+            )
+
+            # A missing group id is passed through rather than short-circuited:
+            # the transfer path then falls back to the agent's own default
+            # group, as a plain agent transfer does. Setting the status alone
+            # left group_id NULL, and an inbox matches on user_id or group_id —
+            # so the handover was invisible to every agent it was meant for.
+            transfer_response = await chat_agent.handle_workflow_transfer(
+                session_id=session_id,
+                org_id=org_id,
+                agent_id=session['agent_id'],
+                customer_id=customer_id,
+                transfer_group_id=workflow_result.transfer_group_id,
+                db=self.db,
+                chat_repo=self.chat_repo,
+                llm_response=llm_response
+            )
+
+            logger.info(f"Transfer completed for session {session_id} to group {workflow_result.transfer_group_id}")
+            # Mark this as a transfer response so widget_chat.py can handle it properly
+            transfer_response._is_transfer_response = True
+            return transfer_response
+        except ValueError as e:
+            # The agent has no groups at all, so there is nobody to route to.
+            # Mark it transferred anyway — an admin can still see and claim it.
+            logger.error(
+                f"Workflow transfer for session {session_id} could not be routed to a group: {e}")
+            self.session_repo.update_session_status(session_id, "TRANSFERRED")
+            return response
+        finally:
+            await chat_agent.safe_cleanup_mcp_tools()
+    
+    async def _handle_workflow_end_chat(
+        self, 
+        response: ChatResponse, 
+        session_id: str, 
+        org_id: str, 
+        session: dict, 
+        customer_id: str
+    ) -> ChatResponse:
+        """Handle end chat requested by workflow"""
+        
+        logger.info(f"Workflow requested end chat for session {session_id}")
+        
+        # Create a ChatAgent instance to use the _handle_end_chat method
+        chat_agent = await ChatAgent.create_async(
+            api_key=decrypt_api_key(session['ai_config'].encrypted_api_key),
+            model_name=session['ai_config'].model_name,
+            model_type=session['ai_config'].model_type,
+            base_url=(session['ai_config'].settings or {}).get('base_url'),
+            org_id=org_id,
+            agent_id=session['agent_id'],
+            customer_id=customer_id,
+            session_id=session_id
+        )
+        
+        try:
+            # Handle end chat using the agent's method with rating from response
+            # The response.request_rating value comes from the workflow execution config
+            return await chat_agent._handle_end_chat(response, session_id, self.db, response.request_rating)
+        finally:
+            await chat_agent.safe_cleanup_mcp_tools()
+    
+    def _store_workflow_response(
+        self, 
+        response: ChatResponse, 
+        session_id: str, 
+        org_id: str, 
+        session: dict, 
+        customer_id: str, 
+        active_session, 
+        workflow_result
+    ):
+        """Store workflow response in chat history"""
+        
+        self.chat_repo.create_message({
+            "message": response.message,
+            "message_type": "bot",
+            "session_id": session_id,
+            "organization_id": org_id,
+            "agent_id": session['agent_id'],
+            "customer_id": customer_id,
+            "attributes": {
+                "workflow_execution": True,
+                "workflow_id": str(active_session.workflow_id),
+                "current_node_id": str(workflow_result.next_node_id) if workflow_result.next_node_id else None,
+                "transfer_to_human": response.transfer_to_human,
+                "end_chat": response.end_chat,
+                "request_rating": response.request_rating,
+            }
+        })
+    
+    async def _handle_failed_workflow(
+        self, 
+        workflow_result, 
+        session_id: str, 
+        org_id: str, 
+        session: dict, 
+        customer_id: str
+    ) -> ChatResponse:
+        """Handle failed workflow execution"""
+        
+        logger.error(f"Workflow execution failed: {workflow_result.error}")
+        
+        response = ChatResponse(
+            message=workflow_result.message or "I apologize, but I'm having trouble processing your request right now.",
+            transfer_to_human=False,
+            transfer_reason=None,
+            transfer_description=None,
+            end_chat=False,
+            request_rating=False,
+            create_ticket=False
+        )
+        
+        # Store error response in chat history
+        self.chat_repo.create_message({
+            "message": response.message,
+            "message_type": "bot",
+            "session_id": session_id,
+            "organization_id": org_id,
+            "agent_id": session['agent_id'],
+            "customer_id": customer_id,
+            "attributes": {
+                "workflow_execution": True,
+                "workflow_error": workflow_result.error,
+                "transfer_to_human": response.transfer_to_human,
+                "end_chat": response.end_chat,
+                "request_rating": response.request_rating,
+            }
+        })
+        
+        return response 

backend/app/services/workflow_conditions.py

@@ -0,0 +1,291 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+# Evaluates workflow condition nodes.
+#
+# The structure of a condition is fixed before any variable is looked up, and
+# variable values are only ever compared, never parsed. A visitor's answer such
+# as `x" OR "a" === "a` is therefore just text and cannot steer routing.
+
+import re
+from typing import Any, Dict, List, Optional, Tuple
+
+from app.core.logger import get_logger
+
+logger = get_logger(__name__)
+
+# Returned by resolve_variable when no value is stored under the name.
+MISSING = object()
+
+OPERATORS = frozenset({"===", "!==", ">", "<", ">=", "<=", "includes", "startsWith", "endsWith"})
+_ALIASES = {"==": "===", "!=": "!==", "contains": "includes"}
+_NUMERIC = {
+    ">": lambda a, b: a > b,
+    "<": lambda a, b: a < b,
+    ">=": lambda a, b: a >= b,
+    "<=": lambda a, b: a <= b,
+}
+_NUMBER = re.compile(r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?")
+
+
+def resolve_variable(name: str, workflow_state: Optional[Dict[str, Any]]) -> Any:
+    """Look up a `{{name}}` value: `node_id.field`, a top-level variable such as
+    user_message, or a field from any node (latest wins). MISSING if absent."""
+    variables = (workflow_state or {}).get("variables") or {}
+    node_variables = {k: v for k, v in variables.items() if isinstance(v, dict)}
+    name = name.strip()
+
+    if "." in name:
+        node_id, field_name = (part.strip() for part in name.split(".", 1))
+        scoped = node_variables.get(node_id, {})
+        return scoped[field_name] if field_name in scoped else MISSING
+
+    if name in variables and name not in node_variables:
+        return variables[name]
+    for node_id in reversed(list(node_variables)):
+        if name in node_variables[node_id]:
+            return node_variables[node_id][name]
+    return MISSING
+
+
+def _as_text(value: Any) -> Optional[str]:
+    if value is MISSING or value is None:
+        return None
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    return str(value).strip()
+
+
+def _as_number(text: str) -> Optional[float]:
+    return float(text) if _NUMBER.fullmatch(text) else None
+
+
+def compare(left: Any, operator: str, right: Any) -> bool:
+    """Apply one operator to two values. Case-sensitive; surrounding whitespace
+    is ignored. A missing value only satisfies `!==`."""
+    operator = _ALIASES.get(operator, operator)
+    if operator not in OPERATORS:
+        logger.warning(f"Unknown condition operator '{operator}', treating as false")
+        return False
+
+    left_text, right_text = _as_text(left), _as_text(right)
+    if operator == "!==":
+        return left_text != right_text
+    if left_text is None or right_text is None:
+        return False
+    if operator == "===":
+        return left_text == right_text
+    if operator == "includes":
+        return right_text in left_text
+    if operator == "startsWith":
+        return left_text.startswith(right_text)
+    if operator == "endsWith":
+        return left_text.endswith(right_text)
+
+    left_num, right_num = _as_number(left_text), _as_number(right_text)
+    if left_num is None or right_num is None:
+        logger.debug(f"Non-numeric '{operator}' comparison is false: {left_text!r} vs {right_text!r}")
+        return False
+    return _NUMERIC[operator](left_num, right_num)
+
+
+# --- Builder groups -------------------------------------------------------
+
+def _rule_is_complete(rule: Any) -> bool:
+    # Mirrors the dashboard, which leaves half-filled rules out of the expression.
+    if not isinstance(rule, dict):
+        return False
+    value = rule.get("value")
+    return bool(rule.get("variable")) and bool(rule.get("operator")) and value is not None and str(value).strip() != ""
+
+
+def _evaluate_rule(rule: Dict[str, Any], workflow_state: Dict[str, Any]) -> bool:
+    name = str(rule["variable"]).strip().removeprefix("{{").removesuffix("}}")
+    left = resolve_variable(name, workflow_state)
+    if left is MISSING:
+        logger.warning(f"Condition variable '{name}' has no value")
+    return compare(left, str(rule["operator"]), rule["value"])
+
+
+def evaluate_groups(groups: List[Any], workflow_state: Dict[str, Any]) -> Optional[bool]:
+    """Groups are OR'ed; rules within a group use its logicalOperator (AND by
+    default). None when no group holds a complete rule."""
+    results = []
+    for group in groups:
+        if not isinstance(group, dict):
+            continue
+        rules = [rule for rule in (group.get("rules") or []) if _rule_is_complete(rule)]
+        if not rules:
+            continue
+        outcomes = [_evaluate_rule(rule, workflow_state) for rule in rules]
+        any_rule = str(group.get("logicalOperator", "AND")).upper() == "OR"
+        results.append(any(outcomes) if any_rule else all(outcomes))
+    return any(results) if results else None
+
+
+# --- Legacy expression strings ---------------------------------------------
+
+class ConditionSyntaxError(ValueError):
+    pass
+
+
+_TOKEN = re.compile(r"""
+    \s*(?:
+        (?P<var>\{\{[^{}]+\}\})
+      | (?P<str>"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')
+      | (?P<method>\.(?:includes|startsWith|endsWith)\()
+      | (?P<op>===|!==|==|!=|>=|<=|>|<|&&|\|\|)
+      | (?P<paren>[()])
+      | (?P<word>[^\s(){}"'=!<>&|.]+(?:\.(?!(?:includes|startsWith|endsWith)\()[^\s(){}"'=!<>&|.]+)*)
+    )\s*""", re.VERBOSE)
+_METHODS = frozenset({"includes", "startsWith", "endsWith"})
+_WORD_KEYWORDS = {"AND": ("and", None), "OR": ("or", None), "contains": ("op", "contains")}
+
+Token = Tuple[str, Optional[str]]
+
+
+def _tokenize(expression: str) -> List[Token]:
+    tokens: List[Token] = []
+    pos = 0
+    while pos < len(expression):
+        match = _TOKEN.match(expression, pos)
+        if not match or match.end() == pos:
+            if expression[pos:].strip():
+                raise ConditionSyntaxError(f"unexpected text at position {pos}")
+            break
+        pos = match.end()
+        kind = match.lastgroup
+        text = match.group(kind)
+        if kind == "var":
+            tokens.append(("var", text[2:-2].strip()))
+        elif kind == "str":
+            tokens.append(("lit", re.sub(r"\\(.)", r"\1", text[1:-1])))
+        elif kind == "method":
+            tokens.append(("op", text[1:-1]))
+        elif kind == "op" and text in ("&&", "||"):
+            tokens.append(("and" if text == "&&" else "or", None))
+        elif kind == "paren":
+            tokens.append((text, None))
+        elif kind == "word":
+            tokens.append(_WORD_KEYWORDS.get(text, ("lit", text)))
+        else:
+            tokens.append((kind, text))
+    return tokens
+
+
+class _Parser:
+    """or := and (OR and)* ; and := primary (AND primary)* ;
+    primary := '(' or ')' | operand op operand | operand .method( operand )"""
+
+    def __init__(self, tokens: List[Token]):
+        self.tokens = tokens
+        self.pos = 0
+
+    def parse(self):
+        if not self.tokens:
+            raise ConditionSyntaxError("empty condition")
+        node = self._or()
+        if self.pos != len(self.tokens):
+            raise ConditionSyntaxError("unexpected trailing tokens")
+        return node
+
+    def _peek(self) -> Optional[str]:
+        return self.tokens[self.pos][0] if self.pos < len(self.tokens) else None
+
+    def _take(self, kind: str) -> Token:
+        if self._peek() != kind:
+            raise ConditionSyntaxError(f"expected {kind}")
+        token = self.tokens[self.pos]
+        self.pos += 1
+        return token
+
+    def _or(self):
+        items = [self._and()]
+        while self._peek() == "or":
+            self.pos += 1
+            items.append(self._and())
+        return ("or", items) if len(items) > 1 else items[0]
+
+    def _and(self):
+        items = [self._primary()]
+        while self._peek() == "and":
+            self.pos += 1
+            items.append(self._primary())
+        return ("and", items) if len(items) > 1 else items[0]
+
+    def _primary(self):
+        if self._peek() == "(":
+            self.pos += 1
+            node = self._or()
+            self._take(")")
+            return node
+        left = self._operand()
+        _, operator = self._take("op")
+        right = self._operand()
+        if operator in _METHODS:
+            self._take(")")  # closes `.includes(`
+        return ("cmp", left, operator, right)
+
+    def _operand(self) -> Token:
+        if self._peek() in ("var", "lit"):
+            token = self.tokens[self.pos]
+            self.pos += 1
+            return token
+        raise ConditionSyntaxError("expected a value")
+
+
+def _evaluate_node(node, workflow_state: Dict[str, Any]) -> bool:
+    kind = node[0]
+    if kind == "or":
+        return any(_evaluate_node(item, workflow_state) for item in node[1])
+    if kind == "and":
+        return all(_evaluate_node(item, workflow_state) for item in node[1])
+    _, left, operator, right = node
+    return compare(_operand_value(left, workflow_state), operator, _operand_value(right, workflow_state))
+
+
+def _operand_value(token: Token, workflow_state: Dict[str, Any]) -> Any:
+    kind, text = token
+    if kind == "lit":
+        return text
+    value = resolve_variable(text, workflow_state)
+    if value is MISSING:
+        logger.warning(f"Condition variable '{text}' has no value")
+    return value
+
+
+def evaluate_expression(expression: str, workflow_state: Dict[str, Any]) -> bool:
+    """Evaluate a stored `condition_expression`. Unreadable conditions are false."""
+    try:
+        tree = _Parser(_tokenize(expression)).parse()
+    except (ConditionSyntaxError, RecursionError) as e:
+        logger.warning(f"Unreadable condition {expression!r} ({e}), treating as false")
+        return False
+    return _evaluate_node(tree, workflow_state)
+
+
+def evaluate_condition_config(config: Dict[str, Any], workflow_state: Dict[str, Any]) -> Optional[bool]:
+    """Evaluate a condition node's config, preferring the builder's groups over
+    the generated expression. None when neither holds a usable condition."""
+    groups = config.get("condition_groups")
+    if isinstance(groups, list):
+        result = evaluate_groups(groups, workflow_state)
+        if result is not None:
+            return result
+    expression = config.get("condition_expression")
+    if isinstance(expression, str) and expression.strip():
+        return evaluate_expression(expression, workflow_state)
+    return None

backend/app/services/workflow_node.py

@@ -0,0 +1,463 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from typing import List, Dict, Any, Optional
+from uuid import UUID
+import uuid
+import re
+from sqlalchemy.orm import Session
+from app.repositories.workflow_node import WorkflowNodeRepository
+from app.repositories.workflow import WorkflowRepository
+
+from app.core.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def sanitize_utf8_text(text: str) -> str:
+    """
+    Sanitize text to remove invalid UTF-8 characters and surrogates
+    """
+    if not text:
+        return text
+    
+    # Remove surrogate characters (like \udcac, \udd16)
+    text = re.sub(r'[\ud800-\udfff]', '', text)
+    
+    # Encode to UTF-8 and decode to handle any remaining issues
+    try:
+        text = text.encode('utf-8', errors='ignore').decode('utf-8')
+    except UnicodeError:
+        # If still failing, remove all non-ASCII characters
+        text = re.sub(r'[^\x00-\x7F]', '', text)
+    
+    return text.strip()
+
+
+class WorkflowNodeService:
+    def __init__(self, db: Session):
+        self.db = db
+        self.node_repo = WorkflowNodeRepository(db)
+        self.workflow_repo = WorkflowRepository(db)
+    
+    def _generate_unique_node_name(self, workflow_id: UUID, base_name: str, exclude_node_id: UUID = None) -> str:
+        """
+        Generate a unique node name by appending _001, _002, etc. if needed
+        
+        Args:
+            workflow_id: The workflow ID to check within
+            base_name: The base name to make unique
+            exclude_node_id: Node ID to exclude from uniqueness check (for updates)
+            
+        Returns:
+            Unique node name
+        """
+        try:
+            # Get all existing nodes in the workflow
+            workflow = self.workflow_repo.get_by_id(workflow_id)
+            if not workflow:
+                return base_name
+            
+            existing_names = set()
+            for node in workflow.nodes:
+                if exclude_node_id and node.id == exclude_node_id:
+                    continue  # Skip the node being updated
+                if node.name:
+                    existing_names.add(node.name.lower())
+            
+            # Check if base name is already unique
+            if base_name.lower() not in existing_names:
+                return base_name
+            
+            # Generate unique name with suffix
+            counter = 1
+            while True:
+                candidate_name = f"{base_name}_{counter:03d}"
+                if candidate_name.lower() not in existing_names:
+                    return candidate_name
+                counter += 1
+                
+                # Safety check to prevent infinite loop
+                if counter > 999:
+                    # Fallback to UUID suffix
+                    return f"{base_name}_{str(uuid.uuid4())[:8]}"
+                    
+        except Exception as e:
+            logger.error(f"Error generating unique node name: {str(e)}")
+            # Fallback to original name with UUID suffix
+            return f"{base_name}_{str(uuid.uuid4())[:8]}"
+    
+    def _validate_node_name_uniqueness(self, workflow_id: UUID, node_name: str, exclude_node_id: UUID = None) -> bool:
+        """
+        Validate that a node name is unique within the workflow
+        
+        Args:
+            workflow_id: The workflow ID to check within
+            node_name: The name to validate
+            exclude_node_id: Node ID to exclude from uniqueness check (for updates)
+            
+        Returns:
+            True if name is unique, False otherwise
+        """
+        try:
+            workflow = self.workflow_repo.get_by_id(workflow_id)
+            if not workflow:
+                return True
+            
+            for node in workflow.nodes:
+                if exclude_node_id and node.id == exclude_node_id:
+                    continue  # Skip the node being updated
+                if node.name and node.name.lower() == node_name.lower():
+                    return False
+            
+            return True
+            
+        except Exception as e:
+            logger.error(f"Error validating node name uniqueness: {str(e)}")
+            return False
+
+    def _ensure_unicode_support_in_config(self, config: dict) -> dict:
+        """
+        Ensure Unicode characters are properly preserved in config data
+        by applying JSON serialization with ensure_ascii=False
+        """
+        import json
+        import copy
+        
+        if not config:
+            return config
+            
+        try:
+            # Create a deep copy to avoid modifying the original
+            config_copy = copy.deepcopy(config)
+            
+            # Convert to JSON with ensure_ascii=False to preserve Unicode characters
+            # Then parse it back to ensure it's properly formatted for database storage
+            json_str = json.dumps(config_copy, ensure_ascii=False, separators=(',', ':'))
+            unicode_safe_config = json.loads(json_str)
+            
+            logger.debug(f"Applied Unicode-safe processing to config: {len(str(config_copy))} chars -> {len(str(unicode_safe_config))} chars")
+            return unicode_safe_config
+            
+        except Exception as e:
+            logger.error(f"Error applying Unicode-safe processing to config: {str(e)}")
+            return config
+
+    def replace_workflow_nodes_and_connections(
+        self, 
+        workflow_id: UUID, 
+        nodes_data: List[Dict[str, Any]], 
+        connections_data: List[Dict[str, Any]], 
+        organization_id: UUID
+    ) -> Dict[str, List]:
+        """Replace all workflow nodes and connections with cached data (delete existing, create new)"""
+        
+        # Validate workflow exists and belongs to organization
+        workflow = self.workflow_repo.get_by_id(workflow_id)
+        if not workflow:
+            raise ValueError("Workflow not found")
+        
+        if workflow.organization_id != organization_id:
+            raise ValueError("Workflow does not belong to your organization")
+        
+        try:
+            # Delete all existing connections first (due to foreign key constraints)
+            existing_connections = self.node_repo.get_connections_by_workflow(workflow_id)
+            for connection in existing_connections:
+                self.db.delete(connection)
+            
+            # Delete all existing nodes
+            existing_nodes = self.node_repo.get_nodes_by_workflow(workflow_id)
+            for node in existing_nodes:
+                self.db.delete(node)
+            
+            # Commit deletions
+            self.db.commit()
+            
+            logger.info(f"Deleted {len(existing_nodes)} existing nodes and {len(existing_connections)} existing connections for workflow {workflow_id}")
+            
+            # Create ID mapping for text IDs to UUIDs
+            id_mapping = {}
+            
+            def is_text_id(node_id: str) -> bool:
+                """Check if ID is a text ID (like message-1, llm-2) instead of UUID"""
+                try:
+                    UUID(node_id)
+                    return False
+                except ValueError:
+                    return True
+            
+            # First pass: Create ID mappings for text IDs
+            for node_data in nodes_data:
+                node_id = str(node_data.get('id', ''))
+                if node_id and is_text_id(node_id):
+                    if node_id not in id_mapping:
+                        id_mapping[node_id] = str(uuid.uuid4())
+                        logger.info(f"Mapped text ID '{node_id}' to UUID '{id_mapping[node_id]}'")
+            
+            # Also create mappings for connection IDs if they are text IDs
+            for conn_data in connections_data:
+                conn_id = str(conn_data.get('id', ''))
+                if conn_id and is_text_id(conn_id):
+                    if conn_id not in id_mapping:
+                        id_mapping[conn_id] = str(uuid.uuid4())
+                        logger.info(f"Mapped connection text ID '{conn_id}' to UUID '{id_mapping[conn_id]}'")
+            
+            # Create all new nodes
+            created_nodes = []
+            for node_data in nodes_data:
+                node_data_copy = node_data.copy()
+                node_data_copy['workflow_id'] = workflow_id
+                
+                # Convert workflow_id to UUID if it's a string
+                if isinstance(node_data_copy['workflow_id'], str):
+                    node_data_copy['workflow_id'] = UUID(node_data_copy['workflow_id'])
+                
+                # Extract and clean node-specific fields into config
+                config_fields = [
+                    'message_text', 'system_prompt', 'temperature', 'model_id',
+                    'condition_expression', 'action_type', 'action_url', 'action_config',
+                    'transfer_department', 'transfer_message', 'transfer_rules',
+                    'wait_duration', 'wait_unit', 'wait_until_condition',
+                    'final_message', 'form_fields', 'form_title', 'form_description', 
+                    'submit_button_text', 'form_full_screen',
+                    'landing_page_heading', 'landing_page_content',
+                    'enabled_guardrails', 'pii_action', 'jailbreak_sensitivity', 
+                    'text_source', 'block_message'
+                ]
+                
+                # Initialize config if not present
+                if 'config' not in node_data_copy:
+                    node_data_copy['config'] = {}
+                
+                # Move config fields from root to config object and filter out blank values
+                for field in config_fields:
+                    if field in node_data_copy:
+                        value = node_data_copy[field]
+                        # Only add non-blank values to config
+                        if value is not None and value != "" and value != []:
+                            node_data_copy['config'][field] = value
+                        # Remove from root level
+                        del node_data_copy[field]
+                
+                # Apply Unicode-safe processing to the entire config to handle multi-language content
+                node_data_copy['config'] = self._ensure_unicode_support_in_config(node_data_copy['config'])
+                
+                # Sanitize text fields to prevent UTF-8 encoding errors
+                base_text_fields = ['name', 'description']
+                for field in base_text_fields:
+                    if field in node_data_copy and isinstance(node_data_copy[field], str):
+                        node_data_copy[field] = sanitize_utf8_text(node_data_copy[field])
+                
+                # Sanitize config text fields
+                config_text_fields = ['message_text', 'system_prompt', 'condition_expression', 
+                                     'action_type', 'action_url', 'transfer_department', 'transfer_message',
+                                     'final_message', 'form_title', 'form_description', 'submit_button_text',
+                                     'landing_page_heading', 'landing_page_content', 'text_source', 'block_message']
+                for field in config_text_fields:
+                    if field in node_data_copy['config'] and isinstance(node_data_copy['config'][field], str):
+                        node_data_copy['config'][field] = sanitize_utf8_text(node_data_copy['config'][field])
+                
+                # Replace text ID with UUID if mapped
+                original_id = str(node_data_copy.get('id', ''))
+                if original_id in id_mapping:
+                    node_data_copy['id'] = id_mapping[original_id]
+                elif 'id' not in node_data_copy or not node_data_copy['id']:
+                    # Generate new ID if not provided
+                    node_data_copy['id'] = uuid.uuid4()
+                else:
+                    # Convert string ID to UUID
+                    node_data_copy['id'] = UUID(str(node_data_copy['id']))
+                
+                # Filter out any remaining None or empty values from base fields
+                filtered_data = {}
+                for key, value in node_data_copy.items():
+                    if value is not None and value != "":
+                        filtered_data[key] = value
+                
+                new_node = self.node_repo.create_node(**filtered_data)
+                created_nodes.append(new_node)
+            
+            # Commit node creations before processing connections
+            self.db.commit()
+            
+            # Create all new connections
+            created_connections = []
+            for conn_data in connections_data:
+                conn_data_copy = conn_data.copy()
+                conn_data_copy['workflow_id'] = workflow_id
+                
+                # Convert workflow_id to UUID if it's a string
+                if isinstance(conn_data_copy['workflow_id'], str):
+                    conn_data_copy['workflow_id'] = UUID(conn_data_copy['workflow_id'])
+                
+                # Sanitize text fields to prevent UTF-8 encoding errors
+                text_fields = ['label', 'condition']
+                for field in text_fields:
+                    if field in conn_data_copy and isinstance(conn_data_copy[field], str):
+                        conn_data_copy[field] = sanitize_utf8_text(conn_data_copy[field])
+                
+                # Replace text ID with UUID if mapped
+                original_id = str(conn_data_copy.get('id', ''))
+                if original_id in id_mapping:
+                    conn_data_copy['id'] = id_mapping[original_id]
+                elif 'id' not in conn_data_copy or not conn_data_copy['id']:
+                    # Generate new ID if not provided
+                    conn_data_copy['id'] = uuid.uuid4()
+                else:
+                    # Convert string ID to UUID
+                    conn_data_copy['id'] = UUID(str(conn_data_copy['id']))
+                
+                # Map source and target node IDs if they are text IDs
+                source_node_id = str(conn_data_copy.get('source_node_id', ''))
+                if source_node_id in id_mapping:
+                    conn_data_copy['source_node_id'] = id_mapping[source_node_id]
+                
+                target_node_id = str(conn_data_copy.get('target_node_id', ''))
+                if target_node_id in id_mapping:
+                    conn_data_copy['target_node_id'] = id_mapping[target_node_id]
+                
+                # Convert source and target node IDs to UUIDs
+                if 'source_node_id' in conn_data_copy:
+                    conn_data_copy['source_node_id'] = UUID(str(conn_data_copy['source_node_id']))
+                if 'target_node_id' in conn_data_copy:
+                    conn_data_copy['target_node_id'] = UUID(str(conn_data_copy['target_node_id']))
+                
+                new_conn = self.node_repo.create_connection(**conn_data_copy)
+                created_connections.append(new_conn)
+            
+            # Final commit for connections
+            self.db.commit()
+            
+            logger.info(f"Created {len(created_nodes)} new nodes and {len(created_connections)} new connections for workflow {workflow_id}")
+            
+            # Return all current nodes and connections
+            final_nodes = self.node_repo.get_nodes_by_workflow(workflow_id)
+            final_connections = self.node_repo.get_connections_by_workflow(workflow_id)
+            
+            return {
+                "nodes": final_nodes,
+                "connections": final_connections
+            }
+            
+        except Exception as e:
+            logger.error(f"Error replacing workflow nodes and connections: {str(e)}")
+            self.db.rollback()
+            raise
+
+    def get_workflow_nodes_and_connections(self, workflow_id: UUID, organization_id: UUID) -> Dict[str, List]:
+        """Get all nodes and connections for a workflow"""
+        
+        # Validate workflow exists and belongs to organization
+        workflow = self.workflow_repo.get_by_id(workflow_id)
+        if not workflow:
+            raise ValueError("Workflow not found")
+        
+        if workflow.organization_id != organization_id:
+            raise ValueError("Workflow does not belong to your organization")
+        
+        nodes = self.node_repo.get_nodes_by_workflow(workflow_id)
+        connections = self.node_repo.get_connections_by_workflow(workflow_id)
+        
+        return {
+            "nodes": nodes,
+            "connections": connections
+        }
+
+    def update_single_node(
+        self, 
+        workflow_id: UUID, 
+        node_id: UUID, 
+        node_data: Dict[str, Any], 
+        organization_id: UUID = None
+    ) -> Dict[str, Any]:
+        """Update a single workflow node with its properties"""
+        
+        # Validate workflow exists and belongs to organization
+        workflow = self.workflow_repo.get_by_id(workflow_id)
+        if not workflow:
+            raise ValueError("Workflow not found")
+        
+        if organization_id and workflow.organization_id != organization_id:
+            raise ValueError("Workflow does not belong to your organization")
+        
+        # Validate node exists and belongs to workflow
+        node = self.node_repo.get_node_by_id(node_id)
+        if not node:
+            raise ValueError("Node not found")
+        
+        if node.workflow_id != workflow_id:
+            raise ValueError("Node does not belong to this workflow")
+        
+        try:
+            logger.debug(f"Node data: {node_data}")
+            # Sanitize text fields to prevent UTF-8 encoding errors
+            node_data_copy = node_data.copy()
+            text_fields = ['name', 'description', 'message_text', 'system_prompt', 
+                          'condition_expression', 'action_type']
+            for field in text_fields:
+                if field in node_data_copy and isinstance(node_data_copy[field], str):
+                    node_data_copy[field] = sanitize_utf8_text(node_data_copy[field])
+            
+            # Validate and ensure unique node name
+            if 'name' in node_data_copy and node_data_copy['name']:
+                if not self._validate_node_name_uniqueness(workflow_id, node_data_copy['name'], node_id):
+                    # Generate unique name if not unique
+                    node_data_copy['name'] = self._generate_unique_node_name(workflow_id, node_data_copy['name'], node_id)
+                    logger.info(f"Generated unique node name: {node_data_copy['name']}")
+            
+            # Apply Unicode-safe processing to config if present
+            if 'config' in node_data_copy and isinstance(node_data_copy['config'], dict):
+                node_data_copy['config'] = self._ensure_unicode_support_in_config(node_data_copy['config'])
+            
+            logger.debug(f"Node data copy: {node_data_copy}")
+            # Update node properties
+            updated_node = self.node_repo.update_node(node_id, **node_data_copy)
+            if not updated_node:
+                raise ValueError("Failed to update node")
+            
+            # Commit changes
+            self.db.commit()
+            
+            logger.info(f"Updated node {node_id}")
+            
+            return updated_node
+            
+        except Exception as e:
+            logger.error(f"Error updating node {node_id}: {str(e)}")
+            self.db.rollback()
+            raise
+
+    def get_single_node(self, workflow_id: UUID, node_id: UUID, organization_id: UUID = None):
+        """Get a single node"""
+        
+        # Validate workflow exists and belongs to organization
+        workflow = self.workflow_repo.get_by_id(workflow_id)
+        if not workflow:
+            raise ValueError("Workflow not found")
+        
+        if organization_id and workflow.organization_id != organization_id:
+            raise ValueError("Workflow does not belong to your organization")
+        
+        # Validate node exists and belongs to workflow
+        node = self.node_repo.get_node_by_id(node_id)
+        if not node:
+            raise ValueError("Node not found")
+        
+        if node.workflow_id != workflow_id:
+            raise ValueError("Node does not belong to this workflow")
+        
+        return node