fix: improve maintenance and synchronize live conversations

backend/app/services/agent_rooms.py

@@ -14,13 +14,18 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
+import asyncio
+from dataclasses import dataclass
 from datetime import datetime, timezone
 from typing import Optional
 
 from sqlalchemy.orm import Session
 
-from app.core.auth import has_any_permission
+from app.core.auth import CHAT_VIEW_PERMISSIONS, has_any_permission
+from app.core.logger import get_logger
 from app.core.socketio import sio
+from app.repositories.chat import ChatRepository
+from app.repositories.session_to_agent import SessionToAgentRepository
 from app.repositories.user import UserRepository
 from app.services.chat_notifications import UNASSIGNED_CHAT_VIEWERS
 
@@ -28,6 +33,19 @@ from app.services.chat_notifications import UNASSIGNED_CHAT_VIEWERS
 # updates about any of the org's conversations — including AI-owned ones,
 # which have no handling agent to address.
 ORG_INBOX_ROOM_PREFIX = "org_inbox_"
+ORG_CHAT_UPDATES_PREFIX = "org_chat_updates_"
+CHAT_WATCH_PREFIX = "chat_watch_"
+logger = get_logger(__name__)
+_INBOX_COALESCE_SECONDS = 0.25
+
+
+@dataclass
+class _InboxUpdate:
+    dirty: bool = True
+    task: Optional[asyncio.Task] = None
+
+
+_pending_inbox_updates: dict[str, _InboxUpdate] = {}
 
 
 # chat_reply `type`s the inbox appends to a conversation
@@ -54,6 +72,85 @@ def may_join_org_inbox(db: Session, socket_session: dict, room: str) -> bool:
             and has_any_permission(user, UNASSIGNED_CHAT_VIEWERS))
 
 
+def may_join_chat_updates(db: Session, socket_session: dict, room: str) -> bool:
+    """The organization-wide room carries only a content-free change signal."""
+    organization_id = room.removeprefix(ORG_CHAT_UPDATES_PREFIX)
+    if str(socket_session.get('organization_id')) != organization_id:
+        return False
+    user_id = socket_session.get('user_id')
+    user = UserRepository(db).get_user(user_id) if user_id else None
+    return bool(user and user.is_active and str(user.organization_id) == organization_id
+                and has_any_permission(user, CHAT_VIEW_PERMISSIONS))
+
+
+async def may_watch_chat(db: Session, socket_session: dict, room: str) -> bool:
+    """Mirror the detail endpoint's organization and chat-scope checks."""
+    session_id = room.removeprefix(CHAT_WATCH_PREFIX)
+    user_id = socket_session.get('user_id')
+    user = UserRepository(db).get_user(user_id) if user_id else None
+    if not user or not user.is_active or not has_any_permission(user, CHAT_VIEW_PERMISSIONS):
+        return False
+    if str(user.organization_id) != str(socket_session.get('organization_id')):
+        return False
+    try:
+        chat = SessionToAgentRepository(db).get_session(session_id)
+    except (ValueError, TypeError):
+        return False
+    if not chat or str(chat.organization_id) != str(user.organization_id):
+        return False
+    if has_any_permission(user, ('view_all_chats',)):
+        return True
+    can_view_assigned = has_any_permission(user, ('view_assigned_chats',))
+    return await ChatRepository(db).check_session_access(
+        session_id=session_id,
+        user_id=user.id if can_view_assigned else None,
+        user_groups=[str(group.id) for group in user.groups] if can_view_assigned else [],
+        include_unassigned=has_any_permission(user, ('view_unassigned_chats',)),
+    )
+
+
+async def notify_conversation_changed(organization_id, session_id) -> None:
+    """Invalidate the selected thread and inbox after a committed change.
+
+    Never send message text or identifiers to the org-wide room. The thread
+    room also carries no content: every refresh goes through the normal HTTP
+    authorization, including after a chat is reassigned.
+    """
+    if not organization_id or not session_id:
+        return
+    await _emit_change('chat_changed', f'{CHAT_WATCH_PREFIX}{session_id}')
+    org_key = str(organization_id)
+    state = _pending_inbox_updates.get(org_key)
+    if state is not None:
+        state.dirty = True
+        return
+    state = _InboxUpdate()
+    _pending_inbox_updates[org_key] = state
+    # One task per active organization, not one task per message. Each server
+    # worker coalesces a burst before broadcasting it through the Redis manager.
+    state.task = asyncio.create_task(_flush_inbox_changes(org_key, state))
+
+
+async def _emit_change(event: str, room: str) -> None:
+    try:
+        await sio.emit(event, {}, room=room, namespace='/agent')
+    except Exception as exc:
+        # A broker outage must not turn an already-committed chat into an error.
+        logger.warning('Failed to notify inbox of a conversation change: %s', type(exc).__name__)
+
+
+async def _flush_inbox_changes(org_key: str, state: _InboxUpdate) -> None:
+    try:
+        while state.dirty:
+            await asyncio.sleep(_INBOX_COALESCE_SECONDS)
+            # Everything received before this point fits in this signal. An
+            # arrival during publish sets dirty again and gets a trailing one.
+            state.dirty = False
+            await _emit_change('inbox_changed', f'{ORG_CHAT_UPDATES_PREFIX}{org_key}')
+    finally:
+        _pending_inbox_updates.pop(org_key, None)
+
+
 async def emit_to_session_agents(event: str, payload: dict, session_id,
                                  user_id: Optional[object] = None) -> None:
     """Push a live update to the dashboard for one conversation.
@@ -92,4 +189,5 @@ async def relay_stored_message(session_record, stored, message_type: str) -> Non
         # Carries e.g. the button-reply marker, so tags show live too
         'attributes': stored.attributes or {},
     }
+    await notify_conversation_changed(session_record.organization_id, session_id)
     await emit_to_session_agents('chat_reply', payload, session_id, session_record.user_id)

backend/app/services/shopify.py

@@ -15,14 +15,12 @@ limitations under the License.
 """
 
 import requests
-from typing import Dict, Any, List, Optional
+from typing import Dict, Any, Optional
 from app.core.logger import get_logger
 from app.models.shopify import ShopifyShop
 from sqlalchemy.orm import Session
 from app.repositories.shopify_shop_repository import ShopifyShopRepository
 from app.repositories.agent_shopify_config_repository import AgentShopifyConfigRepository
-import json
-import re # Import regex module
 import aiohttp
 from app.services.shopify_urls import shopify_admin_url
 

backend/app/services/shopify_auth_service.py

@@ -15,20 +15,11 @@ limitations under the License.
 """
 
 from sqlalchemy.orm import Session
-from typing import Optional, Tuple, List, Dict, Any
+from typing import Optional, Tuple
 from app.core.logger import get_logger
 from app.repositories.agent_shopify_config_repository import AgentShopifyConfigRepository
 from app.repositories.widget import WidgetRepository
 from app.services.shopify import ShopifyService
-from app.repositories.agent import AgentRepository
-from fastapi import Request
-from fastapi.responses import RedirectResponse, HTMLResponse
-from app.models.organization import Organization
-from app.core.config import settings
-from urllib.parse import quote
-import hmac
-import hashlib
-import base64
 
 logger = get_logger(__name__)
 

backend/app/services/shopify_helper_service.py

@@ -17,12 +17,9 @@ limitations under the License.
 from sqlalchemy.orm import Session
 from typing import Optional
 from fastapi import Request
-from fastapi.responses import RedirectResponse, HTMLResponse
 from app.core.logger import get_logger
 from app.core.config import settings
-from app.models.organization import Organization
 from app.repositories.widget import WidgetRepository
-from app.services.shopify_auth_service import ShopifyAuthService
 from app.services.shopify_urls import shopify_oauth_token_url
 from urllib.parse import quote
 import hmac

backend/app/services/shopify_session.py

@@ -59,9 +59,6 @@ class ShopifySessionService:
         Raises HTTPException if invalid.
         """
         try:
-            # Decode without verification first to get the shop
-            unverified = jwt.decode(token, options={"verify_signature": False})
-            
             # Verify the token with Shopify's public key
             # Note: In production, you should fetch and cache Shopify's public keys
             # For now, we'll do basic validation

backend/app/services/ticket.py

@@ -831,18 +831,19 @@ class TicketService:
             }, message_id=stored.id, source=MessageSource.SYSTEM)
             if not delivery.ok:
                 chat_repo.mark_delivery_failed(stored.id, delivery.reason)
-            if ticket.first_response_at is None:
-                ticket.first_response_at = datetime.now(timezone.utc)
-            if record_activity:
-                self._add_activity(
-                    ticket, TicketActivityType.CUSTOMER_NOTIFIED,
-                    actor_type=TicketActorType.SYSTEM,
-                    body=message,
-                    metadata={
-                        "session_id": str(session_record.session_id),
-                        "channel": channel,
-                    },
-                )
+            else:
+                if ticket.first_response_at is None:
+                    ticket.first_response_at = datetime.now(timezone.utc)
+                if record_activity:
+                    self._add_activity(
+                        ticket, TicketActivityType.CUSTOMER_NOTIFIED,
+                        actor_type=TicketActorType.SYSTEM,
+                        body=message,
+                        metadata={
+                            "session_id": str(session_record.session_id),
+                            "channel": channel,
+                        },
+                    )
         except Exception as e:
             logger.error(f"Customer notification failed for {ticket.display_number}: {e}")
             # Stored but the hand-over itself raised: don't let the thread

backend/app/services/ticket_investigation.py

@@ -32,7 +32,6 @@ from app.models.investigation import (
     InvestigationEventType,
     InvestigationHypothesis,
     InvestigationRun,
-    InvestigationRunStatus,
     RCADocument,
 )
 from app.models.ticket import Ticket

backend/app/services/workflow_chat.py

@@ -14,7 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
-import asyncio
 from app.core.logger import get_logger
 from app.agents.chat_agent import ChatAgent, ChatResponse
 from app.repositories.chat import ChatRepository
@@ -64,6 +63,8 @@ class WorkflowChatService:
             ChatResponse: The response from workflow execution
         """
         
+        from app.services.agent_rooms import notify_conversation_changed
+
         # Store user message in chat history
         self.chat_repo.create_message({
             "message": message,
@@ -73,6 +74,7 @@ class WorkflowChatService:
             "agent_id": session['agent_id'],
             "customer_id": customer_id,
         })
+        await notify_conversation_changed(org_id, session_id)
         
         # Execute workflow
         workflow_result = await self.workflow_service.execute_workflow(
@@ -92,14 +94,16 @@ class WorkflowChatService:
         )
         
         if workflow_result.success:
-            return await self._handle_successful_workflow(
+            response = await self._handle_successful_workflow(
                 workflow_result, active_session, session_id, org_id, 
                 customer_id, session, sio, namespace
             )
         else:
-            return await self._handle_failed_workflow(
+            response = await self._handle_failed_workflow(
                 workflow_result, session_id, org_id, session, customer_id
             )
+        await notify_conversation_changed(org_id, session_id)
+        return response
     
     async def _handle_successful_workflow(
         self, 
@@ -174,6 +178,7 @@ class WorkflowChatService:
         namespace: str
     ):
         """Handle intermediate messages from MESSAGE nodes"""
+        from app.services.agent_rooms import notify_conversation_changed
         
         logger.info(f"Found {len(intermediate_messages)} intermediate messages from MESSAGE nodes")
         
@@ -195,6 +200,7 @@ class WorkflowChatService:
                     "message_node": True
                 }
             })
+            await notify_conversation_changed(org_id, session_id)
             
             # Emit intermediate message to client immediately
             await sio.emit('chat_response', {
@@ -390,4 +396,4 @@ class WorkflowChatService:
             }
         })
         
-        return response 
+        return response

backend/app/services/workflow_conditions.py

@@ -97,7 +97,7 @@ def compare(left: Any, operator: str, right: Any) -> bool:
 
     left_num, right_num = _as_number(left_text), _as_number(right_text)
     if left_num is None or right_num is None:
-        logger.debug(f"Non-numeric '{operator}' comparison is false: {left_text!r} vs {right_text!r}")
+        logger.debug(f"Non-numeric '{operator}' comparison is false")
         return False
     return _NUMERIC[operator](left_num, right_num)
 

backend/app/services/workflow_execution.py

@@ -14,24 +14,19 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
-import traceback
-import asyncio
-from typing import Optional, Dict, Any, List, Tuple
+from typing import Optional, Dict, Any, List
 from uuid import UUID
 from sqlalchemy.orm import Session
 from dataclasses import dataclass
 import json
 import re
-from datetime import datetime, timedelta
 
 from app.models.workflow import Workflow, WorkflowStatus
 from app.models.workflow_node import WorkflowNode, NodeType, ExitCondition
-from app.models.workflow_connection import WorkflowConnection
-from app.models.session_to_agent import SessionToAgent
 from app.repositories.workflow import WorkflowRepository
 from app.repositories.session_to_agent import SessionToAgentRepository
 from app.repositories.chat import ChatRepository
-from app.agents.chat_agent import ChatAgent, ChatResponse
+from app.agents.chat_agent import ChatAgent
 from app.core.logger import get_logger
 from app.services.workflow_conditions import (
     MISSING,
@@ -114,7 +109,6 @@ class WorkflowExecutionService:
         try:
             logger.info(f"Executing workflow {workflow_id} for session {session_id}")
             logger.debug(f"Current node ID: {current_node_id}")
-            logger.debug(f"Workflow state: {workflow_state}")
             logger.debug(f"User message: {user_message}")
             logger.debug(f"Is initial execution: {is_initial_execution}")
 
@@ -200,7 +194,7 @@ class WorkflowExecutionService:
                 # Collect messages from MESSAGE nodes during automatic execution
                 if current_node.node_type == NodeType.MESSAGE and result.message:
                     intermediate_messages.append(result.message)
-                    logger.debug(f"Collected intermediate message from MESSAGE node: {result.message}")
+                    logger.debug(f"Collected intermediate message from MESSAGE node {current_node.id}")
                 
                 # Also collect messages from LLM nodes that are not the final stopping node
                 # This handles the case where LLM node with single execution produces a response
@@ -208,21 +202,21 @@ class WorkflowExecutionService:
                 if (current_node.node_type == NodeType.LLM and result.message and 
                     result.should_continue and result.next_node_id is not None):
                     intermediate_messages.append(result.message)
-                    logger.debug(f"Collected intermediate message from LLM node: {result.message}")
+                    logger.debug(f"Collected intermediate message from LLM node {current_node.id}")
                 
                 # Also collect confirmation messages from USER_INPUT nodes that continue to next node
                 # This ensures confirmation messages are displayed to the user
                 if (current_node.node_type == NodeType.USER_INPUT and result.message and 
                     result.should_continue and result.next_node_id is not None):
                     intermediate_messages.append(result.message)
-                    logger.debug(f"Collected confirmation message from USER_INPUT node: {result.message}")
+                    logger.debug(f"Collected confirmation message from USER_INPUT node {current_node.id}")
                 
                 final_result = result
                 final_message_collected = len(intermediate_messages) > collected_before
 
                 # Check if we should continue to next node automatically
                 if not result.success:
-                    logger.error(f"Node execution failed: {result.error}")
+                    logger.error(f"Node execution failed for session {session_id} at node {current_node.id}")
                     break
                 
                 # Check stopping conditions
@@ -318,8 +312,7 @@ class WorkflowExecutionService:
             )
             
         except Exception as e:
-            traceback.print_exc()
-            logger.error(f"Error executing workflow: {str(e)}")
+            logger.error(f"Error executing workflow {workflow_id} for session {session_id}: {type(e).__name__}")
             return WorkflowExecutionResult(
                 success=False,
                 message="An error occurred while executing the workflow",
@@ -359,15 +352,14 @@ class WorkflowExecutionService:
         """
         try:
             logger.info(f"Submitting form for session {session_id}")
-            logger.debug(f"Form data: {form_data}")
             logger.debug(f"Workflow ID: {workflow_id}")
             logger.debug(f"Org ID: {org_id}")
             logger.debug(f"Agent ID: {agent_id}")
             logger.debug(f"Customer ID: {customer_id}")
-            logger.debug(f"API key: {api_key}")
             # Get current session state
             session = self.session_repo.get_session(session_id)
             if not session:
+                logger.warning(f"Form submission failed for session {session_id}: session not found")
                 return WorkflowExecutionResult(
                     success=False,
                     message="Session not found",
@@ -379,6 +371,7 @@ class WorkflowExecutionService:
             
             # Validate that we're in a form waiting state
             if workflow_state.get("form_state") != "waiting":
+                logger.warning(f"No form submission expected for session {session_id} at node {current_node_id}")
                 return WorkflowExecutionResult(
                     success=False,
                     message="No form submission expected",
@@ -389,9 +382,7 @@ class WorkflowExecutionService:
             workflow_state["form_data"] = form_data
             workflow_state["form_state"] = "submitted"
             
-            logger.debug(f"Form submission received for session {session_id}")
-            logger.debug(f"Form data: {form_data}")
-            logger.debug(f"Updated workflow state: {workflow_state}")
+            logger.debug(f"Form submission received for session {session_id} at node {current_node_id}")
             
             # Continue workflow execution with form submission
             return await self.execute_workflow(
@@ -411,7 +402,7 @@ class WorkflowExecutionService:
             )
             
         except Exception as e:
-            logger.error(f"Error submitting form: {str(e)}")
+            logger.error(f"Error submitting form for session {session_id}: {type(e).__name__}")
             return WorkflowExecutionResult(
                 success=False,
                 message="An error occurred while submitting the form",
@@ -500,7 +491,7 @@ class WorkflowExecutionService:
                 )
                 
         except Exception as e:
-            logger.error(f"Error executing node {node.id}: {str(e)}")
+            logger.error(f"Error executing node {node.id} for session {session_id}: {type(e).__name__}")
             return WorkflowExecutionResult(
                 success=False,
                 message=f"Error executing node: {str(e)}",
@@ -598,11 +589,9 @@ class WorkflowExecutionService:
             # Transfer to human setting only applies to continuous execution
             auto_transfer = False
             transfer_group_id = None
-            ask_for_rating_config = False
             if exit_condition == ExitCondition.CONTINUOUS_EXECUTION:
                 auto_transfer = config.get("auto_transfer_enabled", False)
                 transfer_group_id = config.get("transfer_group_id")
-                ask_for_rating_config = config.get("ask_for_rating", True)  # Default to True
             
             # Handle empty or null user message by creating structured context
             processed_user_message = user_message
@@ -636,7 +625,7 @@ class WorkflowExecutionService:
             finally:
                 await chat_agent.safe_cleanup_mcp_tools()
             
-            logger.debug(f"Response: {response}")
+            logger.debug(f"LLM node {node.id} returned a response")
             # Handle exit conditions
             next_node_id = None
             should_continue = False
@@ -693,7 +682,7 @@ class WorkflowExecutionService:
             )
             
         except Exception as e:
-            logger.error(f"Error executing LLM node: {str(e)}")
+            logger.error(f"Error executing LLM node {node.id}: {type(e).__name__}")
             return WorkflowExecutionResult(
                 success=False,
                 message="Sorry, I encountered an error processing your request.",
@@ -730,7 +719,7 @@ class WorkflowExecutionService:
             )
             
         except Exception as e:
-            logger.error(f"Error executing condition node: {str(e)}")
+            logger.error(f"Error executing condition node {node.id}: {type(e).__name__}")
             return WorkflowExecutionResult(
                 success=False,
                 message="Error evaluating condition",
@@ -770,7 +759,6 @@ class WorkflowExecutionService:
                 "fields": form_fields,
                 "form_full_screen": config.get("form_full_screen", False)
             }
-            logger.debug(f"Form data: {form_data}")
             # Mark that we're waiting for form submission
             workflow_state["form_state"] = "waiting"
             
@@ -781,12 +769,10 @@ class WorkflowExecutionService:
                 should_continue=False,  # Wait for form submission
                 form_data=form_data  # Include form data for display
             )
-            logger.debug(f"Form result: {result}")
             return result
         elif current_state == "submitted":
             # Form has been submitted, process and continue
             form_submission = workflow_state.get("form_data", {})
-            logger.debug(f"Form submission: {form_submission}")
             
             # Store form variables for use in other nodes with node-based naming
             node_based_variables = {}
@@ -874,7 +860,6 @@ class WorkflowExecutionService:
             should_continue=False,  # Wait for user to proceed
             landing_page_data=landing_page_data  # Include landing page data for display
         )
-        logger.debug(f"Landing page result: {result}")
         return result
     
     def _execute_action_node(self, node: WorkflowNode, workflow_state: Dict[str, Any]) -> WorkflowExecutionResult:
@@ -884,7 +869,6 @@ class WorkflowExecutionService:
         
         config = node.config or {}
         action_type = config.get("action_type")
-        action_config = config.get("action_config", {})
         
         logger.info(f"Executing action: {action_type}")
         
@@ -992,7 +976,7 @@ class WorkflowExecutionService:
                 text_to_check = user_message
             elif "variables" in workflow_state and "user_message" in workflow_state["variables"]:
                 text_to_check = workflow_state["variables"]["user_message"]
-                logger.debug(f"Using user_message from workflow state: {text_to_check}")
+                logger.debug(f"Using user message from workflow state for guardrails node {node.id}")
             else:
                 text_to_check = ""
         else:
@@ -1031,7 +1015,7 @@ class WorkflowExecutionService:
             "checked_text": text_to_check[:100] + "..." if len(text_to_check) > 100 else text_to_check
         }
         
-        logger.info(f"Guardrails node {node.id} - Passed: {passed}, Results: {results}")
+        logger.info(f"Guardrails node {node.id} - Passed: {passed}, checks: {len(results)}")
         
         # Find connections - guardrails nodes can have two outputs: "pass" and "fail"
         pass_node_id = None
@@ -1262,7 +1246,6 @@ class WorkflowExecutionService:
         """Update session with new workflow state"""
         try:
             logger.info(f"Updating session {session_id} with next_node_id: {next_node_id}")
-            logger.info(f"Updating session {session_id} with workflow_state: {workflow_state}")
             
             success = self.session_repo.update_workflow_state(session_id, next_node_id, workflow_state)
             if success:
@@ -1271,7 +1254,7 @@ class WorkflowExecutionService:
                 logger.error(f"Failed to update session {session_id} workflow state")
             
         except Exception as e:
-            logger.error(f"Error updating session workflow state: {str(e)}")
+            logger.error(f"Error updating workflow state for session {session_id}: {type(e).__name__}")
     
     async def _build_context_message(self, session_id: str, workflow_state: Dict[str, Any]) -> str:
         """
@@ -1383,11 +1366,10 @@ class WorkflowExecutionService:
             # Store variables namespaced by node ID
             workflow_state["variables"][str(node_id)] = form_data.copy()
             
-            logger.info(f"Stored form variables from node {node_id}: {list(form_data.keys())}")
-            logger.debug(f"Variable values: {form_data}")
+            logger.info(f"Stored {len(form_data)} form variables from node {node_id}")
             
         except Exception as e:
-            logger.error(f"Error storing form variables: {str(e)}")
+            logger.error(f"Error storing form variables from node {node_id}: {type(e).__name__}")
     
     def _interpolate_variables(self, text: str, workflow_state: Dict[str, Any]) -> str:
         """
@@ -1427,13 +1409,8 @@ class WorkflowExecutionService:
             # Replace all variable placeholders
             result = re.sub(pattern, replace_variable, text)
             
-            if result != text:
-                logger.debug(f"Variable interpolation: '{text}' -> '{result}'")
-            
             return result
             
         except Exception as e:
-            logger.error(f"Error interpolating variables: {str(e)}")
+            logger.error(f"Error interpolating workflow variables: {type(e).__name__}")
             return text  # Return original text on error
-
- 

backend/app/services/workflow_node.py

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
-from typing import List, Dict, Any, Optional
+from typing import List, Dict, Any
 from uuid import UUID
 import uuid
 import re