feat(knowledge): 任务队列、独立 worker 与 pgvector 向量索引

知识处理改为数据库任务队列,独立 worker 做切块与向量化,失败可重试。
This commit is contained in:
WangShen007 committed 2026-09-30 22:05:00 +08:00
1 parent 53825b0cfb
commit 738e37fbb0
36 files changed
+7590

No files matched your search

+101
View File
@@ -0,0 +1,101 @@
"""
Copyright 2024-2026 ChatterMate
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import asyncio
import sys
import os
from pathlib import Path
# Add the parent directory to Python path to allow imports
sys.path.append(str(Path(__file__).parent.parent.parent))
from app.database import SessionLocal
from app.repositories.session_to_agent import SessionToAgentRepository
from app.core.logger import get_logger
logger = get_logger(__name__)
# Configure check interval (default to 24 hours - once a day)
CHECK_INTERVAL = int(os.environ.get('CHAT_AUTO_CLOSE_INTERVAL', '86400')) # 24 hours in seconds
WORKER_ID = os.environ.get('GUNICORN_WORKER_ID', 'standalone')
async def run_auto_closer():
"""Single run of the auto closer"""
db = None
try:
logger.info(f"Worker {WORKER_ID}: Running chat auto-closer")
db = SessionLocal()
session_repo = SessionToAgentRepository(db)
# Auto-close inactive agent chats
closed_count = session_repo.auto_close_inactive_agent_chats()
if closed_count > 0:
logger.info(f"Worker {WORKER_ID}: Auto-closed {closed_count} inactive agent chats")
else:
logger.info(f"Worker {WORKER_ID}: No inactive agent chats found to close")
except Exception as e:
logger.error(f"Worker {WORKER_ID}: Error in chat auto-closer: {str(e)}")
raise
finally:
if db:
db.close()
async def run_auto_closer_loop():
"""Run the chat auto-closer in a continuous loop"""
logger.info(f"Worker {WORKER_ID}: Starting chat auto-closer with {CHECK_INTERVAL}s check interval (once per day)")
try:
iteration = 0
while True:
iteration += 1
start_time = asyncio.get_event_loop().time()
try:
logger.info(f"Worker {WORKER_ID}: Iteration {iteration} - Checking for inactive agent chats...")
await run_auto_closer()
logger.info(f"Worker {WORKER_ID}: Iteration {iteration} - Auto-closer completed successfully")
except Exception as e:
logger.error(f"Worker {WORKER_ID}: Iteration {iteration} - Error in auto-closer: {str(e)}")
# Calculate how long the processing took
elapsed = asyncio.get_event_loop().time() - start_time
logger.info(f"Worker {WORKER_ID}: Iteration {iteration} - Processing took {elapsed:.2f} seconds")
# Sleep for the configured interval (24 hours)
logger.info(f"Worker {WORKER_ID}: Iteration {iteration} - Sleeping for {CHECK_INTERVAL} seconds (24 hours) before next check")
await asyncio.sleep(CHECK_INTERVAL)
except Exception as e:
logger.error(f"Worker {WORKER_ID}: Fatal error in chat auto-closer: {str(e)}")
sys.exit(1)
def app(environ, start_response):
"""WSGI application for health checks"""
status = '200 OK'
headers = [('Content-type', 'text/plain')]
start_response(status, headers)
return [b'Chat Auto-Closer Worker is running']
# Main entry point for running as a standalone service
if __name__ == "__main__":
logger.info("Starting chat auto-closer service")
# Run the auto-closer loop
asyncio.run(run_auto_closer_loop())