feat(widget): Widget 配置、loader 与右下角悬浮入口

生成可嵌入站点的 loader 与悬浮按钮，支持展开收起和移动端；附独立宿主测试页。

backend/app/core/crawler_policy.py

@@ -0,0 +1,88 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Keep crawlers out of the API origin.
+
+The API host had no robots.txt at all, so search engines were free to crawl
+(and then report as 404s) endpoints that were never pages: /api/v1/... routes
+and the YOUR_WIDGET_ID placeholder our docs use in examples.
+
+Two halves, because they do different jobs. robots.txt stops the fetch;
+X-Robots-Tag stops a URL discovered elsewhere from being indexed anyway.
+"""
+
+from fastapi import FastAPI
+from fastapi.responses import PlainTextResponse
+
+from app.core.config import settings
+
+NOINDEX_HEADER = (b"x-robots-tag", b"noindex")
+
+
+def api_robots_txt() -> str:
+    """The API origin's robots.txt body.
+
+    Disallows everything — except, under path dispatch, the help center. That
+    mode (the self-host default) serves public help-center pages from
+    {BACKEND_URL}/help/{slug} on this very origin, and those are meant to be
+    indexed; a bare "Disallow: /" would quietly de-index every self-hosted
+    help center. Cloud runs subdomain mode, where nothing on this origin is a
+    page and the carve-out is absent.
+
+    The Allow goes FIRST. RFC 9309 crawlers take the most specific match and
+    would get this right either way, but ones that take the first match read
+    a leading "Disallow: /" and stop — skipping the very pages the carve-out
+    exists to keep crawlable.
+    """
+    lines = ["User-agent: *"]
+    if settings.HELP_CENTER_PUBLIC_MODE == "path":
+        lines.append("Allow: /help/")
+    lines.append("Disallow: /")
+    return "\n".join(lines) + "\n"
+
+
+class NoIndexHeaderMiddleware:
+    """Stamp ``X-Robots-Tag: noindex`` on every response under ``prefix``.
+
+    Pure ASGI rather than BaseHTTPMiddleware: this fronts every API request,
+    and BaseHTTPMiddleware would pull streaming responses through a queue for
+    the sake of one header.
+    """
+
+    def __init__(self, app, prefix: str):
+        self.app = app
+        self.prefix = prefix
+
+    async def __call__(self, scope, receive, send):
+        if scope["type"] != "http" or not scope.get("path", "").startswith(self.prefix):
+            await self.app(scope, receive, send)
+            return
+
+        async def send_with_noindex(message):
+            if message["type"] == "http.response.start":
+                message["headers"] = [*(message.get("headers") or []), NOINDEX_HEADER]
+            await send(message)
+
+        await self.app(scope, receive, send_with_noindex)
+
+
+def install_crawler_policy(app: FastAPI) -> None:
+    """Register the robots.txt route and the noindex header middleware."""
+
+    @app.get("/robots.txt", response_class=PlainTextResponse, include_in_schema=False)
+    async def robots_txt():
+        return PlainTextResponse(api_robots_txt())
+
+    app.add_middleware(NoIndexHeaderMiddleware, prefix=settings.API_V1_STR)