270 lines
14 KiB
Python
270 lines
14 KiB
Python
|
"""Prepare without downtime; back up, switch, verify and recover independently of FastAPI."""
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
import shutil
|
||
|
|
import subprocess
|
||
|
|
import tarfile
|
||
|
|
import time
|
||
|
|
import urllib.request
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import Any
|
||
|
|
|
||
|
|
import repository
|
||
|
|
from state import BUSY, directory, now, progress, read, transaction
|
||
|
|
|
||
|
|
|
||
|
|
class Deployment:
|
||
|
|
def __init__(self, cfg: dict[str, Any]):
|
||
|
|
self.cfg = cfg
|
||
|
|
self.root = Path(cfg['project_dir'])
|
||
|
|
self.state_dir = directory(cfg)
|
||
|
|
self.log = self.state_dir / 'update.log'
|
||
|
|
|
||
|
|
def run(self, args: list[str], *, cwd: Path | None = None,
|
||
|
|
env: dict[str, str] | None = None, timeout: int = 1800,
|
||
|
|
stdin: Any = None, stdout: Any = None) -> None:
|
||
|
|
with self.log.open('ab') as log:
|
||
|
|
result = subprocess.run(args, cwd=cwd, env={**os.environ, **(env or {})},
|
||
|
|
stdin=stdin, stdout=stdout or log, stderr=log, timeout=timeout)
|
||
|
|
if result.returncode:
|
||
|
|
raise RuntimeError(f'{args[0]} 执行失败（退出码 {result.returncode}），请查看服务器更新日志。')
|
||
|
|
|
||
|
|
def services(self, action: str) -> None:
|
||
|
|
self.run(['systemctl', action, 'replypier-backend.service', 'replypier-knowledge.service'], timeout=90)
|
||
|
|
|
||
|
|
def healthy(self) -> None:
|
||
|
|
for _ in range(45):
|
||
|
|
try:
|
||
|
|
with urllib.request.urlopen('http://127.0.0.1:8018/health', timeout=3) as response:
|
||
|
|
if json.load(response).get('status') == 'healthy':
|
||
|
|
return
|
||
|
|
except (OSError, ValueError):
|
||
|
|
pass
|
||
|
|
time.sleep(2)
|
||
|
|
raise RuntimeError('新版本后端未通过健康检查。')
|
||
|
|
|
||
|
|
def prepare(self, target: str) -> Path:
|
||
|
|
stage = self.state_dir / 'releases' / f'{target}-{int(time.time())}'
|
||
|
|
stage.mkdir(parents=True)
|
||
|
|
archive = stage / 'source.tar'
|
||
|
|
self.run(['git', '--git-dir', str(self.state_dir / 'repository.git'),
|
||
|
|
'archive', '--format=tar', target, '-o', str(archive)])
|
||
|
|
source = stage / 'source'
|
||
|
|
source.mkdir()
|
||
|
|
with tarfile.open(archive) as tar:
|
||
|
|
tar.extractall(source, filter='data')
|
||
|
|
archive.unlink()
|
||
|
|
self.new_paths = repository.paths(self.cfg, target)
|
||
|
|
if any(repository.protected(p) for p in self.new_paths):
|
||
|
|
raise RuntimeError('仓库包含受保护的密钥或运行目录，已停止更新。')
|
||
|
|
front = source / 'frontend'
|
||
|
|
shutil.copy2(self.root / 'frontend/.env', front / '.env')
|
||
|
|
progress(self.cfg, 'building', '正在安装前端依赖并构建，网站仍正常接待。')
|
||
|
|
self.run(['npm', 'ci', '--no-audit', '--no-fund'], cwd=front)
|
||
|
|
node_env = {'NODE_OPTIONS': '--max-old-space-size=1024'}
|
||
|
|
vite = str(front / 'node_modules/.bin/vite')
|
||
|
|
self.run([vite, 'build', '--outDir', str(stage / 'dashboard')], cwd=front, env=node_env)
|
||
|
|
self.run([vite, 'build', '--config', 'vite.widget.config.ts', '--outDir', str(stage / 'assets')],
|
||
|
|
cwd=front, env=node_env)
|
||
|
|
if not (stage / 'assets/widget.css').exists():
|
||
|
|
styles = list((stage / 'assets/unused').glob('*.css'))
|
||
|
|
if len(styles) != 1:
|
||
|
|
raise RuntimeError('网站组件的样式构建结果不完整。')
|
||
|
|
shutil.copy2(styles[0], stage / 'assets/widget.css')
|
||
|
|
loader = stage / 'dashboard/webclient/replypier.min.js'
|
||
|
|
loader.parent.mkdir(exist_ok=True)
|
||
|
|
self.run([str(front / 'node_modules/.bin/esbuild'), 'src/webclient/replypier.js',
|
||
|
|
'--bundle', '--minify', '--format=iife', '--target=es2015', '--loader:.svg=text',
|
||
|
|
'--define:process.env.NODE_ENV="production"',
|
||
|
|
'--define:__REPLYPIER_API_URL__=' + json.dumps(self.cfg['public_url'] + '/api/v1'),
|
||
|
|
'--outfile=' + str(loader)], cwd=front)
|
||
|
|
# Runtime configuration belongs to this installation, not to the repository.
|
||
|
|
live_web = self.root / 'deploy/local/.state/dashboard'
|
||
|
|
shutil.copy2(live_web / 'config.js', stage / 'dashboard/config.js')
|
||
|
|
if (live_web / 'local-demo.html').exists():
|
||
|
|
shutil.copy2(live_web / 'local-demo.html', stage / 'dashboard/local-demo.html')
|
||
|
|
shutil.rmtree(front / 'node_modules')
|
||
|
|
lock = Path('deploy/local/requirements.lock.txt')
|
||
|
|
self.new_venv = None
|
||
|
|
if (source / lock).read_bytes() != (self.root / lock).read_bytes():
|
||
|
|
progress(self.cfg, 'preparing', '后端依赖已变更，正在创建独立环境，网站仍正常接待。')
|
||
|
|
self.new_venv = stage / 'venv'
|
||
|
|
self.run(['/usr/bin/python3', '-m', 'venv', str(self.new_venv)])
|
||
|
|
self.run([str(self.new_venv / 'bin/python'), '-m', 'pip', 'install', '-r', str(source / lock)], timeout=3600)
|
||
|
|
return stage
|
||
|
|
|
||
|
|
def backup(self, stage: Path, current: str) -> Path:
|
||
|
|
backup = self.state_dir / 'backups' / stage.name
|
||
|
|
backup.mkdir(parents=True)
|
||
|
|
(backup / 'agent').mkdir()
|
||
|
|
(backup / 'systemd').mkdir()
|
||
|
|
for file in Path('/opt/replypier-updater').glob('*.py'):
|
||
|
|
shutil.copy2(file, backup / 'agent' / file.name)
|
||
|
|
for file in Path('/etc/systemd/system').glob('replypier-update*'):
|
||
|
|
if file.is_file():
|
||
|
|
shutil.copy2(file, backup / 'systemd' / file.name)
|
||
|
|
old_paths = repository.paths(self.cfg, current)
|
||
|
|
with tarfile.open(backup / 'code.tar.gz', 'w:gz') as tar:
|
||
|
|
for path in old_paths:
|
||
|
|
file = self.root / path
|
||
|
|
if file.exists() and not repository.protected(path):
|
||
|
|
tar.add(file, arcname=path, recursive=False)
|
||
|
|
(backup / 'manifest.json').write_text(json.dumps({'old': old_paths, 'new': self.new_paths}))
|
||
|
|
venv = self.root / 'backend/.venv'
|
||
|
|
if venv.is_symlink():
|
||
|
|
(backup / 'venv-link').write_text(os.readlink(venv))
|
||
|
|
with transaction(self.cfg) as state:
|
||
|
|
state['update']['backup_path'] = str(backup)
|
||
|
|
state['update']['previous_revision'] = current
|
||
|
|
state['update']['backup_ready'] = False
|
||
|
|
progress(self.cfg, 'backing_up', '正在暂停接待并备份数据库；页面会自动恢复。')
|
||
|
|
(self.state_dir / 'maintenance').touch()
|
||
|
|
self.services('stop')
|
||
|
|
with (backup / 'database.dump').open('wb') as out:
|
||
|
|
self.run(['docker', 'exec', self.cfg['db_container'], 'pg_dump', '-Fc',
|
||
|
|
'-U', self.cfg['db_user'], '-d', self.cfg['db_name']], stdout=out)
|
||
|
|
with (backup / 'database.dump').open('rb') as dump:
|
||
|
|
self.run(['docker', 'exec', '-i', self.cfg['db_container'], 'pg_restore', '--list'], stdin=dump)
|
||
|
|
with transaction(self.cfg) as state:
|
||
|
|
state['update']['backup_ready'] = True
|
||
|
|
return backup
|
||
|
|
|
||
|
|
def switch(self, stage: Path, backup: Path) -> None:
|
||
|
|
progress(self.cfg, 'deploying', '正在安装新版本源码。')
|
||
|
|
manifest = json.loads((backup / 'manifest.json').read_text())
|
||
|
|
for path in manifest['new']:
|
||
|
|
target = self.root / path
|
||
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||
|
|
shutil.copy2(stage / 'source' / path, target, follow_symlinks=False)
|
||
|
|
for path in set(manifest['old']) - set(manifest['new']):
|
||
|
|
if not repository.protected(path):
|
||
|
|
(self.root / path).unlink(missing_ok=True)
|
||
|
|
if self.new_venv:
|
||
|
|
venv = self.root / 'backend/.venv'
|
||
|
|
if venv.is_symlink():
|
||
|
|
venv.unlink()
|
||
|
|
else:
|
||
|
|
venv.rename(backup / 'original-venv')
|
||
|
|
venv.symlink_to(self.new_venv, target_is_directory=True)
|
||
|
|
progress(self.cfg, 'migrating', '正在应用数据库迁移。')
|
||
|
|
self.run([str(self.root / 'backend/.venv/bin/python'), '-m', 'alembic', 'upgrade', 'head'],
|
||
|
|
cwd=self.root / 'backend', env={'ENV_FILE': str(self.root / 'backend/.env'),
|
||
|
|
'NO_ENTERPRISE': '1', 'PYTHONPATH': str(self.root / 'backend')})
|
||
|
|
progress(self.cfg, 'verifying', '正在启动服务并检查新版本。')
|
||
|
|
self.run(['systemctl', 'start', 'replypier-backend.service'], timeout=90)
|
||
|
|
self.healthy()
|
||
|
|
state = self.root / 'deploy/local/.state'
|
||
|
|
for name in ('assets', 'dashboard'):
|
||
|
|
(state / name).rename(backup / name)
|
||
|
|
(stage / name).rename(state / name)
|
||
|
|
|
||
|
|
def restore(self, backup: Path, *, boot: bool = False) -> None:
|
||
|
|
if not boot:
|
||
|
|
self.services('stop')
|
||
|
|
manifest = json.loads((backup / 'manifest.json').read_text())
|
||
|
|
for path in set(manifest['new']) - set(manifest['old']):
|
||
|
|
if not repository.protected(path):
|
||
|
|
(self.root / path).unlink(missing_ok=True)
|
||
|
|
with tarfile.open(backup / 'code.tar.gz') as tar:
|
||
|
|
tar.extractall(self.root, filter='data')
|
||
|
|
venv = self.root / 'backend/.venv'
|
||
|
|
if (backup / 'original-venv').exists():
|
||
|
|
if venv.is_symlink():
|
||
|
|
venv.unlink()
|
||
|
|
(backup / 'original-venv').rename(venv)
|
||
|
|
elif (backup / 'venv-link').exists():
|
||
|
|
if venv.is_symlink():
|
||
|
|
venv.unlink()
|
||
|
|
venv.symlink_to((backup / 'venv-link').read_text(), target_is_directory=True)
|
||
|
|
state = self.root / 'deploy/local/.state'
|
||
|
|
for name in ('assets', 'dashboard'):
|
||
|
|
if (backup / name).exists():
|
||
|
|
if (state / name).exists():
|
||
|
|
(state / name).rename(backup / f'failed-{name}')
|
||
|
|
(backup / name).rename(state / name)
|
||
|
|
for _ in range(30):
|
||
|
|
ready = subprocess.run(['docker', 'exec', self.cfg['db_container'], 'pg_isready',
|
||
|
|
'-U', self.cfg['db_user'], '-d', 'postgres'], capture_output=True)
|
||
|
|
if ready.returncode == 0:
|
||
|
|
break
|
||
|
|
time.sleep(2)
|
||
|
|
else:
|
||
|
|
raise RuntimeError('数据库尚未就绪，备份和维护标记已保留。')
|
||
|
|
with (backup / 'database.dump').open('rb') as dump:
|
||
|
|
self.run(['docker', 'exec', '-i', self.cfg['db_container'], 'pg_restore',
|
||
|
|
'--clean', '--if-exists', '--create', '--exit-on-error', '--no-owner', '--no-privileges',
|
||
|
|
'-U', self.cfg['db_user'], '-d', 'postgres'], stdin=dump)
|
||
|
|
for source in (backup / 'agent').glob('*.py'):
|
||
|
|
shutil.copy2(source, Path('/opt/replypier-updater') / source.name)
|
||
|
|
for source in (backup / 'systemd').glob('*'):
|
||
|
|
shutil.copy2(source, Path('/etc/systemd/system') / source.name)
|
||
|
|
self.run(['systemctl', 'daemon-reload'], timeout=30)
|
||
|
|
if not boot:
|
||
|
|
self.services('start')
|
||
|
|
self.healthy()
|
||
|
|
|
||
|
|
def recover(self, *, boot: bool = False) -> None:
|
||
|
|
if not (self.state_dir / 'maintenance').exists():
|
||
|
|
if read(self.cfg)['update']['status'] in BUSY:
|
||
|
|
progress(self.cfg, 'failed', '更新进程已中断，在线版本未改变。', finished_at=now())
|
||
|
|
return
|
||
|
|
state = read(self.cfg)
|
||
|
|
update = state['update']
|
||
|
|
if update['status'] == 'succeeded':
|
||
|
|
if not boot:
|
||
|
|
self.services('start')
|
||
|
|
self.healthy()
|
||
|
|
(self.state_dir / 'maintenance').unlink(missing_ok=True)
|
||
|
|
return
|
||
|
|
progress(self.cfg, 'rolling_back', '正在恢复更新前的版本。')
|
||
|
|
if update.get('backup_ready'):
|
||
|
|
self.restore(Path(update['backup_path']), boot=boot)
|
||
|
|
with transaction(self.cfg) as state:
|
||
|
|
state['current'] = repository.commit(self.cfg, update['previous_revision'])
|
||
|
|
elif not boot:
|
||
|
|
self.services('start')
|
||
|
|
self.healthy()
|
||
|
|
(self.state_dir / 'maintenance').unlink(missing_ok=True)
|
||
|
|
progress(self.cfg, 'rolled_back', '更新未完成，已恢复原版本。', finished_at=now())
|
||
|
|
|
||
|
|
def apply(self, target: str) -> None:
|
||
|
|
backup = None
|
||
|
|
try:
|
||
|
|
if shutil.disk_usage(self.root).free < 4 * 1024**3:
|
||
|
|
raise RuntimeError('可用磁盘空间不足 4 GiB，请先清理磁盘。')
|
||
|
|
progress(self.cfg, 'preparing', '正在获取已确认的版本。', started_at=now(), error=None)
|
||
|
|
repository.git(self.cfg, 'cat-file', '-e', target + '^{commit}')
|
||
|
|
current = read(self.cfg)['current']['revision']
|
||
|
|
repository.ensure_forward(self.cfg, current, target)
|
||
|
|
repository.ensure_clean(self.cfg, current, self.root)
|
||
|
|
stage = self.prepare(target)
|
||
|
|
backup = self.backup(stage, current)
|
||
|
|
self.switch(stage, backup)
|
||
|
|
# Keep the independent agent recoverable if copying its next version fails.
|
||
|
|
agent = stage / 'source/deploy/server/updater'
|
||
|
|
for source in agent.glob('*.py'):
|
||
|
|
temp = Path('/opt/replypier-updater') / (source.name + '.next')
|
||
|
|
shutil.copy2(source, temp)
|
||
|
|
temp.replace(temp.with_suffix(''))
|
||
|
|
for source in (stage / 'source/deploy/server/systemd').glob('replypier-update*'):
|
||
|
|
shutil.copy2(source, Path('/etc/systemd/system') / source.name)
|
||
|
|
self.run(['systemctl', 'daemon-reload'], timeout=30)
|
||
|
|
self.run(['systemctl', 'start', 'replypier-knowledge.service'], timeout=90)
|
||
|
|
time.sleep(3)
|
||
|
|
self.run(['systemctl', 'is-active', '--quiet', 'replypier-knowledge.service'], timeout=10)
|
||
|
|
with transaction(self.cfg) as state:
|
||
|
|
state['current'] = repository.commit(self.cfg, target)
|
||
|
|
state['update'].update(status='succeeded', message='更新完成，服务已恢复。', finished_at=now())
|
||
|
|
(self.state_dir / 'maintenance').unlink(missing_ok=True)
|
||
|
|
except Exception as exc:
|
||
|
|
error = str(exc)
|
||
|
|
try:
|
||
|
|
self.recover()
|
||
|
|
status = 'rolled_back' if backup else 'failed'
|
||
|
|
progress(self.cfg, status, '更新失败，原版本仍可使用。', error=error, finished_at=now())
|
||
|
|
except Exception as recovery_error:
|
||
|
|
progress(self.cfg, 'recovery_failed', '自动恢复失败，请管理员执行服务器恢复命令。',
|
||
|
|
error=f'{error}；恢复错误：{recovery_error}', finished_at=now())
|
||
|
|
raise
|