feat(wordcloud): 收口在途开发（布局/存储/前端）+ R4 WCD 生产任务(jobs wcd_file)与生产订单列表
This commit is contained in:
+483
-11
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
@@ -10,6 +11,7 @@ import sqlite3
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
@@ -24,6 +26,7 @@ from core.fonts import get_cached_font
|
||||
from .job_manager import JobManager
|
||||
from .line_spacing import analyze_svg_line_spacing_file
|
||||
from .log_config import get_logger
|
||||
from .metadata_store import MetadataStore
|
||||
from .runner import JobRunner
|
||||
from .schemas import (
|
||||
Asset,
|
||||
@@ -57,12 +60,17 @@ ASSETS_DIR = PROJECT_ROOT / "service_assets"
|
||||
PROJECTS_DIR = PROJECT_ROOT / "service_projects"
|
||||
FONTS_DIR = PROJECT_ROOT / "service_fonts"
|
||||
DESIGN_TEMPLATES_DIR = PROJECT_ROOT / "service_design_templates"
|
||||
METADATA_DIR = PROJECT_ROOT / "service_metadata"
|
||||
ORDERS_DIR = PROJECT_ROOT / "service_orders"
|
||||
ASSETS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
PROJECTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
FONTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
DESIGN_TEMPLATES_DIR.mkdir(parents=True, exist_ok=True)
|
||||
METADATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
ORDERS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
manager = JobManager()
|
||||
metadata_store = MetadataStore(METADATA_DIR / "app.db")
|
||||
manager = JobManager(metadata_store)
|
||||
storage = Storage(WORKSPACE_DIR)
|
||||
runner = JobRunner(PROJECT_ROOT, manager)
|
||||
|
||||
@@ -144,6 +152,31 @@ def health() -> dict:
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@app.get("/api/maintenance/storage-summary")
|
||||
def storage_summary() -> dict:
|
||||
referenced_jobs: set[str] = set()
|
||||
for d in _list_dirs(ASSETS_DIR):
|
||||
meta = _read_asset_meta(d)
|
||||
job_id = meta.get("job_id") or ""
|
||||
if job_id:
|
||||
referenced_jobs.add(job_id)
|
||||
stale = storage.stale_job_dirs(
|
||||
referenced_job_ids=referenced_jobs,
|
||||
exclude_job_ids=metadata_store.job_ids(),
|
||||
max_age_days=0,
|
||||
)
|
||||
return {
|
||||
"job_dir_count": len(storage.list_job_ids()),
|
||||
"referenced_job_ids": len(referenced_jobs),
|
||||
"stale_job_count": len(stale),
|
||||
"reclaimable_bytes": sum(item["size_bytes"] for item in stale),
|
||||
"metadata_db_bytes": metadata_store.summarize()["db_size_bytes"],
|
||||
"jobs_in_db": metadata_store.summarize()["jobs"],
|
||||
"events_in_db": metadata_store.summarize()["events"],
|
||||
"dry_run_only": True,
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/jobs", response_model=list[JobStatus])
|
||||
def list_jobs() -> list[JobStatus]:
|
||||
with manager._lock:
|
||||
@@ -153,26 +186,21 @@ def list_jobs() -> list[JobStatus]:
|
||||
@app.post("/api/jobs", response_model=JobCreateResponse)
|
||||
async def create_job(
|
||||
mask_image: Optional[UploadFile] = File(None),
|
||||
name_list: UploadFile = File(...),
|
||||
name_list: Optional[UploadFile] = File(None),
|
||||
wcd_file: Optional[UploadFile] = File(None),
|
||||
font_file: Optional[UploadFile] = File(None),
|
||||
font_id: str = Form(""),
|
||||
params: str = Form("{}"),
|
||||
) -> JobCreateResponse:
|
||||
log.info("─" * 50)
|
||||
log.info("[API] POST /api/jobs 收到新任务请求")
|
||||
log.info(" name_list.filename = %s", name_list.filename)
|
||||
log.info(" name_list.filename = %s", name_list.filename if name_list else "无")
|
||||
log.info(" wcd_file.filename = %s", wcd_file.filename if wcd_file else "无")
|
||||
log.info(" mask_image.filename = %s", mask_image.filename if mask_image else "无")
|
||||
log.info(" font_file.filename = %s", font_file.filename if font_file else "无")
|
||||
log.info(" font_id = %s", font_id or "(未指定)")
|
||||
log.info(" params (raw) = %s", params)
|
||||
|
||||
if not name_list.filename:
|
||||
raise HTTPException(status_code=400, detail="name_list is required")
|
||||
|
||||
ext_xlsx = Path(name_list.filename).suffix.lower()
|
||||
if ext_xlsx not in {".xlsx"}:
|
||||
raise HTTPException(status_code=400, detail="name_list must be xlsx")
|
||||
|
||||
try:
|
||||
user_params = json.loads(params)
|
||||
except json.JSONDecodeError:
|
||||
@@ -181,6 +209,23 @@ async def create_job(
|
||||
if not isinstance(user_params, dict):
|
||||
raise HTTPException(status_code=400, detail="params must be JSON object")
|
||||
|
||||
mode = str(user_params.get("MODE", "IMAGE")).upper()
|
||||
|
||||
# WCD 生产任务：传入 .wcd 画布导入导出包（还原设计 -> 生产，见 docs/wordcloud-contract.md v1.1）
|
||||
is_wcd = mode == "WCD" or bool(wcd_file and wcd_file.filename)
|
||||
if is_wcd:
|
||||
if not (wcd_file and wcd_file.filename):
|
||||
raise HTTPException(status_code=400, detail="wcd_file is required when MODE=WCD")
|
||||
return await _create_wcd_job(wcd_file, user_params)
|
||||
|
||||
# 名单/xlsx 模式（既有逻辑；必填校验移到 mode/WCD 判断之后）
|
||||
if not name_list or not name_list.filename:
|
||||
raise HTTPException(status_code=400, detail="name_list is required")
|
||||
|
||||
ext_xlsx = Path(name_list.filename).suffix.lower()
|
||||
if ext_xlsx not in {".xlsx"}:
|
||||
raise HTTPException(status_code=400, detail="name_list must be xlsx")
|
||||
|
||||
log.info(" 解析后 params = %s", json.dumps(user_params, ensure_ascii=False))
|
||||
|
||||
mode = str(user_params.get("MODE", "IMAGE")).upper()
|
||||
@@ -234,8 +279,12 @@ async def create_job(
|
||||
"MODE": mode,
|
||||
"EXCEL_PATH": str(paths.excel_path),
|
||||
"OUTPUT_DIR": str(paths.output_dir),
|
||||
"SAVE_DEBUG_IMAGES": True,
|
||||
# Debug masks are useful during local diagnosis but add extra image
|
||||
# writes to every request. Keep the fast service path disk-light; the
|
||||
# explicit CLI/config option remains available when debugging.
|
||||
"SAVE_DEBUG_IMAGES": False,
|
||||
"DEBUG_OUTPUT_DIR": str(paths.output_dir / "debug"),
|
||||
"FAST_MODE": True,
|
||||
}
|
||||
if mask_image and mask_image.filename:
|
||||
config["MASK_IMAGE_PATH"] = str(paths.mask_path)
|
||||
@@ -273,6 +322,174 @@ async def create_job(
|
||||
return JobCreateResponse(job_id=job_id)
|
||||
|
||||
|
||||
def _parse_hex(color: object):
|
||||
"""把 '#RRGGBB' / '#RGB' / 空值 解析为 RGBA tuple。"""
|
||||
raw = str(color or "#ffffff").strip().lstrip("#")
|
||||
if len(raw) == 3:
|
||||
raw = "".join(c + c for c in raw)
|
||||
try:
|
||||
return tuple(int(raw[i : i + 2], 16) for i in (0, 2, 4)) + (255,)
|
||||
except ValueError:
|
||||
return (255, 255, 255, 255)
|
||||
|
||||
|
||||
def _compose_design_png(document: dict, file_map: dict, output_path: Path) -> None:
|
||||
"""把 CanvasDocument 合成一张扁平 PNG（生产任务产物）：背景 + 按 zIndex 叠贴纸。
|
||||
|
||||
file_map: {pkg_asset_id: {"path": str}}，即 WCD 内临时 assetId -> 落盘素材文件。
|
||||
"""
|
||||
try:
|
||||
width = int(document.get("width") or 1200)
|
||||
height = int(document.get("height") or 1200)
|
||||
except (TypeError, ValueError):
|
||||
width, height = 1200, 1200
|
||||
canvas = Image.new("RGBA", (max(width, 1), max(height, 1)), _parse_hex(document.get("background")))
|
||||
elements = [
|
||||
e
|
||||
for e in document.get("elements", [])
|
||||
if isinstance(e, dict) and e.get("type") == "sticker"
|
||||
]
|
||||
elements.sort(key=lambda e: e.get("zIndex", 0))
|
||||
for e in elements:
|
||||
info = file_map.get(str(e.get("assetId") or ""))
|
||||
if not info:
|
||||
continue
|
||||
path = info.get("path")
|
||||
if not path or not Path(path).exists():
|
||||
continue
|
||||
try:
|
||||
img = Image.open(path).convert("RGBA")
|
||||
except Exception:
|
||||
continue
|
||||
ew = e.get("width")
|
||||
eh = e.get("height")
|
||||
ew = int(ew) if isinstance(ew, (int, float)) and ew > 0 else img.width
|
||||
eh = int(eh) if isinstance(eh, (int, float)) and eh > 0 else img.height
|
||||
if (ew, eh) != (img.width, img.height):
|
||||
img = img.resize((int(ew), int(eh)))
|
||||
opacity = e.get("opacity", 1)
|
||||
if isinstance(opacity, (int, float)) and opacity >= 0 and opacity != 1:
|
||||
img = img.copy()
|
||||
img.putalpha(img.getchannel("A").point(lambda v: round(v * float(opacity))))
|
||||
canvas.alpha_composite(img, (int(e.get("x") or 0), int(e.get("y") or 0)))
|
||||
canvas.convert("RGB").save(output_path, "PNG")
|
||||
|
||||
|
||||
async def _create_wcd_job(wcd_file: UploadFile, user_params: dict) -> JobCreateResponse:
|
||||
"""WCD 生产任务：校验并还原 .wcd，落库设计，合成生产 PNG 作为任务产物。
|
||||
|
||||
与 POST /api/design-templates/import 共用素材去重/注册/重映射逻辑；
|
||||
状态机与产物管线复用 jobs（queued/running/success/failed）。
|
||||
"""
|
||||
if Path(wcd_file.filename).suffix.lower() != ".wcd":
|
||||
raise HTTPException(status_code=400, detail="wcd_file must be .wcd")
|
||||
content = await wcd_file.read()
|
||||
if not content:
|
||||
raise HTTPException(status_code=400, detail="wcd_file is empty")
|
||||
try:
|
||||
zf = zipfile.ZipFile(io.BytesIO(content))
|
||||
except zipfile.BadZipFile as exc:
|
||||
raise HTTPException(status_code=400, detail="file is not a valid zip/wcd package") from exc
|
||||
|
||||
with zf:
|
||||
manifest = _read_import_json(zf, "manifest.json")
|
||||
order_meta = manifest.get("meta") or {}
|
||||
order_no = str(order_meta.get("orderNo") or "") if isinstance(order_meta, dict) else ""
|
||||
# 兼容旧命名 order-{orderNo}
|
||||
if not order_no and str(manifest.get("name") or "").startswith("order-"):
|
||||
order_no = str(manifest.get("name"))[6:]
|
||||
if manifest.get("format") != "wordcloud-canvas":
|
||||
raise HTTPException(status_code=400, detail="不是 wordcloud-canvas 格式")
|
||||
try:
|
||||
version = int(manifest.get("version", 1))
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise HTTPException(status_code=400, detail="manifest version 不是合法数字") from exc
|
||||
if version != 1:
|
||||
raise HTTPException(status_code=400, detail=f"不支持的包版本: {version}")
|
||||
|
||||
document = _read_import_json(zf, "document.json")
|
||||
asset_items = manifest.get("assets")
|
||||
if not isinstance(asset_items, list):
|
||||
asset_items = []
|
||||
|
||||
remap: dict[str, str] = {}
|
||||
file_map: dict[str, dict] = {}
|
||||
reference_ids: list[str] = []
|
||||
for item in asset_items:
|
||||
pkg_id = str(item.get("id") or "")
|
||||
if not pkg_id:
|
||||
continue
|
||||
entry_name = _zip_asset_entry(zf, pkg_id)
|
||||
mime = _imported_asset_mime(item)
|
||||
asset_bytes = zf.read(entry_name)
|
||||
meta = _register_import_asset(str(item.get("name") or pkg_id), asset_bytes, mime)
|
||||
real_id = str(meta["asset_id"])
|
||||
remap[pkg_id] = real_id
|
||||
file_map[pkg_id] = {
|
||||
"path": str(_asset_dir(real_id) / f"asset{_asset_extension_for_mime(mime)}"),
|
||||
"mime": mime,
|
||||
}
|
||||
reference_ids.append(real_id)
|
||||
_remap_import_asset_ids(document, remap)
|
||||
|
||||
# 落库生产设计（与模板导入一致，便于追溯/复用/在画布中继续编辑）
|
||||
display_name = str(manifest.get("name") or "导入生产设计").strip()[:120] or "导入生产设计"
|
||||
template_id = f"tmpl_{uuid.uuid4().hex}"
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
_write_design_template(_design_template_dir(template_id), {
|
||||
"template_id": template_id,
|
||||
"name": display_name,
|
||||
"description": str(manifest.get("description") or "下单派单生产设计"),
|
||||
"document": document,
|
||||
"reference_asset_ids": reference_ids,
|
||||
"cover_asset_id": reference_ids[0] if reference_ids else "",
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
})
|
||||
|
||||
job_id = manager.create_job()
|
||||
paths = storage.prepare_job_dirs(job_id)
|
||||
(paths.input_dir / f"{template_id}.wcd").write_bytes(content)
|
||||
log.info("[WCD] 生产任务 job_id=%s template=%s assets=%d", job_id, template_id, len(reference_ids))
|
||||
|
||||
# 登记生产订单（供 wordcloud 侧订单列表查看）
|
||||
if order_no:
|
||||
_write_order(_order_dir(order_no), {
|
||||
"order_id": order_no,
|
||||
"order_no": order_no,
|
||||
"job_id": job_id,
|
||||
"template_id": template_id,
|
||||
"status": "running",
|
||||
"created_at": datetime.now(timezone.utc).isoformat(),
|
||||
})
|
||||
log.info("[Order] 登记生产订单 %s → job %s", order_no, job_id)
|
||||
manager.set_status(
|
||||
job_id, status="running", stage="composing", progress_percent=10, message="WCD 生产任务"
|
||||
)
|
||||
|
||||
def _run_wcd_safe() -> None:
|
||||
try:
|
||||
png_path = paths.output_dir / "result.png"
|
||||
_compose_design_png(document, file_map, png_path)
|
||||
manager.set_artifacts(job_id, {"png": str(png_path)})
|
||||
manager.set_status(
|
||||
job_id, status="success", stage="done", progress_percent=100, message="生产完成"
|
||||
)
|
||||
except Exception as exc:
|
||||
logging.exception("wcd job crashed", extra={"job_id": job_id})
|
||||
manager.set_status(
|
||||
job_id,
|
||||
status="failed",
|
||||
stage="failed",
|
||||
progress_percent=100,
|
||||
message="任务失败",
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
threading.Thread(target=_run_wcd_safe, daemon=True).start()
|
||||
return JobCreateResponse(job_id=job_id)
|
||||
|
||||
|
||||
@app.get("/api/jobs/{job_id}", response_model=JobStatus)
|
||||
def get_job(job_id: str) -> JobStatus:
|
||||
if not manager.exists(job_id):
|
||||
@@ -329,6 +546,7 @@ def get_result(job_id: str) -> JobResult:
|
||||
svg_stroke_url=u("svg_stroke"),
|
||||
db_url=u("db"),
|
||||
metrics_url=u("metrics"),
|
||||
elapsed_seconds=status.elapsed_seconds,
|
||||
)
|
||||
|
||||
|
||||
@@ -676,6 +894,45 @@ def get_file(job_id: str, kind: str):
|
||||
return FileResponse(path, media_type=media, filename=path.name)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# 3.9 生产订单列表（下单派单投递的 WCD 生产任务）
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
@app.get("/api/orders")
|
||||
def list_orders() -> list[dict]:
|
||||
"""生产订单列表：来自小程序下单派单投递到 wordcloud 的 WCD 生产任务。"""
|
||||
orders = []
|
||||
if ORDERS_DIR.exists():
|
||||
for item in list(ORDERS_DIR.iterdir()):
|
||||
if not item.is_dir() or not (item / "order.json").exists():
|
||||
continue
|
||||
o = _read_order(item)
|
||||
if not o:
|
||||
continue
|
||||
# 用 job 的最新状态回填
|
||||
try:
|
||||
st = manager.get_status(str(o.get("job_id") or ""))
|
||||
o["status"] = st.status
|
||||
except Exception:
|
||||
pass
|
||||
orders.append(o)
|
||||
return list(reversed(orders))
|
||||
|
||||
|
||||
@app.get("/api/orders/{order_no}")
|
||||
def get_order(order_no: str) -> dict:
|
||||
path = Path(_order_dir(order_no)) / "order.json"
|
||||
if not path.exists():
|
||||
raise HTTPException(status_code=404, detail="order not found")
|
||||
o = json.loads(path.read_text(encoding="utf-8"))
|
||||
try:
|
||||
st = manager.get_status(str(o.get("job_id") or ""))
|
||||
o["status"] = st.status
|
||||
except Exception:
|
||||
pass
|
||||
return o
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# 4. 模板接口
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
@@ -700,6 +957,26 @@ def _write_design_template(template_dir: Path, data: dict) -> None:
|
||||
)
|
||||
|
||||
|
||||
# ── 生产订单存储（下单派单投递的 WCD 生产任务）────────────
|
||||
def _order_dir(order_no: str) -> Path:
|
||||
return ORDERS_DIR / order_no
|
||||
|
||||
|
||||
def _read_order(order_dir: Path) -> dict:
|
||||
path = order_dir / "order.json"
|
||||
if not path.exists():
|
||||
return {}
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def _write_order(order_dir: Path, data: dict) -> None:
|
||||
order_dir.mkdir(parents=True, exist_ok=True)
|
||||
(order_dir / "order.json").write_text(
|
||||
json.dumps(data, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _parse_json_array(value: str, field_name: str) -> list:
|
||||
try:
|
||||
parsed = json.loads(value or "[]")
|
||||
@@ -758,6 +1035,201 @@ async def create_design_template(
|
||||
return DesignTemplate(**data)
|
||||
|
||||
|
||||
def _read_import_json(zf: zipfile.ZipFile, name: str) -> dict:
|
||||
try:
|
||||
with zf.open(name) as fh:
|
||||
raw = fh.read().decode("utf-8")
|
||||
except KeyError as exc:
|
||||
raise HTTPException(status_code=400, detail=f"{name} 缺失") from exc
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise HTTPException(status_code=400, detail=f"{name} 不是合法 JSON") from exc
|
||||
if not isinstance(data, dict):
|
||||
raise HTTPException(status_code=400, detail=f"{name} 必须是 JSON 对象")
|
||||
return data
|
||||
|
||||
|
||||
def _imported_asset_mime(item: dict) -> str:
|
||||
mime = str(item.get("mimeType") or item.get("mime_type") or "").strip().lower()
|
||||
if mime in {"image/svg+xml", "image/png", "image/jpeg"}:
|
||||
return mime
|
||||
asset_type = str(item.get("type") or "").strip().lower()
|
||||
if asset_type in {"svg", "image/svg+xml"}:
|
||||
return "image/svg+xml"
|
||||
if asset_type in {"image", "png"}:
|
||||
return "image/png"
|
||||
return "image/svg+xml" if str(item.get("id", "")).endswith(".svg") else "image/png"
|
||||
|
||||
|
||||
def _asset_meta_items() -> list[dict]:
|
||||
result: list[dict] = []
|
||||
for d in _list_dirs(ASSETS_DIR):
|
||||
meta = _read_asset_meta(d)
|
||||
if meta:
|
||||
result.append(meta)
|
||||
return result
|
||||
|
||||
|
||||
def _find_asset_by_sha256(digest: str) -> dict | None:
|
||||
for meta in _asset_meta_items():
|
||||
if meta.get("sha256") == digest:
|
||||
return meta
|
||||
for meta in _asset_meta_items():
|
||||
try:
|
||||
ext = _asset_extension_for_mime(str(meta.get("mime_type") or ""))
|
||||
path = _asset_dir(str(meta["asset_id"])) / f"asset{ext}"
|
||||
if path.exists() and hashlib.sha256(path.read_bytes()).hexdigest() == digest:
|
||||
meta["sha256"] = digest
|
||||
_write_asset_meta(_asset_dir(str(meta["asset_id"])), meta)
|
||||
return meta
|
||||
except Exception:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _register_import_asset(name: str, content: bytes, mime: str) -> dict:
|
||||
digest = hashlib.sha256(content).hexdigest()
|
||||
existing = _find_asset_by_sha256(digest)
|
||||
if existing:
|
||||
return existing
|
||||
|
||||
asset_id = f"asset_{uuid.uuid4().hex}"
|
||||
asset_path = _asset_dir(asset_id)
|
||||
asset_path.mkdir(parents=True, exist_ok=True)
|
||||
ext = _asset_extension_for_mime(mime)
|
||||
dest = asset_path / f"asset{ext}"
|
||||
dest.write_bytes(content)
|
||||
|
||||
width, height = 0, 0
|
||||
if mime == "image/svg+xml":
|
||||
width, height = _parse_svg_viewbox(dest)
|
||||
else:
|
||||
try:
|
||||
with Image.open(dest) as img:
|
||||
width, height = img.size
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta = {
|
||||
"asset_id": asset_id,
|
||||
"name": name[:120] or "导入素材",
|
||||
"type": "sticker",
|
||||
"mime_type": mime,
|
||||
"width": width,
|
||||
"height": height,
|
||||
"file_size": len(content),
|
||||
"sha256": digest,
|
||||
"file_url": f"/api/assets/{asset_id}/download",
|
||||
"job_id": "",
|
||||
"created_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
_write_asset_meta(asset_path, meta)
|
||||
return meta
|
||||
|
||||
|
||||
def _zip_asset_entry(zf: zipfile.ZipFile, pkg_id: str) -> str:
|
||||
candidates: list[str] = []
|
||||
for entry in zf.namelist():
|
||||
parts = entry.split("/")
|
||||
if (
|
||||
len(parts) >= 2
|
||||
and parts[0] == "assets"
|
||||
and parts[1].startswith(pkg_id)
|
||||
and ".." not in parts
|
||||
and not entry.endswith("/")
|
||||
):
|
||||
candidates.append(entry)
|
||||
if not candidates:
|
||||
raise HTTPException(status_code=400, detail=f"包内缺少素材: {pkg_id}")
|
||||
for entry in candidates:
|
||||
if Path(entry).stem == pkg_id:
|
||||
return entry
|
||||
return sorted(candidates)[0]
|
||||
|
||||
|
||||
def _remap_import_asset_ids(document: dict, asset_map: dict[str, str]) -> None:
|
||||
elements = document.get("elements")
|
||||
if not isinstance(elements, list):
|
||||
return
|
||||
for element in elements:
|
||||
if isinstance(element, dict) and element.get("type") == "sticker":
|
||||
old_id = element.get("assetId")
|
||||
if isinstance(old_id, str) and old_id in asset_map:
|
||||
element["assetId"] = asset_map[old_id]
|
||||
|
||||
|
||||
@app.post("/api/design-templates/import", response_model=DesignTemplate)
|
||||
async def import_design_template(
|
||||
file: UploadFile = File(...),
|
||||
name: str = Form(""),
|
||||
description: str = Form(""),
|
||||
) -> DesignTemplate:
|
||||
if not file.filename:
|
||||
raise HTTPException(status_code=400, detail="file is required")
|
||||
if Path(file.filename).suffix.lower() != ".wcd":
|
||||
raise HTTPException(status_code=400, detail="file must be .wcd")
|
||||
|
||||
content = await file.read()
|
||||
if not content:
|
||||
raise HTTPException(status_code=400, detail="file is empty")
|
||||
|
||||
try:
|
||||
zf = zipfile.ZipFile(io.BytesIO(content))
|
||||
except zipfile.BadZipFile as exc:
|
||||
raise HTTPException(status_code=400, detail="file is not a valid zip/wcd package") from exc
|
||||
|
||||
with zf:
|
||||
manifest = _read_import_json(zf, "manifest.json")
|
||||
if manifest.get("format") != "wordcloud-canvas":
|
||||
raise HTTPException(status_code=400, detail="不是 wordcloud-canvas 格式")
|
||||
try:
|
||||
version = int(manifest.get("version", 1))
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise HTTPException(status_code=400, detail="manifest version 不是合法数字") from exc
|
||||
if version != 1:
|
||||
raise HTTPException(status_code=400, detail=f"不支持的包版本: {version}")
|
||||
|
||||
document = _read_import_json(zf, "document.json")
|
||||
asset_items = manifest.get("assets")
|
||||
if not isinstance(asset_items, list):
|
||||
asset_items = []
|
||||
|
||||
asset_map: dict[str, str] = {}
|
||||
reference_ids: list[str] = []
|
||||
for item in asset_items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
pkg_id = str(item.get("id") or "")
|
||||
if not pkg_id:
|
||||
continue
|
||||
entry_name = _zip_asset_entry(zf, pkg_id)
|
||||
mime = _imported_asset_mime(item)
|
||||
asset_bytes = zf.read(entry_name)
|
||||
meta = _register_import_asset(str(item.get("name") or pkg_id), asset_bytes, mime)
|
||||
asset_map[pkg_id] = str(meta["asset_id"])
|
||||
reference_ids.append(str(meta["asset_id"]))
|
||||
|
||||
_remap_import_asset_ids(document, asset_map)
|
||||
|
||||
display_name = name.strip() or str(manifest.get("name") or "导入设计").strip() or "导入设计"
|
||||
display_description = description.strip() or str(manifest.get("description") or "").strip()
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
template_id = f"tmpl_{uuid.uuid4().hex}"
|
||||
data = {
|
||||
"template_id": template_id,
|
||||
"name": display_name[:120],
|
||||
"description": display_description,
|
||||
"document": document,
|
||||
"reference_asset_ids": reference_ids,
|
||||
"cover_asset_id": reference_ids[0] if reference_ids else "",
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
_write_design_template(_design_template_dir(template_id), data)
|
||||
return DesignTemplate(**data)
|
||||
|
||||
|
||||
@app.get("/api/design-templates/{template_id}", response_model=DesignTemplate)
|
||||
def get_design_template(template_id: str) -> DesignTemplate:
|
||||
data = _read_design_template(_design_template_dir(template_id))
|
||||
|
||||
@@ -6,7 +6,9 @@ import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from .metadata_store import MetadataStore
|
||||
from .schemas import JobDetail, JobEvent, JobStatus
|
||||
|
||||
|
||||
@@ -18,9 +20,20 @@ class JobState:
|
||||
|
||||
|
||||
class JobManager:
|
||||
def __init__(self) -> None:
|
||||
def __init__(self, store: Optional[MetadataStore] = None) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self.store = store
|
||||
self._jobs: dict[str, JobState] = {}
|
||||
self._restore_from_store()
|
||||
|
||||
def _restore_from_store(self) -> None:
|
||||
if not self.store:
|
||||
return
|
||||
for status in self.store.load_jobs():
|
||||
self._jobs[status.job_id] = JobState(
|
||||
status=status,
|
||||
events=self.store.load_events(status.job_id, limit=100),
|
||||
)
|
||||
|
||||
def create_job(self) -> str:
|
||||
job_id = uuid.uuid4().hex
|
||||
@@ -38,6 +51,8 @@ class JobManager:
|
||||
)
|
||||
with self._lock:
|
||||
self._jobs[job_id] = JobState(status=status)
|
||||
if self.store:
|
||||
self.store.upsert_job(status)
|
||||
return job_id
|
||||
|
||||
def exists(self, job_id: str) -> bool:
|
||||
@@ -53,11 +68,19 @@ class JobManager:
|
||||
state = self._jobs[job_id]
|
||||
return JobDetail(status=state.status, recent_events=state.events[-100:])
|
||||
|
||||
def delete_job(self, job_id: str) -> None:
|
||||
with self._lock:
|
||||
self._jobs.pop(job_id, None)
|
||||
if self.store:
|
||||
self.store.delete_job(job_id)
|
||||
|
||||
def set_artifacts(self, job_id: str, artifacts: dict[str, str]) -> None:
|
||||
with self._lock:
|
||||
status = self._jobs[job_id].status
|
||||
status.artifacts = artifacts
|
||||
status.updated_at = datetime.now(timezone.utc)
|
||||
if self.store:
|
||||
self.store.upsert_job(status)
|
||||
|
||||
def set_status(
|
||||
self,
|
||||
@@ -68,6 +91,7 @@ class JobManager:
|
||||
progress_percent: int,
|
||||
message: str,
|
||||
error: str | None = None,
|
||||
elapsed_seconds: float | None = None,
|
||||
) -> None:
|
||||
with self._lock:
|
||||
s = self._jobs[job_id].status
|
||||
@@ -78,14 +102,28 @@ class JobManager:
|
||||
s.updated_at = datetime.now(timezone.utc)
|
||||
if error is not None:
|
||||
s.error = error
|
||||
if elapsed_seconds is not None:
|
||||
s.elapsed_seconds = elapsed_seconds
|
||||
if self.store:
|
||||
self.store.upsert_job(s)
|
||||
|
||||
def add_event(self, job_id: str, *, kind: str, stage: str, progress_percent: int, message: str) -> None:
|
||||
def add_event(
|
||||
self,
|
||||
job_id: str,
|
||||
*,
|
||||
kind: str,
|
||||
stage: str,
|
||||
progress_percent: int,
|
||||
message: str,
|
||||
elapsed_seconds: float | None = None,
|
||||
) -> None:
|
||||
event = JobEvent(
|
||||
type=kind,
|
||||
stage=stage,
|
||||
progress_percent=progress_percent,
|
||||
message=message,
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
elapsed_seconds=elapsed_seconds,
|
||||
)
|
||||
with self._lock:
|
||||
state = self._jobs[job_id]
|
||||
@@ -94,13 +132,26 @@ class JobManager:
|
||||
state.status.progress_percent = progress_percent
|
||||
state.status.message = message
|
||||
state.status.updated_at = event.timestamp
|
||||
if elapsed_seconds is not None:
|
||||
state.status.elapsed_seconds = elapsed_seconds
|
||||
for sub in state.subscribers:
|
||||
sub.put(event)
|
||||
if self.store:
|
||||
self.store.add_event(job_id, event)
|
||||
self.store.upsert_job(state.status)
|
||||
|
||||
def subscribe(self, job_id: str) -> queue.Queue:
|
||||
q: queue.Queue = queue.Queue()
|
||||
with self._lock:
|
||||
self._jobs[job_id].subscribers.append(q)
|
||||
state = self._jobs[job_id]
|
||||
# A fast job can emit preview_ready/completed before the browser
|
||||
# finishes opening the SSE connection. Replay the existing event
|
||||
# history into this subscriber so timing and preview updates are
|
||||
# never lost; the lock also makes the snapshot/registration
|
||||
# atomic with respect to new events.
|
||||
for event in state.events:
|
||||
q.put(event)
|
||||
state.subscribers.append(q)
|
||||
return q
|
||||
|
||||
def unsubscribe(self, job_id: str, q: queue.Queue) -> None:
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
"""Small SQLite-backed metadata store for jobs and events.
|
||||
|
||||
This is intentionally dependency-free and acts as the first durable layer for
|
||||
business metadata. The schema is shaped so it can be moved to PostgreSQL later
|
||||
without changing callers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Optional
|
||||
|
||||
from .schemas import JobEvent, JobStatus
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
class MetadataStore:
|
||||
def __init__(self, db_path: Path) -> None:
|
||||
self.db_path = db_path
|
||||
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._lock = threading.Lock()
|
||||
self._init_db()
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
conn = sqlite3.connect(str(self.db_path), check_same_thread=False)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA busy_timeout=5000")
|
||||
return conn
|
||||
|
||||
def _init_db(self) -> None:
|
||||
with self._lock, self._connect() as conn:
|
||||
conn.executescript(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS jobs (
|
||||
id TEXT PRIMARY KEY,
|
||||
status TEXT NOT NULL,
|
||||
stage TEXT NOT NULL,
|
||||
progress_percent INTEGER NOT NULL DEFAULT 0,
|
||||
message TEXT NOT NULL DEFAULT '',
|
||||
artifacts TEXT NOT NULL DEFAULT '{}',
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
elapsed_seconds REAL,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS job_events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
job_id TEXT NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
stage TEXT NOT NULL,
|
||||
progress_percent INTEGER NOT NULL DEFAULT 0,
|
||||
message TEXT NOT NULL DEFAULT '',
|
||||
timestamp TEXT NOT NULL,
|
||||
elapsed_seconds REAL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_job_events_job_timestamp
|
||||
ON job_events(job_id, id);
|
||||
"""
|
||||
)
|
||||
|
||||
def upsert_job(self, status: JobStatus) -> None:
|
||||
with self._lock, self._connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO jobs (
|
||||
id, status, stage, progress_percent, message,
|
||||
artifacts, error, elapsed_seconds, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
status = excluded.status,
|
||||
stage = excluded.stage,
|
||||
progress_percent = excluded.progress_percent,
|
||||
message = excluded.message,
|
||||
artifacts = excluded.artifacts,
|
||||
error = excluded.error,
|
||||
elapsed_seconds = excluded.elapsed_seconds,
|
||||
updated_at = excluded.updated_at
|
||||
""",
|
||||
(
|
||||
status.job_id,
|
||||
status.status,
|
||||
status.stage,
|
||||
status.progress_percent,
|
||||
status.message,
|
||||
json.dumps(status.artifacts, ensure_ascii=False),
|
||||
status.error,
|
||||
status.elapsed_seconds,
|
||||
status.created_at.isoformat(),
|
||||
status.updated_at.isoformat(),
|
||||
),
|
||||
)
|
||||
|
||||
def add_event(self, job_id: str, event: JobEvent) -> None:
|
||||
with self._lock, self._connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO job_events (
|
||||
job_id, type, stage, progress_percent, message, timestamp, elapsed_seconds
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
job_id,
|
||||
event.type,
|
||||
event.stage,
|
||||
event.progress_percent,
|
||||
event.message,
|
||||
event.timestamp.isoformat(),
|
||||
event.elapsed_seconds,
|
||||
),
|
||||
)
|
||||
|
||||
def load_jobs(self) -> list[JobStatus]:
|
||||
with self._lock, self._connect() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT * FROM jobs
|
||||
"""
|
||||
).fetchall()
|
||||
result: list[JobStatus] = []
|
||||
for row in rows:
|
||||
try:
|
||||
result.append(
|
||||
JobStatus(
|
||||
job_id=row["id"],
|
||||
status=row["status"],
|
||||
stage=row["stage"],
|
||||
progress_percent=row["progress_percent"],
|
||||
message=row["message"],
|
||||
artifacts=json.loads(row["artifacts"] or "{}"),
|
||||
error=row["error"],
|
||||
created_at=datetime.fromisoformat(row["created_at"]),
|
||||
updated_at=datetime.fromisoformat(row["updated_at"]),
|
||||
elapsed_seconds=row["elapsed_seconds"],
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
return result
|
||||
|
||||
def load_events(self, job_id: str, limit: int = 100) -> list[JobEvent]:
|
||||
with self._lock, self._connect() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT type, stage, progress_percent, message, timestamp, elapsed_seconds
|
||||
FROM job_events
|
||||
WHERE job_id = ?
|
||||
ORDER BY id DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
(job_id, limit),
|
||||
).fetchall()
|
||||
return [
|
||||
JobEvent(
|
||||
type=row["type"],
|
||||
stage=row["stage"],
|
||||
progress_percent=row["progress_percent"],
|
||||
message=row["message"],
|
||||
timestamp=datetime.fromisoformat(row["timestamp"]),
|
||||
elapsed_seconds=row["elapsed_seconds"],
|
||||
)
|
||||
for row in reversed(rows)
|
||||
]
|
||||
|
||||
def delete_job(self, job_id: str) -> None:
|
||||
with self._lock, self._connect() as conn:
|
||||
conn.execute("DELETE FROM jobs WHERE id = ?", (job_id,))
|
||||
conn.execute("DELETE FROM job_events WHERE job_id = ?", (job_id,))
|
||||
|
||||
def job_ids(self) -> set[str]:
|
||||
with self._lock, self._connect() as conn:
|
||||
rows = conn.execute("SELECT id FROM jobs").fetchall()
|
||||
return {row["id"] for row in rows}
|
||||
|
||||
def summarize(self) -> dict:
|
||||
with self._lock, self._connect() as conn:
|
||||
jobs = conn.execute("SELECT COUNT(*) AS n FROM jobs").fetchone()
|
||||
events = conn.execute("SELECT COUNT(*) AS n FROM job_events").fetchone()
|
||||
return {
|
||||
"db_size_bytes": self.db_path.stat().st_size if self.db_path.exists() else 0,
|
||||
"jobs": jobs["n"] if jobs else 0,
|
||||
"events": events["n"] if events else 0,
|
||||
}
|
||||
@@ -45,7 +45,7 @@ class JobRunner:
|
||||
return current_stage, current_progress
|
||||
|
||||
def run(self, job_id: str, paths: JobPaths, config: dict) -> None:
|
||||
t_start = time.time()
|
||||
t_start = time.perf_counter()
|
||||
log.info("=" * 50)
|
||||
log.info("[Runner] 任务启动 job_id=%s", job_id)
|
||||
log.info(" config_path = %s", paths.config_path)
|
||||
@@ -65,6 +65,11 @@ class JobRunner:
|
||||
]
|
||||
|
||||
env = os.environ.copy()
|
||||
# Force unbuffered stdout so every print() flushes immediately and the
|
||||
# frontend SSE log view shows each step in real time. Without this,
|
||||
# Python block-buffers stdout when it is a pipe, so lines pile up and
|
||||
# only arrive in bursts after the buffer fills or the process exits.
|
||||
env["PYTHONUNBUFFERED"] = "1"
|
||||
process = subprocess.Popen(
|
||||
cmd,
|
||||
cwd=str(self.project_root),
|
||||
@@ -77,24 +82,63 @@ class JobRunner:
|
||||
|
||||
stage = "starting"
|
||||
progress = 1
|
||||
output_lines: list[str] = [] # collect all output lines for error reporting
|
||||
preview_sent = False
|
||||
|
||||
assert process.stdout is not None
|
||||
for raw in process.stdout:
|
||||
line = raw.rstrip("\n")
|
||||
output_lines.append(line)
|
||||
log.info("[Pipeline] %s", line)
|
||||
stage, progress = self._parse_stage(line, stage, progress)
|
||||
elapsed = time.perf_counter() - t_start
|
||||
self.manager.add_event(
|
||||
job_id,
|
||||
kind="log",
|
||||
stage=stage,
|
||||
progress_percent=progress,
|
||||
message=line,
|
||||
elapsed_seconds=round(elapsed, 3),
|
||||
)
|
||||
# The PNG is complete before SVG/DB export starts. Publish it as a
|
||||
# preview so the frontend does not wait for the slower artifacts.
|
||||
if not preview_sent and line.startswith("已保存:"):
|
||||
candidate = Path(line.split(":", 1)[1].strip())
|
||||
if candidate.suffix.lower() == ".png" and candidate.exists():
|
||||
self.manager.set_artifacts(job_id, {"png": str(candidate)})
|
||||
self.manager.add_event(
|
||||
job_id,
|
||||
kind="status",
|
||||
stage="preview_ready",
|
||||
progress_percent=max(progress, 94),
|
||||
message="预览已生成，后台继续导出其余文件",
|
||||
elapsed_seconds=round(elapsed, 3),
|
||||
)
|
||||
preview_sent = True
|
||||
|
||||
ret = process.wait()
|
||||
elapsed = time.time() - t_start
|
||||
elapsed = time.perf_counter() - t_start
|
||||
log.info("[Runner] 子进程退出 code=%d 耗时=%.2fs", ret, elapsed)
|
||||
|
||||
# ── 错误时：截取最后 30 行输出作为详细错误信息 ─────────────
|
||||
error_detail = None
|
||||
if ret != 0:
|
||||
# 找到 "生成失败" 或 "错误" 或 traceback 之后的内容
|
||||
error_lines = []
|
||||
captured = False
|
||||
for line in reversed(output_lines):
|
||||
if not captured:
|
||||
error_lines.append(line)
|
||||
if any(kw in line for kw in ("生成失败", "error", "Error", "Traceback", "traceback", "未满足", "放置")):
|
||||
captured = True
|
||||
elif len(error_lines) < 30:
|
||||
error_lines.append(line)
|
||||
else:
|
||||
break
|
||||
error_lines.reverse()
|
||||
error_detail = "\n".join(error_lines) if error_lines else f"script exited with code {ret}"
|
||||
log.info("[Runner] 错误详情:\n%s", error_detail)
|
||||
|
||||
png = next(paths.output_dir.glob("*.png"), None)
|
||||
# NOTE: do NOT use "*[!_stroke].svg" — in glob, [!...] is a character class,
|
||||
# so filenames ending with "e.svg" (e.g. AutoResize.svg) are incorrectly skipped.
|
||||
@@ -142,15 +186,24 @@ class JobRunner:
|
||||
return
|
||||
|
||||
if ret == 0:
|
||||
done_message = f"任务完成，用时 {elapsed:.2f} 秒"
|
||||
log.info("[Runner] ✅ 任务完成 job_id=%s 总耗时=%.2fs", job_id, elapsed)
|
||||
self.manager.add_event(
|
||||
job_id,
|
||||
kind="status",
|
||||
stage="completed",
|
||||
progress_percent=100,
|
||||
message="任务完成",
|
||||
message=done_message,
|
||||
elapsed_seconds=round(elapsed, 3),
|
||||
)
|
||||
self.manager.set_status(
|
||||
job_id,
|
||||
status="success",
|
||||
stage="completed",
|
||||
progress_percent=100,
|
||||
message=done_message,
|
||||
elapsed_seconds=round(elapsed, 3),
|
||||
)
|
||||
self.manager.set_status(job_id, status="success", stage="completed", progress_percent=100, message="任务完成")
|
||||
else:
|
||||
log.error("[Runner] ❌ 任务失败 job_id=%s exit_code=%d 耗时=%.2fs", job_id, ret, elapsed)
|
||||
self.manager.add_event(
|
||||
@@ -158,7 +211,8 @@ class JobRunner:
|
||||
kind="status",
|
||||
stage="failed",
|
||||
progress_percent=100,
|
||||
message=f"任务失败，退出码: {ret}",
|
||||
message=f"任务失败，退出码: {ret}，用时 {elapsed:.2f} 秒",
|
||||
elapsed_seconds=round(elapsed, 3),
|
||||
)
|
||||
self.manager.set_status(
|
||||
job_id,
|
||||
@@ -166,5 +220,6 @@ class JobRunner:
|
||||
stage="failed",
|
||||
progress_percent=100,
|
||||
message="任务失败",
|
||||
error=f"script exited with code {ret}",
|
||||
error=error_detail or f"script exited with code {ret}",
|
||||
elapsed_seconds=round(elapsed, 3),
|
||||
)
|
||||
|
||||
@@ -17,6 +17,7 @@ class JobEvent(BaseModel):
|
||||
progress_percent: int = Field(ge=0, le=100)
|
||||
message: str
|
||||
timestamp: datetime
|
||||
elapsed_seconds: float | None = None
|
||||
|
||||
|
||||
class JobStatus(BaseModel):
|
||||
@@ -29,6 +30,7 @@ class JobStatus(BaseModel):
|
||||
updated_at: datetime
|
||||
artifacts: dict[str, str]
|
||||
error: str = ""
|
||||
elapsed_seconds: float | None = None
|
||||
|
||||
|
||||
class JobDetail(BaseModel):
|
||||
@@ -44,6 +46,7 @@ class JobResult(BaseModel):
|
||||
svg_stroke_url: str = ""
|
||||
db_url: str = ""
|
||||
metrics_url: str = ""
|
||||
elapsed_seconds: float | None = None
|
||||
|
||||
|
||||
class WordLocation(BaseModel):
|
||||
|
||||
@@ -1,10 +1,17 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from .schemas import JobPaths
|
||||
|
||||
|
||||
_JOB_ID_RE = re.compile(r"^[0-9a-f]{32,}$")
|
||||
|
||||
|
||||
class Storage:
|
||||
def __init__(self, base_dir: Path) -> None:
|
||||
self.base_dir = base_dir
|
||||
@@ -25,3 +32,70 @@ class Storage:
|
||||
excel_path=input_dir / "names.xlsx",
|
||||
config_path=root / "config.json",
|
||||
)
|
||||
|
||||
def job_root(self, job_id: str) -> Path:
|
||||
return self.base_dir / job_id
|
||||
|
||||
def job_dir_size(self, job_id: str) -> int:
|
||||
root = self.job_root(job_id)
|
||||
if not root.exists():
|
||||
return 0
|
||||
total = 0
|
||||
for dirpath, _, filenames in os.walk(root):
|
||||
for filename in filenames:
|
||||
try:
|
||||
total += os.path.getsize(os.path.join(dirpath, filename))
|
||||
except OSError:
|
||||
continue
|
||||
return total
|
||||
|
||||
def job_dir_info(self, job_id: str) -> dict | None:
|
||||
root = self.job_root(job_id)
|
||||
if not root.is_dir():
|
||||
return None
|
||||
try:
|
||||
mtime = root.stat().st_mtime
|
||||
except OSError:
|
||||
return None
|
||||
return {
|
||||
"job_id": job_id,
|
||||
"path": str(root),
|
||||
"size_bytes": self.job_dir_size(job_id),
|
||||
"age_days": round(max(0.0, time.time() - mtime) / 86400.0, 2),
|
||||
}
|
||||
|
||||
def list_job_ids(self) -> list[str]:
|
||||
if not self.base_dir.exists():
|
||||
return []
|
||||
return [
|
||||
item.name
|
||||
for item in self.base_dir.iterdir()
|
||||
if item.is_dir() and _JOB_ID_RE.match(item.name)
|
||||
]
|
||||
|
||||
def stale_job_dirs(
|
||||
self,
|
||||
referenced_job_ids: set[str],
|
||||
exclude_job_ids: set[str] | None = None,
|
||||
max_age_days: float | None = None,
|
||||
) -> list[dict]:
|
||||
exclude = exclude_job_ids or set()
|
||||
result: list[dict] = []
|
||||
for job_id in self.list_job_ids():
|
||||
if job_id in referenced_job_ids or job_id in exclude:
|
||||
continue
|
||||
info = self.job_dir_info(job_id)
|
||||
if not info:
|
||||
continue
|
||||
if max_age_days is not None and info["age_days"] < max_age_days:
|
||||
continue
|
||||
result.append(info)
|
||||
return sorted(result, key=lambda item: item["size_bytes"], reverse=True)
|
||||
|
||||
def remove_job_dir(self, job_id: str) -> int:
|
||||
root = self.job_root(job_id)
|
||||
if not root.exists():
|
||||
return 0
|
||||
size = self.job_dir_size(job_id)
|
||||
shutil.rmtree(root, ignore_errors=True)
|
||||
return size
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Inspect and optionally clean stale job directories.
|
||||
|
||||
Default mode is a safe dry-run that reports reclaimable bytes. Pass --apply to
|
||||
actually remove unreferenced job directories.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from .metadata_store import MetadataStore
|
||||
from .storage import Storage
|
||||
|
||||
|
||||
BACKEND_ROOT = Path(__file__).resolve().parents[1]
|
||||
PROJECT_ROOT = BACKEND_ROOT
|
||||
WORKSPACE_DIR = PROJECT_ROOT / "service_workspace"
|
||||
ASSETS_DIR = PROJECT_ROOT / "service_assets"
|
||||
METADATA_DIR = PROJECT_ROOT / "service_metadata"
|
||||
|
||||
|
||||
def read_asset_meta(path: Path) -> dict:
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def referenced_job_ids() -> set[str]:
|
||||
refs: set[str] = set()
|
||||
for meta_path in ASSETS_DIR.glob("*/*/meta.json"):
|
||||
job_id = read_asset_meta(meta_path).get("job_id") or ""
|
||||
if job_id:
|
||||
refs.add(job_id)
|
||||
return refs
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--max-age-days", type=float, default=0)
|
||||
parser.add_argument("--apply", action="store_true", help="Actually delete stale job directories")
|
||||
parser.add_argument("--json", type=Path, default=None, help="Write JSON report")
|
||||
args = parser.parse_args()
|
||||
|
||||
store = MetadataStore(METADATA_DIR / "app.db")
|
||||
storage = Storage(WORKSPACE_DIR)
|
||||
known_job_ids = store.job_ids() if store.db_path.exists() else set()
|
||||
referenced = referenced_job_ids()
|
||||
stale = storage.stale_job_dirs(
|
||||
referenced_job_ids=referenced,
|
||||
exclude_job_ids=known_job_ids,
|
||||
max_age_days=args.max_age_days,
|
||||
)
|
||||
|
||||
reclaimable = sum(item["size_bytes"] for item in stale)
|
||||
report = {
|
||||
"scanned_at": datetime.now(timezone.utc).isoformat(),
|
||||
"job_dir_count": len(storage.list_job_ids()),
|
||||
"referenced_job_ids": len(referenced),
|
||||
"known_job_ids": len(known_job_ids),
|
||||
"stale_job_count": len(stale),
|
||||
"reclaimable_bytes": reclaimable,
|
||||
"max_age_days": args.max_age_days,
|
||||
"apply": args.apply,
|
||||
"stale_jobs": stale[:200],
|
||||
}
|
||||
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if args.json:
|
||||
args.json.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
if args.apply:
|
||||
freed = 0
|
||||
for item in stale:
|
||||
freed += storage.remove_job_dir(item["job_id"])
|
||||
store.delete_job(item["job_id"])
|
||||
print(f"freed_bytes={freed}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user