feat: add product archive retention cleanup
This commit is contained in:
+99
-13
@@ -12,6 +12,7 @@ import threading
|
||||
import time
|
||||
import uuid
|
||||
import zipfile
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
@@ -23,6 +24,7 @@ from PIL import Image, ImageDraw, ImageFont
|
||||
|
||||
from core import config as wc_config
|
||||
from core.fonts import get_cached_font
|
||||
from .cleanup_service import CleanupReport, CleanupService
|
||||
from .job_manager import JobManager
|
||||
from .line_spacing import analyze_svg_line_spacing_file
|
||||
from .log_config import get_logger
|
||||
@@ -67,7 +69,7 @@ FONTS_DIR = PROJECT_ROOT / "service_fonts"
|
||||
DESIGN_TEMPLATES_DIR = PROJECT_ROOT / "service_design_templates"
|
||||
METADATA_DIR = PROJECT_ROOT / "service_metadata"
|
||||
ORDERS_DIR = PROJECT_ROOT / "service_orders"
|
||||
PRODUCT_ARCHIVES_DIR = PROJECT_ROOT / "service_product_archives"
|
||||
PRODUCT_ARCHIVES_DIR = PROJECT_ROOT / "service_products"
|
||||
ASSETS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
PROJECTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
FONTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
@@ -81,8 +83,19 @@ manager = JobManager(metadata_store)
|
||||
storage = Storage(WORKSPACE_DIR)
|
||||
runner = JobRunner(PROJECT_ROOT, manager)
|
||||
product_archive_store = ProductArchiveStore(PRODUCT_ARCHIVES_DIR / "metadata")
|
||||
cleanup_service = CleanupService(storage, metadata_store, product_archive_store)
|
||||
|
||||
app = FastAPI(title="WordCloud Test Service", version="0.1.0")
|
||||
|
||||
@asynccontextmanager
|
||||
async def _app_lifespan(_app: FastAPI):
|
||||
_start_cleanup_scheduler()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_stop_cleanup_scheduler()
|
||||
|
||||
|
||||
app = FastAPI(title="WordCloud Test Service", version="0.1.0", lifespan=_app_lifespan)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
@@ -162,26 +175,27 @@ def health() -> dict:
|
||||
|
||||
@app.get("/api/maintenance/storage-summary")
|
||||
def storage_summary() -> dict:
|
||||
referenced_jobs: set[str] = set()
|
||||
asset_referenced_jobs: set[str] = set()
|
||||
for d in _list_dirs(ASSETS_DIR):
|
||||
meta = _read_asset_meta(d)
|
||||
job_id = meta.get("job_id") or ""
|
||||
if job_id:
|
||||
referenced_jobs.add(job_id)
|
||||
stale = storage.stale_job_dirs(
|
||||
referenced_job_ids=referenced_jobs,
|
||||
exclude_job_ids=metadata_store.job_ids(),
|
||||
max_age_days=0,
|
||||
)
|
||||
asset_referenced_jobs.add(job_id)
|
||||
report = cleanup_service.preview(datetime.now(timezone.utc))
|
||||
archive_protected_jobs = cleanup_service.archive_protected_job_ids()
|
||||
return {
|
||||
"job_dir_count": len(storage.list_job_ids()),
|
||||
"referenced_job_ids": len(referenced_jobs),
|
||||
"stale_job_count": len(stale),
|
||||
"reclaimable_bytes": sum(item["size_bytes"] for item in stale),
|
||||
"referenced_job_ids": len(asset_referenced_jobs),
|
||||
"asset_referenced_job_ids": sorted(asset_referenced_jobs),
|
||||
"archive_protected_job_ids": sorted(archive_protected_jobs),
|
||||
"archive_protected_job_count": len(archive_protected_jobs),
|
||||
"temporary_job_ids": [item.job_id for item in report.temporary_jobs],
|
||||
"stale_job_count": len(report.temporary_jobs),
|
||||
"reclaimable_bytes": report.reclaimable_bytes,
|
||||
"metadata_db_bytes": metadata_store.summarize()["db_size_bytes"],
|
||||
"jobs_in_db": metadata_store.summarize()["jobs"],
|
||||
"events_in_db": metadata_store.summarize()["events"],
|
||||
"dry_run_only": True,
|
||||
"dry_run_only": not _cleanup_apply_enabled(),
|
||||
}
|
||||
|
||||
|
||||
@@ -1023,6 +1037,78 @@ def _require_orders_auth(request: Request) -> None:
|
||||
raise HTTPException(status_code=403, detail="需要登录") # noqa: S105
|
||||
|
||||
|
||||
def _cleanup_apply_enabled() -> bool:
|
||||
return os.environ.get("CLEANUP_APPLY_ENABLED", "false").strip().lower() == "true"
|
||||
|
||||
|
||||
def _run_scheduled_cleanup_once(now: datetime | None = None) -> CleanupReport:
|
||||
run_at = now or datetime.now(timezone.utc)
|
||||
apply_enabled = _cleanup_apply_enabled()
|
||||
report = cleanup_service.apply(run_at) if apply_enabled else cleanup_service.preview(run_at)
|
||||
log.info(
|
||||
"cleanup cycle apply=%s temporary_jobs=%d pending_products=%d reclaimable_bytes=%d "
|
||||
"reminder_job_ids=%s deleted_job_ids=%s deleted_product_ids=%s",
|
||||
apply_enabled,
|
||||
len(report.temporary_jobs),
|
||||
len(report.pending_products),
|
||||
report.reclaimable_bytes,
|
||||
report.reminder_job_ids,
|
||||
report.deleted_job_ids,
|
||||
report.deleted_product_ids,
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
_cleanup_scheduler_stop = threading.Event()
|
||||
_cleanup_scheduler_thread: threading.Thread | None = None
|
||||
|
||||
|
||||
def _cleanup_scheduler_loop() -> None:
|
||||
while not _cleanup_scheduler_stop.wait(24 * 60 * 60):
|
||||
try:
|
||||
_run_scheduled_cleanup_once()
|
||||
except Exception:
|
||||
log.exception("scheduled cleanup cycle failed")
|
||||
|
||||
|
||||
def _start_cleanup_scheduler() -> None:
|
||||
global _cleanup_scheduler_thread
|
||||
_run_scheduled_cleanup_once()
|
||||
if _cleanup_scheduler_thread is not None and _cleanup_scheduler_thread.is_alive():
|
||||
return
|
||||
_cleanup_scheduler_stop.clear()
|
||||
_cleanup_scheduler_thread = threading.Thread(
|
||||
target=_cleanup_scheduler_loop,
|
||||
name="retention-cleanup",
|
||||
daemon=True,
|
||||
)
|
||||
_cleanup_scheduler_thread.start()
|
||||
|
||||
|
||||
def _stop_cleanup_scheduler() -> None:
|
||||
global _cleanup_scheduler_thread
|
||||
_cleanup_scheduler_stop.set()
|
||||
if _cleanup_scheduler_thread is not None:
|
||||
_cleanup_scheduler_thread.join(timeout=1)
|
||||
_cleanup_scheduler_thread = None
|
||||
|
||||
|
||||
@app.get("/api/maintenance/cleanup-candidates", response_model=CleanupReport)
|
||||
def cleanup_candidates(request: Request) -> CleanupReport:
|
||||
_require_orders_auth(request)
|
||||
return cleanup_service.preview(datetime.now(timezone.utc))
|
||||
|
||||
|
||||
@app.post("/api/maintenance/cleanup-run", response_model=CleanupReport)
|
||||
def cleanup_run(request: Request, payload: dict) -> CleanupReport:
|
||||
_require_orders_auth(request)
|
||||
if payload.get("confirm") is not True:
|
||||
raise HTTPException(status_code=400, detail="confirm must be true")
|
||||
if not _cleanup_apply_enabled():
|
||||
raise HTTPException(status_code=409, detail="cleanup apply is disabled")
|
||||
return cleanup_service.apply(datetime.now(timezone.utc))
|
||||
|
||||
|
||||
@app.post("/api/login")
|
||||
async def login(request: Request) -> dict:
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
"""Deterministic retention preview and physical cleanup for jobs and products."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from .metadata_store import MetadataStore
|
||||
from .product_archive_store import ProductArchiveStore
|
||||
from .schemas import ProductRecord
|
||||
from .storage import Storage
|
||||
|
||||
|
||||
TEMPORARY_RETENTION = timedelta(days=30)
|
||||
REMINDER_AGE = timedelta(days=23)
|
||||
|
||||
|
||||
class TemporaryJob(BaseModel):
|
||||
job_id: str
|
||||
created_at: datetime
|
||||
size_bytes: int
|
||||
|
||||
|
||||
class CleanupReport(BaseModel):
|
||||
temporary_jobs: list[TemporaryJob] = Field(default_factory=list)
|
||||
pending_products: list[ProductRecord] = Field(default_factory=list)
|
||||
reclaimable_bytes: int = 0
|
||||
reminder_job_ids: list[str] = Field(default_factory=list)
|
||||
deleted_job_ids: list[str] = Field(default_factory=list)
|
||||
deleted_product_ids: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
def _as_utc(value: datetime) -> datetime:
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
class CleanupService:
|
||||
"""Calculate candidates first and apply only those still eligible."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
storage: Storage,
|
||||
metadata_store: MetadataStore,
|
||||
product_store: ProductArchiveStore,
|
||||
) -> None:
|
||||
self.storage = storage
|
||||
self.metadata_store = metadata_store
|
||||
self.product_store = product_store
|
||||
|
||||
def archive_protected_job_ids(self) -> set[str]:
|
||||
rows = self.product_store._fetchall(
|
||||
"""SELECT DISTINCT source_job_id
|
||||
FROM product_wordcloud_archives
|
||||
WHERE source_job_id != ''"""
|
||||
)
|
||||
return {str(row["source_job_id"]) for row in rows}
|
||||
|
||||
def _is_archive_protected(self, job_id: str) -> bool:
|
||||
row = self.product_store._fetchone(
|
||||
"""SELECT 1 FROM product_wordcloud_archives
|
||||
WHERE source_job_id = ? LIMIT 1""",
|
||||
(job_id,),
|
||||
)
|
||||
return row is not None
|
||||
|
||||
def preview(self, now: datetime) -> CleanupReport:
|
||||
now_utc = _as_utc(now)
|
||||
protected = self.archive_protected_job_ids()
|
||||
temporary_jobs: list[TemporaryJob] = []
|
||||
reminder_job_ids: list[str] = []
|
||||
|
||||
for job in self.metadata_store.load_jobs():
|
||||
if job.status != "success" or not job.artifacts.get("db") or job.job_id in protected:
|
||||
continue
|
||||
age = now_utc - _as_utc(job.created_at)
|
||||
if age >= TEMPORARY_RETENTION:
|
||||
temporary_jobs.append(
|
||||
TemporaryJob(
|
||||
job_id=job.job_id,
|
||||
created_at=job.created_at,
|
||||
size_bytes=self.storage.job_dir_size(job.job_id),
|
||||
)
|
||||
)
|
||||
elif age >= REMINDER_AGE:
|
||||
reminder_job_ids.append(job.job_id)
|
||||
|
||||
temporary_jobs.sort(key=lambda item: (item.created_at, item.job_id))
|
||||
reminder_job_ids.sort()
|
||||
pending_products = self.product_store.due_product_cleanups(now_utc)
|
||||
product_bytes = sum(
|
||||
self._directory_size(self.product_store.root.parent / product.product_id)
|
||||
for product in pending_products
|
||||
)
|
||||
return CleanupReport(
|
||||
temporary_jobs=temporary_jobs,
|
||||
pending_products=pending_products,
|
||||
reclaimable_bytes=sum(item.size_bytes for item in temporary_jobs) + product_bytes,
|
||||
reminder_job_ids=reminder_job_ids,
|
||||
)
|
||||
|
||||
def apply(self, now: datetime) -> CleanupReport:
|
||||
now_utc = _as_utc(now)
|
||||
report = self.preview(now_utc)
|
||||
|
||||
for item in report.temporary_jobs:
|
||||
# An archive can be committed after preview. Protect it at the last
|
||||
# possible point before removing the original workspace.
|
||||
if self._is_archive_protected(item.job_id):
|
||||
continue
|
||||
self.storage.remove_job_dir(item.job_id)
|
||||
self.metadata_store.delete_job(item.job_id)
|
||||
report.deleted_job_ids.append(item.job_id)
|
||||
|
||||
for candidate in report.pending_products:
|
||||
# Re-read logical state so a concurrent restore wins over cleanup.
|
||||
try:
|
||||
product = self.product_store.get_product(candidate.product_id)
|
||||
except ValueError:
|
||||
continue
|
||||
if (
|
||||
product.status != "pending_cleanup"
|
||||
or product.purge_after is None
|
||||
or _as_utc(product.purge_after) > now_utc
|
||||
):
|
||||
continue
|
||||
product_dir = self.product_store.root.parent / product.product_id
|
||||
if product_dir.exists():
|
||||
shutil.rmtree(product_dir)
|
||||
self.product_store.purge_product(product.product_id, now=now_utc)
|
||||
report.deleted_product_ids.append(product.product_id)
|
||||
|
||||
return report
|
||||
|
||||
@staticmethod
|
||||
def _directory_size(root: Path) -> int:
|
||||
if not root.exists():
|
||||
return 0
|
||||
return sum(path.stat().st_size for path in root.rglob("*") if path.is_file())
|
||||
@@ -9,10 +9,13 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from .cleanup_service import CleanupService
|
||||
from .metadata_store import MetadataStore
|
||||
from .product_archive_store import ProductArchiveStore
|
||||
from .storage import Storage
|
||||
|
||||
|
||||
@@ -21,6 +24,7 @@ PROJECT_ROOT = BACKEND_ROOT
|
||||
WORKSPACE_DIR = PROJECT_ROOT / "service_workspace"
|
||||
ASSETS_DIR = PROJECT_ROOT / "service_assets"
|
||||
METADATA_DIR = PROJECT_ROOT / "service_metadata"
|
||||
PRODUCTS_DIR = PROJECT_ROOT / "service_products"
|
||||
|
||||
|
||||
def read_asset_meta(path: Path) -> dict:
|
||||
@@ -41,32 +45,40 @@ def referenced_job_ids() -> set[str]:
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--max-age-days", type=float, default=0)
|
||||
parser.add_argument("--max-age-days", type=float, default=30)
|
||||
parser.add_argument("--apply", action="store_true", help="Actually delete stale job directories")
|
||||
parser.add_argument("--json", type=Path, default=None, help="Write JSON report")
|
||||
args = parser.parse_args()
|
||||
|
||||
store = MetadataStore(METADATA_DIR / "app.db")
|
||||
storage = Storage(WORKSPACE_DIR)
|
||||
product_store = ProductArchiveStore(PRODUCTS_DIR / "metadata")
|
||||
cleanup = CleanupService(storage, store, product_store)
|
||||
known_job_ids = store.job_ids() if store.db_path.exists() else set()
|
||||
referenced = referenced_job_ids()
|
||||
stale = storage.stale_job_dirs(
|
||||
referenced_job_ids=referenced,
|
||||
asset_referenced = referenced_job_ids()
|
||||
archive_protected = cleanup.archive_protected_job_ids()
|
||||
orphaned = storage.stale_job_dirs(
|
||||
referenced_job_ids=archive_protected,
|
||||
exclude_job_ids=known_job_ids,
|
||||
max_age_days=args.max_age_days,
|
||||
)
|
||||
|
||||
reclaimable = sum(item["size_bytes"] for item in stale)
|
||||
cleanup_report = cleanup.preview(datetime.now(timezone.utc))
|
||||
reclaimable = cleanup_report.reclaimable_bytes + sum(item["size_bytes"] for item in orphaned)
|
||||
report = {
|
||||
"scanned_at": datetime.now(timezone.utc).isoformat(),
|
||||
"job_dir_count": len(storage.list_job_ids()),
|
||||
"referenced_job_ids": len(referenced),
|
||||
"referenced_job_ids": len(asset_referenced),
|
||||
"asset_referenced_job_ids": sorted(asset_referenced),
|
||||
"archive_protected_job_ids": sorted(archive_protected),
|
||||
"known_job_ids": len(known_job_ids),
|
||||
"stale_job_count": len(stale),
|
||||
"temporary_jobs": [item.model_dump(mode="json") for item in cleanup_report.temporary_jobs],
|
||||
"pending_product_ids": [item.product_id for item in cleanup_report.pending_products],
|
||||
"reminder_job_ids": cleanup_report.reminder_job_ids,
|
||||
"stale_job_count": len(orphaned),
|
||||
"reclaimable_bytes": reclaimable,
|
||||
"max_age_days": args.max_age_days,
|
||||
"apply": args.apply,
|
||||
"stale_jobs": stale[:200],
|
||||
"stale_jobs": orphaned[:200],
|
||||
}
|
||||
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
@@ -74,11 +86,18 @@ def main() -> None:
|
||||
args.json.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
if args.apply:
|
||||
if os.environ.get("CLEANUP_APPLY_ENABLED", "false").strip().lower() != "true":
|
||||
parser.error("--apply requires CLEANUP_APPLY_ENABLED=true")
|
||||
applied = cleanup.apply(datetime.now(timezone.utc))
|
||||
freed = 0
|
||||
for item in stale:
|
||||
for item in orphaned:
|
||||
freed += storage.remove_job_dir(item["job_id"])
|
||||
store.delete_job(item["job_id"])
|
||||
print(f"freed_bytes={freed}")
|
||||
print(
|
||||
"freed_bytes="
|
||||
f"{freed} deleted_job_ids={applied.deleted_job_ids} "
|
||||
f"deleted_product_ids={applied.deleted_product_ids}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user