feat: add product archive retention cleanup

backend/service/storage_metrics.py

@@ -9,10 +9,13 @@ from __future__ import annotations
 
 import argparse
 import json
+import os
 from datetime import datetime, timezone
 from pathlib import Path
 
+from .cleanup_service import CleanupService
 from .metadata_store import MetadataStore
+from .product_archive_store import ProductArchiveStore
 from .storage import Storage
 
 
@@ -21,6 +24,7 @@ PROJECT_ROOT = BACKEND_ROOT
 WORKSPACE_DIR = PROJECT_ROOT / "service_workspace"
 ASSETS_DIR = PROJECT_ROOT / "service_assets"
 METADATA_DIR = PROJECT_ROOT / "service_metadata"
+PRODUCTS_DIR = PROJECT_ROOT / "service_products"
 
 
 def read_asset_meta(path: Path) -> dict:
@@ -41,32 +45,40 @@ def referenced_job_ids() -> set[str]:
 
 def main() -> None:
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--max-age-days", type=float, default=0)
+    parser.add_argument("--max-age-days", type=float, default=30)
     parser.add_argument("--apply", action="store_true", help="Actually delete stale job directories")
     parser.add_argument("--json", type=Path, default=None, help="Write JSON report")
     args = parser.parse_args()
 
     store = MetadataStore(METADATA_DIR / "app.db")
     storage = Storage(WORKSPACE_DIR)
+    product_store = ProductArchiveStore(PRODUCTS_DIR / "metadata")
+    cleanup = CleanupService(storage, store, product_store)
     known_job_ids = store.job_ids() if store.db_path.exists() else set()
-    referenced = referenced_job_ids()
-    stale = storage.stale_job_dirs(
-        referenced_job_ids=referenced,
+    asset_referenced = referenced_job_ids()
+    archive_protected = cleanup.archive_protected_job_ids()
+    orphaned = storage.stale_job_dirs(
+        referenced_job_ids=archive_protected,
         exclude_job_ids=known_job_ids,
         max_age_days=args.max_age_days,
     )
-
-    reclaimable = sum(item["size_bytes"] for item in stale)
+    cleanup_report = cleanup.preview(datetime.now(timezone.utc))
+    reclaimable = cleanup_report.reclaimable_bytes + sum(item["size_bytes"] for item in orphaned)
     report = {
         "scanned_at": datetime.now(timezone.utc).isoformat(),
         "job_dir_count": len(storage.list_job_ids()),
-        "referenced_job_ids": len(referenced),
+        "referenced_job_ids": len(asset_referenced),
+        "asset_referenced_job_ids": sorted(asset_referenced),
+        "archive_protected_job_ids": sorted(archive_protected),
         "known_job_ids": len(known_job_ids),
-        "stale_job_count": len(stale),
+        "temporary_jobs": [item.model_dump(mode="json") for item in cleanup_report.temporary_jobs],
+        "pending_product_ids": [item.product_id for item in cleanup_report.pending_products],
+        "reminder_job_ids": cleanup_report.reminder_job_ids,
+        "stale_job_count": len(orphaned),
         "reclaimable_bytes": reclaimable,
         "max_age_days": args.max_age_days,
         "apply": args.apply,
-        "stale_jobs": stale[:200],
+        "stale_jobs": orphaned[:200],
     }
 
     print(json.dumps(report, ensure_ascii=False, indent=2))
@@ -74,11 +86,18 @@ def main() -> None:
         args.json.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
 
     if args.apply:
+        if os.environ.get("CLEANUP_APPLY_ENABLED", "false").strip().lower() != "true":
+            parser.error("--apply requires CLEANUP_APPLY_ENABLED=true")
+        applied = cleanup.apply(datetime.now(timezone.utc))
         freed = 0
-        for item in stale:
+        for item in orphaned:
             freed += storage.remove_job_dir(item["job_id"])
             store.delete_job(item["job_id"])
-        print(f"freed_bytes={freed}")
+        print(
+            "freed_bytes="
+            f"{freed} deleted_job_ids={applied.deleted_job_ids} "
+            f"deleted_product_ids={applied.deleted_product_ids}"
+        )
 
 
 if __name__ == "__main__":