fix: quarantine failed product cleanups in failed_cleanup state

backend/tests/test_cleanup_service.py

@@ -197,7 +197,9 @@ def test_atomic_purge_claim_blocks_restore_before_product_files_are_deleted(
     assert service.product_store.get_product(product.product_id).status == "purged"
 
 
-def test_failed_product_file_deletion_releases_claim_back_to_pending(tmp_path, monkeypatch):
+def test_failed_product_file_deletion_enters_failed_cleanup_and_is_not_retried(
+    tmp_path, monkeypatch
+):
     service = make_cleanup_service(tmp_path)
     product = create_pending_product(service, purge_after=NOW)
 
@@ -209,11 +211,47 @@ def test_failed_product_file_deletion_releases_claim_back_to_pending(tmp_path, m
     with pytest.raises(OSError, match="disk failure"):
         service.apply(now=NOW)
 
-    assert service.product_store.get_product(product.product_id).status == "pending_cleanup"
-    assert service.product_store._fetchone(
+    record = service.product_store.get_product(product.product_id)
+    cleanup = service.product_store._fetchone(
         "SELECT status FROM cleanup_records WHERE product_id = ?", (product.product_id,)
-    )["status"] == "pending_cleanup"
-    assert service.product_store.due_product_cleanups(NOW)[0].product_id == product.product_id
+    )["status"]
+
+    assert record.status == "failed_cleanup"
+    assert record.purge_after == NOW
+    assert cleanup == "failed_cleanup"
+    assert service.product_store.due_product_cleanups(NOW) == []
+    assert (service.product_store.root.parent / product.product_id).exists()
+
+
+def test_failed_cleanup_product_can_be_restored_or_requeued(tmp_path, monkeypatch):
+    service = make_cleanup_service(tmp_path)
+
+    def _fail_purge(purge_after: datetime) -> str:
+        product = create_pending_product(service, purge_after=purge_after)
+        monkeypatch.setattr(
+            "service.cleanup_service.shutil.rmtree",
+            lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("disk failure")),
+        )
+        with pytest.raises(OSError):
+            service.apply(now=purge_after)
+        assert service.product_store.get_product(product.product_id).status == "failed_cleanup"
+        return product.product_id
+
+    failed = _fail_purge(NOW)
+    restored = service.product_store.restore_product(failed, now=NOW + timedelta(days=1))
+    assert restored.status == "active"
+    assert restored.purge_after is None
+
+    failed_requeue = _fail_purge(NOW)
+    requeued = service.product_store.mark_pending_cleanup(
+        failed_requeue, now=NOW + timedelta(days=1)
+    )
+    assert requeued.status == "pending_cleanup"
+    assert requeued.purge_after == NOW + timedelta(days=31)
+    assert service.product_store.due_product_cleanups(NOW + timedelta(days=5)) == []
+    assert service.product_store.due_product_cleanups(NOW + timedelta(days=31)) == [
+        requeued
+    ]
 
 
 def test_apply_rechecks_archive_reference_immediately_before_job_deletion(tmp_path):