ENG-2526: Reprocess DSRs via /resubmit instead of /retry

changelog/8199-reprocess-dsr-picks-up-integration-changes.yaml

@@ -0,0 +1,4 @@
+type: Changed
+description: The privacy request Reprocess action now picks up integrations added or removed after the original failure; the request is now resubmitted instead of retried (DAG rebuilt, prior execution logs cleared) instead of retried against the stale DAG. Adds a new `POST /privacy-request/bulk/resubmit` endpoint to support bulk reprocessing.
+pr: 8199
+labels: ["high-risk"]

clients/admin-ui/src/features/privacy-requests/buttons/ReprocessButton.tsx

@@ -7,8 +7,8 @@ import { getErrorMessage, isErrorResult } from "~/features/common/helpers";
 import {
   selectRetryRequests,
   setRetryRequests,
-  useBulkRetryMutation,
-  useRetryMutation,
+  useBulkResubmitMutation,
+  useResubmitMutation,
 } from "../privacy-requests.slice";
 import { PrivacyRequestEntity } from "../types";
 
@@ -24,12 +24,12 @@ const ReprocessButton = forwardRef<HTMLButtonElement, ReprocessButtonProps>(
     const message = useMessage();
 
     const { errorRequests } = useAppSelector(selectRetryRequests);
-    const [bulkRetry] = useBulkRetryMutation();
-    const [retry] = useRetryMutation();
+    const [bulkResubmit] = useBulkResubmitMutation();
+    const [resubmit] = useResubmitMutation();
 
     const handleBulkReprocessClick = async () => {
       setIsReprocessing(true);
-      const payload = await bulkRetry(errorRequests);
+      const payload = await bulkResubmit(errorRequests);
       if (isErrorResult(payload)) {
         dispatch(setRetryRequests({ checkAll: false, errorRequests: [] }));
         message.error({
@@ -64,7 +64,7 @@ const ReprocessButton = forwardRef<HTMLButtonElement, ReprocessButtonProps>(
         return;
       }
       setIsReprocessing(true);
-      const payload = await retry(subjectRequest);
+      const payload = await resubmit(subjectRequest);
       if (isErrorResult(payload)) {
         message.error({
           content: `DSR automation has failed for this privacy request due to the following: ${getErrorMessage(payload.error)}`,

clients/admin-ui/src/features/privacy-requests/hooks/useReprocessPrivacyRequest.ts

@@ -3,7 +3,7 @@ import { useMessage } from "fidesui";
 import { getErrorMessage } from "~/features/common/helpers";
 import { PrivacyRequestStatus } from "~/types/api";
 
-import { useRetryMutation } from "../privacy-requests.slice";
+import { useResubmitMutation } from "../privacy-requests.slice";
 import { PrivacyRequestEntity } from "../types";
 
 const useReprocessPrivacyRequest = ({
@@ -15,14 +15,14 @@ const useReprocessPrivacyRequest = ({
   const isErrorStatus = privacyRequest.status === PrivacyRequestStatus.ERROR;
   const showReprocess = isErrorStatus;
 
-  const [retry] = useRetryMutation();
+  const [resubmit] = useResubmitMutation();
 
   const reprocessPrivacyRequest = async () => {
     if (!privacyRequest) {
       return;
     }
     const closeMessage = message.loading("Reprocessing privacy request...", 0);
-    const payload = await retry(privacyRequest);
+    const payload = await resubmit(privacyRequest);
     if ("error" in payload) {
       message.error({
         content: `DSR automation has failed for this privacy request due to the following: ${getErrorMessage(payload.error!)}`,

clients/admin-ui/src/features/privacy-requests/privacy-requests.slice.ts

@@ -346,6 +346,14 @@ export const privacyRequestApi = baseApi.injectEndpoints({
       }),
       invalidatesTags: ["Request"],
     }),
+    bulkResubmit: build.mutation<BulkPostPrivacyRequests, string[]>({
+      query: (values) => ({
+        url: `privacy-request/bulk/resubmit`,
+        method: "POST",
+        body: values,
+      }),
+      invalidatesTags: ["Request"],
+    }),
     denyRequest: build.mutation<PrivacyRequestEntity, DenyPrivacyRequest>({
       query: ({ id, reason }) => ({
         url: "privacy-request/administrate/deny",
@@ -520,6 +528,16 @@ export const privacyRequestApi = baseApi.injectEndpoints({
       }),
       invalidatesTags: ["Request"],
     }),
+    resubmit: build.mutation<
+      PrivacyRequestEntity,
+      Pick<PrivacyRequestEntity, "id">
+    >({
+      query: ({ id }) => ({
+        url: `privacy-request/${id}/resubmit`,
+        method: "POST",
+      }),
+      invalidatesTags: ["Request"],
+    }),
     saveNotification: build.mutation<any, PrivacyRequestNotificationInfo>({
       query: (params) => ({
         url: `privacy-request/notification`,
@@ -642,6 +660,7 @@ export const {
   useApproveRequestMutation,
   useBulkApproveRequestMutation,
   useBulkDenyRequestMutation,
+  useBulkResubmitMutation,
   useBulkRetryMutation,
   useBulkSoftDeleteRequestMutation,
   useBulkFinalizeRequestMutation,
@@ -651,6 +670,7 @@ export const {
   useSearchPrivacyRequestsQuery,
   usePostPrivacyRequestMutation,
   useGetNotificationQuery,
+  useResubmitMutation,
   useResumePrivacyRequestFromRequiresInputMutation,
   useRetryMutation,
   useSaveNotificationMutation,

src/fides/api/v1/endpoints/privacy_request_endpoints.py

@@ -143,6 +143,7 @@
     PRIVACY_REQUEST_AUTHENTICATED,
     PRIVACY_REQUEST_BATCH_EMAIL_SEND,
     PRIVACY_REQUEST_BULK_FINALIZE,
+    PRIVACY_REQUEST_BULK_RESUBMIT,
     PRIVACY_REQUEST_BULK_RETRY,
     PRIVACY_REQUEST_BULK_SOFT_DELETE,
     PRIVACY_REQUEST_CANCEL,
@@ -2088,6 +2089,95 @@ def get_test_privacy_request_results(
         }
 
 
+@router.post(
+    PRIVACY_REQUEST_BULK_RESUBMIT,
+    status_code=HTTP_200_OK,
+    response_model=BulkPostPrivacyRequests,
+    dependencies=[Security(verify_oauth_client, scopes=[PRIVACY_REQUEST_CREATE])],
+)
+def bulk_resubmit_privacy_requests(
+    privacy_requests: PrivacyRequestBulkSelection,
+    *,
+    db: Session = Depends(deps.get_db),
+    privacy_request_service: PrivacyRequestService = Depends(
+        get_privacy_request_service
+    ),
+) -> BulkPostPrivacyRequests:
+    """
+    Bulk resubmit privacy requests. Each request is deleted and a new privacy
+    request is created in its place, rebuilding the execution DAG from the
+    current integration configuration.
+
+    You can either provide explicit request_ids OR use filters to select privacy requests.
+    When using filters, you can optionally exclude specific IDs via exclude_ids.
+
+    For backwards compatibility, a plain list of request IDs is also accepted.
+    """
+    succeeded: List[PrivacyRequestResponse] = []
+    failed: List[Dict[str, Any]] = []
+
+    try:
+        request_ids = privacy_request_service.resolve_request_ids(privacy_requests)
+    except ValueError as exc:
+        raise HTTPException(status_code=HTTP_400_BAD_REQUEST, detail=str(exc)) from exc
+    batches = privacy_request_service.get_batches_for_bulk_operation(request_ids)
+
+    for batch in batches:
+        privacy_requests_dict = {
+            pr.id: pr
+            for pr in PrivacyRequest.query_without_large_columns(db)
+            .filter(PrivacyRequest.id.in_(batch))
+            .all()
+        }
+
+        for privacy_request_id in batch:
+            privacy_request = privacy_requests_dict.get(privacy_request_id)
+
+            if not privacy_request:
+                failed.append(
+                    {
+                        "message": f"No privacy request found with id '{privacy_request_id}'",
+                        "data": {"privacy_request_id": privacy_request_id},
+                    }
+                )
+                continue
+
+            if privacy_request.deleted_at is not None:
+                failed.append(
+                    {
+                        "message": "Cannot resubmit a deleted privacy request",
+                        "data": {"privacy_request_id": privacy_request_id},
+                    }
+                )
+                continue
+
+            try:
+                resubmitted = privacy_request_service.resubmit_privacy_request(
+                    privacy_request_id
+                )
+            except FidesopsException as exc:
+                failed.append(
+                    {
+                        "message": exc.message,
+                        "data": {"privacy_request_id": privacy_request_id},
+                    }
+                )
+                continue
+
+            if resubmitted is None:
+                failed.append(
+                    {
+                        "message": f"No privacy request found with id '{privacy_request_id}'",
+                        "data": {"privacy_request_id": privacy_request_id},
+                    }
+                )
+                continue
+
+            succeeded.append(resubmitted)  # type: ignore[arg-type]
+
+    return BulkPostPrivacyRequests(succeeded=succeeded, failed=failed)
+
+
 @router.post(
     PRIVACY_REQUEST_RESUBMIT,
     dependencies=[Security(verify_oauth_client, scopes=[PRIVACY_REQUEST_CREATE])],

src/fides/common/urn_registry.py

@@ -87,6 +87,7 @@
 PRIVACY_REQUEST_ATTACHMENT = "/privacy-request/attachment"
 PRIVACY_REQUEST_AUTHENTICATED = "/privacy-request/authenticated"
 PRIVACY_REQUEST_BULK_FINALIZE = "/privacy-request/bulk/finalize"
+PRIVACY_REQUEST_BULK_RESUBMIT = "/privacy-request/bulk/resubmit"
 PRIVACY_REQUEST_BULK_RETRY = "/privacy-request/bulk/retry"
 PRIVACY_REQUEST_BULK_SOFT_DELETE = "/privacy-request/bulk/soft-delete"
 PRIVACY_REQUEST_CANCEL = "/privacy-request/administrate/cancel"

tests/ops/api/v1/endpoints/privacy_request/test_privacy_request_endpoints.py

@@ -104,6 +104,7 @@
     PRIVACY_REQUEST_AUTHENTICATED,
     PRIVACY_REQUEST_BATCH_EMAIL_SEND,
     PRIVACY_REQUEST_BULK_FINALIZE,
+    PRIVACY_REQUEST_BULK_RESUBMIT,
     PRIVACY_REQUEST_BULK_RETRY,
     PRIVACY_REQUEST_BULK_SOFT_DELETE,
     PRIVACY_REQUEST_CANCEL,
@@ -9931,6 +9932,123 @@ def test_resubmit_privacy_request(
         assert response.status_code == HTTP_200_OK
 
 
+class TestBulkResubmitPrivacyRequests:
+    @pytest.fixture(scope="function")
+    def url(self):
+        return f"{V1_URL_PREFIX}{PRIVACY_REQUEST_BULK_RESUBMIT}"
+
+    def test_bulk_resubmit_not_authenticated(self, url, api_client):
+        response = api_client.post(url, json=["1234", "5678"], headers={})
+        assert response.status_code == 401
+
+    def test_bulk_resubmit_wrong_scope(self, url, api_client, generate_auth_header):
+        # PRIVACY_REQUEST_CALLBACK_RESUME is sufficient for /bulk/retry but not /bulk/resubmit
+        auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CALLBACK_RESUME])
+        response = api_client.post(url, json=["1234", "5678"], headers=auth_header)
+        assert response.status_code == 403
+
+    def test_bulk_resubmit_unknown_ids(self, url, api_client, generate_auth_header):
+        auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE])
+        data = ["does-not-exist-1", "does-not-exist-2"]
+
+        response = api_client.post(url, json=data, headers=auth_header)
+        assert response.status_code == 200
+
+        body = response.json()
+        assert body["succeeded"] == []
+        failed_ids = [x["data"]["privacy_request_id"] for x in body["failed"]]
+        assert sorted(failed_ids) == sorted(data)
+
+    def test_bulk_resubmit_deleted_request(
+        self, url, api_client, generate_auth_header, soft_deleted_privacy_request
+    ):
+        auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE])
+        data = [soft_deleted_privacy_request.id]
+
+        response = api_client.post(url, json=data, headers=auth_header)
+        assert response.status_code == 200
+
+        body = response.json()
+        assert body["succeeded"] == []
+        assert body["failed"] == [
+            {
+                "message": "Cannot resubmit a deleted privacy request",
+                "data": {"privacy_request_id": soft_deleted_privacy_request.id},
+            }
+        ]
+
+    def test_bulk_resubmit_completed_request_rejected(
+        self, url, api_client, generate_auth_header, db, privacy_requests
+    ):
+        auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE])
+        privacy_requests[0].status = PrivacyRequestStatus.complete
+        privacy_requests[0].save(db)
+        data = [privacy_requests[0].id]
+
+        response = api_client.post(url, json=data, headers=auth_header)
+        assert response.status_code == 200
+
+        body = response.json()
+        assert body["succeeded"] == []
+        assert len(body["failed"]) == 1
+        assert body["failed"][0]["data"]["privacy_request_id"] == privacy_requests[0].id
+
+    @mock.patch(
+        "fides.api.service.privacy_request.request_runner_service.run_privacy_request.apply_async"
+    )
+    def test_bulk_resubmit_success(
+        self,
+        submit_mock,
+        url,
+        api_client,
+        generate_auth_header,
+        db,
+        privacy_requests,
+    ):
+        auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE])
+        privacy_requests[0].status = PrivacyRequestStatus.error
+        privacy_requests[0].save(db)
+        data = [privacy_requests[0].id]
+
+        response = api_client.post(url, json=data, headers=auth_header)
+        assert response.status_code == 200
+
+        body = response.json()
+        assert body["failed"] == []
+        succeeded_ids = [x["id"] for x in body["succeeded"]]
+        # Resubmit preserves the original id (uses PrivacyRequestResubmit.id)
+        assert privacy_requests[0].id in succeeded_ids
+
+    @mock.patch(
+        "fides.api.service.privacy_request.request_runner_service.run_privacy_request.apply_async"
+    )
+    def test_bulk_resubmit_mixed_result(
+        self,
+        submit_mock,
+        url,
+        api_client,
+        generate_auth_header,
+        db,
+        privacy_requests,
+    ):
+        auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE])
+        privacy_requests[0].status = PrivacyRequestStatus.error
+        privacy_requests[0].save(db)
+
+        bad_id = "bad_test_id"
+        data = [privacy_requests[0].id, bad_id]
+
+        response = api_client.post(url, json=data, headers=auth_header)
+        assert response.status_code == 200
+
+        body = response.json()
+        succeeded_ids = [x["id"] for x in body["succeeded"]]
+        failed_ids = [x["data"]["privacy_request_id"] for x in body["failed"]]
+
+        assert privacy_requests[0].id in succeeded_ids
+        assert bad_id in failed_ids
+
+
 class TestSendBatchEmailIntegrations:
     @pytest.fixture(scope="function")
     def url(self):