diff --git a/changelog/8199-reprocess-dsr-picks-up-integration-changes.yaml b/changelog/8199-reprocess-dsr-picks-up-integration-changes.yaml new file mode 100644 index 00000000000..1be61db3b2a --- /dev/null +++ b/changelog/8199-reprocess-dsr-picks-up-integration-changes.yaml @@ -0,0 +1,4 @@ +type: Changed +description: The privacy request Reprocess action now picks up integrations added or removed after the original failure; the request is now resubmitted instead of retried (DAG rebuilt, prior execution logs cleared) instead of retried against the stale DAG. Adds a new `POST /privacy-request/bulk/resubmit` endpoint to support bulk reprocessing. +pr: 8199 +labels: ["high-risk"] diff --git a/clients/admin-ui/src/features/privacy-requests/buttons/ReprocessButton.tsx b/clients/admin-ui/src/features/privacy-requests/buttons/ReprocessButton.tsx index 41a647ff783..121b358c262 100644 --- a/clients/admin-ui/src/features/privacy-requests/buttons/ReprocessButton.tsx +++ b/clients/admin-ui/src/features/privacy-requests/buttons/ReprocessButton.tsx @@ -7,8 +7,8 @@ import { getErrorMessage, isErrorResult } from "~/features/common/helpers"; import { selectRetryRequests, setRetryRequests, - useBulkRetryMutation, - useRetryMutation, + useBulkResubmitMutation, + useResubmitMutation, } from "../privacy-requests.slice"; import { PrivacyRequestEntity } from "../types"; @@ -24,12 +24,12 @@ const ReprocessButton = forwardRef( const message = useMessage(); const { errorRequests } = useAppSelector(selectRetryRequests); - const [bulkRetry] = useBulkRetryMutation(); - const [retry] = useRetryMutation(); + const [bulkResubmit] = useBulkResubmitMutation(); + const [resubmit] = useResubmitMutation(); const handleBulkReprocessClick = async () => { setIsReprocessing(true); - const payload = await bulkRetry(errorRequests); + const payload = await bulkResubmit(errorRequests); if (isErrorResult(payload)) { dispatch(setRetryRequests({ checkAll: false, errorRequests: [] })); message.error({ @@ -64,7 +64,7 @@ const ReprocessButton = forwardRef( return; } setIsReprocessing(true); - const payload = await retry(subjectRequest); + const payload = await resubmit(subjectRequest); if (isErrorResult(payload)) { message.error({ content: `DSR automation has failed for this privacy request due to the following: ${getErrorMessage(payload.error)}`, diff --git a/clients/admin-ui/src/features/privacy-requests/hooks/useReprocessPrivacyRequest.ts b/clients/admin-ui/src/features/privacy-requests/hooks/useReprocessPrivacyRequest.ts index 7b0f7f92ee4..4bf936e4666 100644 --- a/clients/admin-ui/src/features/privacy-requests/hooks/useReprocessPrivacyRequest.ts +++ b/clients/admin-ui/src/features/privacy-requests/hooks/useReprocessPrivacyRequest.ts @@ -3,7 +3,7 @@ import { useMessage } from "fidesui"; import { getErrorMessage } from "~/features/common/helpers"; import { PrivacyRequestStatus } from "~/types/api"; -import { useRetryMutation } from "../privacy-requests.slice"; +import { useResubmitMutation } from "../privacy-requests.slice"; import { PrivacyRequestEntity } from "../types"; const useReprocessPrivacyRequest = ({ @@ -15,14 +15,14 @@ const useReprocessPrivacyRequest = ({ const isErrorStatus = privacyRequest.status === PrivacyRequestStatus.ERROR; const showReprocess = isErrorStatus; - const [retry] = useRetryMutation(); + const [resubmit] = useResubmitMutation(); const reprocessPrivacyRequest = async () => { if (!privacyRequest) { return; } const closeMessage = message.loading("Reprocessing privacy request...", 0); - const payload = await retry(privacyRequest); + const payload = await resubmit(privacyRequest); if ("error" in payload) { message.error({ content: `DSR automation has failed for this privacy request due to the following: ${getErrorMessage(payload.error!)}`, diff --git a/clients/admin-ui/src/features/privacy-requests/privacy-requests.slice.ts b/clients/admin-ui/src/features/privacy-requests/privacy-requests.slice.ts index fb1a7779e8c..10dd8ba7d2a 100644 --- a/clients/admin-ui/src/features/privacy-requests/privacy-requests.slice.ts +++ b/clients/admin-ui/src/features/privacy-requests/privacy-requests.slice.ts @@ -346,6 +346,14 @@ export const privacyRequestApi = baseApi.injectEndpoints({ }), invalidatesTags: ["Request"], }), + bulkResubmit: build.mutation({ + query: (values) => ({ + url: `privacy-request/bulk/resubmit`, + method: "POST", + body: values, + }), + invalidatesTags: ["Request"], + }), denyRequest: build.mutation({ query: ({ id, reason }) => ({ url: "privacy-request/administrate/deny", @@ -520,6 +528,16 @@ export const privacyRequestApi = baseApi.injectEndpoints({ }), invalidatesTags: ["Request"], }), + resubmit: build.mutation< + PrivacyRequestEntity, + Pick + >({ + query: ({ id }) => ({ + url: `privacy-request/${id}/resubmit`, + method: "POST", + }), + invalidatesTags: ["Request"], + }), saveNotification: build.mutation({ query: (params) => ({ url: `privacy-request/notification`, @@ -642,6 +660,7 @@ export const { useApproveRequestMutation, useBulkApproveRequestMutation, useBulkDenyRequestMutation, + useBulkResubmitMutation, useBulkRetryMutation, useBulkSoftDeleteRequestMutation, useBulkFinalizeRequestMutation, @@ -651,6 +670,7 @@ export const { useSearchPrivacyRequestsQuery, usePostPrivacyRequestMutation, useGetNotificationQuery, + useResubmitMutation, useResumePrivacyRequestFromRequiresInputMutation, useRetryMutation, useSaveNotificationMutation, diff --git a/src/fides/api/v1/endpoints/privacy_request_endpoints.py b/src/fides/api/v1/endpoints/privacy_request_endpoints.py index 3d8eb062b8d..08041b50415 100644 --- a/src/fides/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fides/api/v1/endpoints/privacy_request_endpoints.py @@ -143,6 +143,7 @@ PRIVACY_REQUEST_AUTHENTICATED, PRIVACY_REQUEST_BATCH_EMAIL_SEND, PRIVACY_REQUEST_BULK_FINALIZE, + PRIVACY_REQUEST_BULK_RESUBMIT, PRIVACY_REQUEST_BULK_RETRY, PRIVACY_REQUEST_BULK_SOFT_DELETE, PRIVACY_REQUEST_CANCEL, @@ -2088,6 +2089,95 @@ def get_test_privacy_request_results( } +@router.post( + PRIVACY_REQUEST_BULK_RESUBMIT, + status_code=HTTP_200_OK, + response_model=BulkPostPrivacyRequests, + dependencies=[Security(verify_oauth_client, scopes=[PRIVACY_REQUEST_CREATE])], +) +def bulk_resubmit_privacy_requests( + privacy_requests: PrivacyRequestBulkSelection, + *, + db: Session = Depends(deps.get_db), + privacy_request_service: PrivacyRequestService = Depends( + get_privacy_request_service + ), +) -> BulkPostPrivacyRequests: + """ + Bulk resubmit privacy requests. Each request is deleted and a new privacy + request is created in its place, rebuilding the execution DAG from the + current integration configuration. + + You can either provide explicit request_ids OR use filters to select privacy requests. + When using filters, you can optionally exclude specific IDs via exclude_ids. + + For backwards compatibility, a plain list of request IDs is also accepted. + """ + succeeded: List[PrivacyRequestResponse] = [] + failed: List[Dict[str, Any]] = [] + + try: + request_ids = privacy_request_service.resolve_request_ids(privacy_requests) + except ValueError as exc: + raise HTTPException(status_code=HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + batches = privacy_request_service.get_batches_for_bulk_operation(request_ids) + + for batch in batches: + privacy_requests_dict = { + pr.id: pr + for pr in PrivacyRequest.query_without_large_columns(db) + .filter(PrivacyRequest.id.in_(batch)) + .all() + } + + for privacy_request_id in batch: + privacy_request = privacy_requests_dict.get(privacy_request_id) + + if not privacy_request: + failed.append( + { + "message": f"No privacy request found with id '{privacy_request_id}'", + "data": {"privacy_request_id": privacy_request_id}, + } + ) + continue + + if privacy_request.deleted_at is not None: + failed.append( + { + "message": "Cannot resubmit a deleted privacy request", + "data": {"privacy_request_id": privacy_request_id}, + } + ) + continue + + try: + resubmitted = privacy_request_service.resubmit_privacy_request( + privacy_request_id + ) + except FidesopsException as exc: + failed.append( + { + "message": exc.message, + "data": {"privacy_request_id": privacy_request_id}, + } + ) + continue + + if resubmitted is None: + failed.append( + { + "message": f"No privacy request found with id '{privacy_request_id}'", + "data": {"privacy_request_id": privacy_request_id}, + } + ) + continue + + succeeded.append(resubmitted) # type: ignore[arg-type] + + return BulkPostPrivacyRequests(succeeded=succeeded, failed=failed) + + @router.post( PRIVACY_REQUEST_RESUBMIT, dependencies=[Security(verify_oauth_client, scopes=[PRIVACY_REQUEST_CREATE])], diff --git a/src/fides/common/urn_registry.py b/src/fides/common/urn_registry.py index fb51670acca..9e0417b754f 100644 --- a/src/fides/common/urn_registry.py +++ b/src/fides/common/urn_registry.py @@ -87,6 +87,7 @@ PRIVACY_REQUEST_ATTACHMENT = "/privacy-request/attachment" PRIVACY_REQUEST_AUTHENTICATED = "/privacy-request/authenticated" PRIVACY_REQUEST_BULK_FINALIZE = "/privacy-request/bulk/finalize" +PRIVACY_REQUEST_BULK_RESUBMIT = "/privacy-request/bulk/resubmit" PRIVACY_REQUEST_BULK_RETRY = "/privacy-request/bulk/retry" PRIVACY_REQUEST_BULK_SOFT_DELETE = "/privacy-request/bulk/soft-delete" PRIVACY_REQUEST_CANCEL = "/privacy-request/administrate/cancel" diff --git a/tests/ops/api/v1/endpoints/privacy_request/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/privacy_request/test_privacy_request_endpoints.py index 5fa3960de8e..c08596e0a5d 100644 --- a/tests/ops/api/v1/endpoints/privacy_request/test_privacy_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/privacy_request/test_privacy_request_endpoints.py @@ -104,6 +104,7 @@ PRIVACY_REQUEST_AUTHENTICATED, PRIVACY_REQUEST_BATCH_EMAIL_SEND, PRIVACY_REQUEST_BULK_FINALIZE, + PRIVACY_REQUEST_BULK_RESUBMIT, PRIVACY_REQUEST_BULK_RETRY, PRIVACY_REQUEST_BULK_SOFT_DELETE, PRIVACY_REQUEST_CANCEL, @@ -9931,6 +9932,123 @@ def test_resubmit_privacy_request( assert response.status_code == HTTP_200_OK +class TestBulkResubmitPrivacyRequests: + @pytest.fixture(scope="function") + def url(self): + return f"{V1_URL_PREFIX}{PRIVACY_REQUEST_BULK_RESUBMIT}" + + def test_bulk_resubmit_not_authenticated(self, url, api_client): + response = api_client.post(url, json=["1234", "5678"], headers={}) + assert response.status_code == 401 + + def test_bulk_resubmit_wrong_scope(self, url, api_client, generate_auth_header): + # PRIVACY_REQUEST_CALLBACK_RESUME is sufficient for /bulk/retry but not /bulk/resubmit + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CALLBACK_RESUME]) + response = api_client.post(url, json=["1234", "5678"], headers=auth_header) + assert response.status_code == 403 + + def test_bulk_resubmit_unknown_ids(self, url, api_client, generate_auth_header): + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE]) + data = ["does-not-exist-1", "does-not-exist-2"] + + response = api_client.post(url, json=data, headers=auth_header) + assert response.status_code == 200 + + body = response.json() + assert body["succeeded"] == [] + failed_ids = [x["data"]["privacy_request_id"] for x in body["failed"]] + assert sorted(failed_ids) == sorted(data) + + def test_bulk_resubmit_deleted_request( + self, url, api_client, generate_auth_header, soft_deleted_privacy_request + ): + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE]) + data = [soft_deleted_privacy_request.id] + + response = api_client.post(url, json=data, headers=auth_header) + assert response.status_code == 200 + + body = response.json() + assert body["succeeded"] == [] + assert body["failed"] == [ + { + "message": "Cannot resubmit a deleted privacy request", + "data": {"privacy_request_id": soft_deleted_privacy_request.id}, + } + ] + + def test_bulk_resubmit_completed_request_rejected( + self, url, api_client, generate_auth_header, db, privacy_requests + ): + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE]) + privacy_requests[0].status = PrivacyRequestStatus.complete + privacy_requests[0].save(db) + data = [privacy_requests[0].id] + + response = api_client.post(url, json=data, headers=auth_header) + assert response.status_code == 200 + + body = response.json() + assert body["succeeded"] == [] + assert len(body["failed"]) == 1 + assert body["failed"][0]["data"]["privacy_request_id"] == privacy_requests[0].id + + @mock.patch( + "fides.api.service.privacy_request.request_runner_service.run_privacy_request.apply_async" + ) + def test_bulk_resubmit_success( + self, + submit_mock, + url, + api_client, + generate_auth_header, + db, + privacy_requests, + ): + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE]) + privacy_requests[0].status = PrivacyRequestStatus.error + privacy_requests[0].save(db) + data = [privacy_requests[0].id] + + response = api_client.post(url, json=data, headers=auth_header) + assert response.status_code == 200 + + body = response.json() + assert body["failed"] == [] + succeeded_ids = [x["id"] for x in body["succeeded"]] + # Resubmit preserves the original id (uses PrivacyRequestResubmit.id) + assert privacy_requests[0].id in succeeded_ids + + @mock.patch( + "fides.api.service.privacy_request.request_runner_service.run_privacy_request.apply_async" + ) + def test_bulk_resubmit_mixed_result( + self, + submit_mock, + url, + api_client, + generate_auth_header, + db, + privacy_requests, + ): + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE]) + privacy_requests[0].status = PrivacyRequestStatus.error + privacy_requests[0].save(db) + + bad_id = "bad_test_id" + data = [privacy_requests[0].id, bad_id] + + response = api_client.post(url, json=data, headers=auth_header) + assert response.status_code == 200 + + body = response.json() + succeeded_ids = [x["id"] for x in body["succeeded"]] + failed_ids = [x["data"]["privacy_request_id"] for x in body["failed"]] + + assert privacy_requests[0].id in succeeded_ids + assert bad_id in failed_ids + + class TestSendBatchEmailIntegrations: @pytest.fixture(scope="function") def url(self):