[Test] Embedding Circuit 장애·Probe 복구 경로 통합 검증

backend/src/main/java/com/opensource/docgrid/domain/embedding/client/EmbeddingClient.java

@@ -8,6 +8,7 @@
 
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.http.HttpHeaders;
+import org.springframework.http.MediaType;
 import org.springframework.stereotype.Component;
 import org.springframework.util.StringUtils;
 import org.springframework.web.client.ResourceAccessException;
@@ -68,6 +69,8 @@ public float[] embed(String text) {
         try {
             response = queryRestClient.post()
                 .uri("/embed")
+                // JSON을 명시해 XML Converter가 Provider 요청 DTO를 먼저 선택하지 못하게 한다.
+                .contentType(MediaType.APPLICATION_JSON)
                 .body(new EmbedRequest(text))
                 .retrieve()
                 .body(EmbedServerResponse.class);
@@ -107,6 +110,8 @@ public EmbedBatchServerResponse embedBatch(List<String> texts, int batchSize) {
         try {
             response = documentRestClient.post()
                 .uri("/embed/batch")
+                // FastAPI Batch 계약은 JSON 객체이므로 classpath의 XML 지원 여부와 무관하게 고정한다.
+                .contentType(MediaType.APPLICATION_JSON)
                 .body(new EmbedBatchRequest(texts, batchSize))
                 .retrieve()
                 .body(EmbedBatchServerResponse.class);

backend/src/test/java/com/opensource/docgrid/domain/embedding/client/EmbeddingClientTest.java

@@ -13,6 +13,7 @@
 import java.util.List;
 
 import org.springframework.http.HttpHeaders;
+import org.springframework.http.MediaType;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
@@ -78,6 +79,7 @@ void embed_success() {
 
         assertThat(result).containsExactly(vector);
         verify(queryRestClient).post();
+        verify(requestBodyUriSpec).contentType(MediaType.APPLICATION_JSON);
         verify(documentRestClient, never()).post();
         verify(circuitBreaker).recordSuccess(permission);
     }
@@ -247,6 +249,7 @@ void embedBatch_returnsValidatedResponse() {
             .extracting(EmbedBatchItemResponse::index)
             .containsExactly(0, 1);
         verify(documentRestClient).post();
+        verify(requestBodyUriSpec).contentType(MediaType.APPLICATION_JSON);
         verify(queryRestClient, never()).post();
     }
 

docs/test-results/evidence/issue-413/circuit-recovery-webhooks.jsonl

@@ -0,0 +1,2 @@
+{"receivedAt":"2026-10-03T17:05:47.781605+00:00","payload":{"receiver":"local-webhook","status":"firing","alerts":[{"status":"firing","labels":{"alertname":"EmbeddingProviderDown","cluster":"docgrid-drill","environment":"local-drill","instance":"embedding-server:8000","job":"embedding-provider","service":"embedding-provider","severity":"critical"},"annotations":{"description":"Prometheus has failed to scrape the Embedding Provider for one minute.","summary":"Embedding Provider metric endpoint is unavailable"},"startsAt":"2026-10-03T17:05:37.76Z","endsAt":"0001-01-01T00:00:00Z","generatorURL":"[internal URL omitted]","fingerprint":"dcc4408eec1a5b0a"}],"notification_reason":"first notification","groupLabels":{"alertname":"EmbeddingProviderDown","cluster":"docgrid-drill","environment":"local-drill","service":"embedding-provider","severity":"critical"},"commonLabels":{"alertname":"EmbeddingProviderDown","cluster":"docgrid-drill","environment":"local-drill","instance":"embedding-server:8000","job":"embedding-provider","service":"embedding-provider","severity":"critical"},"commonAnnotations":{"description":"Prometheus has failed to scrape the Embedding Provider for one minute.","summary":"Embedding Provider metric endpoint is unavailable"},"externalURL":"[internal URL omitted]","version":"4","groupKey":"{}/{severity=\"critical\"}:{alertname=\"EmbeddingProviderDown\", cluster=\"docgrid-drill\", environment=\"local-drill\", service=\"embedding-provider\", severity=\"critical\"}","truncatedAlerts":0}}
+{"receivedAt":"2026-10-03T17:08:17.793669+00:00","payload":{"receiver":"local-webhook","status":"resolved","alerts":[{"status":"resolved","labels":{"alertname":"EmbeddingProviderDown","cluster":"docgrid-drill","environment":"local-drill","instance":"embedding-server:8000","job":"embedding-provider","service":"embedding-provider","severity":"critical"},"annotations":{"description":"Prometheus has failed to scrape the Embedding Provider for one minute.","summary":"Embedding Provider metric endpoint is unavailable"},"startsAt":"2026-10-03T17:05:37.76Z","endsAt":"2026-10-03T17:07:52.76Z","generatorURL":"[internal URL omitted]","fingerprint":"dcc4408eec1a5b0a"}],"notification_reason":"all alerts resolved","groupLabels":{"alertname":"EmbeddingProviderDown","cluster":"docgrid-drill","environment":"local-drill","service":"embedding-provider","severity":"critical"},"commonLabels":{"alertname":"EmbeddingProviderDown","cluster":"docgrid-drill","environment":"local-drill","instance":"embedding-server:8000","job":"embedding-provider","service":"embedding-provider","severity":"critical"},"commonAnnotations":{"description":"Prometheus has failed to scrape the Embedding Provider for one minute.","summary":"Embedding Provider metric endpoint is unavailable"},"externalURL":"[internal URL omitted]","version":"4","groupKey":"{}/{severity=\"critical\"}:{alertname=\"EmbeddingProviderDown\", cluster=\"docgrid-drill\", environment=\"local-drill\", service=\"embedding-provider\", severity=\"critical\"}","truncatedAlerts":0}}

docs/test-results/evidence/issue-413/circuit-recovery.json

@@ -0,0 +1,117 @@
+{
+  "scenario": "circuit-recovery",
+  "status": "PASS",
+  "measuredAt": "2026-10-03T17:08:18.145282+00:00",
+  "gitCommit": "310965b929cff4e682d029ff1bb836379c1fe3ae",
+  "host": {
+    "system": "Darwin",
+    "machine": "arm64"
+  },
+  "images": {
+    "postgres": "pgvector/pgvector:0.8.1-pg17",
+    "prometheus": "prom/prometheus:v3.5.5",
+    "alertmanager": "quay.io/prometheus/alertmanager:v0.33.1"
+  },
+  "conditions": {
+    "providerFailureMode": "docker-stop",
+    "circuitFailureThreshold": 3,
+    "circuitOpenDuration": "30s",
+    "retryJitterRatio": 0,
+    "jobMaxRetryCount": 10,
+    "failedProbeRetryFreeze": "10m",
+    "scrapeInterval": "15s",
+    "evaluationInterval": "15s",
+    "circuitRuleFor": "1m",
+    "criticalGroupWait": "10s",
+    "warningGroupWait": "30s",
+    "testGroupInterval": "30s",
+    "productionGroupInterval": "5m"
+  },
+  "timestamps": {
+    "providerStoppedAt": "2026-10-03T17:04:13.035922+00:00",
+    "providerDownConditionAt": "2026-10-03T17:04:32.110422+00:00",
+    "circuitOpenedAt": "2026-10-03T17:04:51.405977+00:00",
+    "delayedRetryObservedAt": "2026-10-03T17:04:51.628918+00:00",
+    "failedProbeAt": "2026-10-03T17:05:31.612993+00:00",
+    "providerDownFiringAt": "2026-10-03T17:05:39.643678+00:00",
+    "providerDownWebhookAt": "2026-10-03T17:05:47.781605+00:00",
+    "circuitAlertPendingAt": "2026-10-03T17:05:48.687839+00:00",
+    "circuitAlertFiringAt": "2026-10-03T17:06:46.911840+00:00",
+    "circuitSuppressedAt": "2026-10-03T17:06:46.914042+00:00",
+    "recoveryStartedAt": "2026-10-03T17:07:22.044228+00:00",
+    "providerReadyAt": "2026-10-03T17:07:26.326082+00:00",
+    "successfulProbeAt": "2026-10-03T17:07:30.473955+00:00",
+    "jobIndexedAt": "2026-10-03T17:07:30.553405+00:00",
+    "alertsResolvedAt": "2026-10-03T17:07:52.763847+00:00",
+    "providerDownResolvedWebhookAt": "2026-10-03T17:08:17.793669+00:00"
+  },
+  "measurements": {
+    "stopToProviderDownConditionSeconds": 19.075,
+    "stopToCircuitOpenSeconds": 38.37,
+    "stopToFailedProbeSeconds": 78.577,
+    "failedProbeToCircuitFiringSeconds": 75.299,
+    "recoveryToProviderReadySeconds": 4.282,
+    "providerReadyToSuccessfulProbeSeconds": 4.148,
+    "recoveryToIndexedSeconds": 8.509,
+    "recoveryToAllPrometheusAlertsResolvedSeconds": 30.72,
+    "recoveryToProviderResolvedWebhookSeconds": 55.749,
+    "circuitWarningSuppressed": true,
+    "circuitWarningFiringWebhookDeliveries": 0,
+    "storedEmbeddings": 1,
+    "finalJobStatus": "INDEXED",
+    "finalRetryCount": 5,
+    "maxRetryCount": 10
+  },
+  "metricSnapshots": {
+    "circuitOpen": {
+      "open": 1.0,
+      "probeFailed": 0.0,
+      "openTransitions": 1.0,
+      "halfOpenTransitions": 0.0,
+      "closedTransitions": 0.0,
+      "failedProbes": 0.0,
+      "successfulProbes": 0.0,
+      "delayedRetryJobs": 1.0
+    },
+    "delayedRetry": {
+      "open": 1.0,
+      "probeFailed": 0.0,
+      "openTransitions": 1.0,
+      "halfOpenTransitions": 0.0,
+      "closedTransitions": 0.0,
+      "failedProbes": 0.0,
+      "successfulProbes": 0.0,
+      "delayedRetryJobs": 1.0
+    },
+    "failedProbe": {
+      "open": 1.0,
+      "probeFailed": 1.0,
+      "openTransitions": 2.0,
+      "halfOpenTransitions": 1.0,
+      "closedTransitions": 0.0,
+      "failedProbes": 1.0,
+      "successfulProbes": 0.0,
+      "delayedRetryJobs": 1.0
+    },
+    "successfulProbe": {
+      "open": 0.0,
+      "probeFailed": 0.0,
+      "openTransitions": 2.0,
+      "halfOpenTransitions": 2.0,
+      "closedTransitions": 1.0,
+      "failedProbes": 1.0,
+      "successfulProbes": 1.0,
+      "delayedRetryJobs": 0.0
+    },
+    "recovered": {
+      "open": 0.0,
+      "probeFailed": 0.0,
+      "openTransitions": 2.0,
+      "halfOpenTransitions": 2.0,
+      "closedTransitions": 1.0,
+      "failedProbes": 1.0,
+      "successfulProbes": 1.0,
+      "delayedRetryJobs": 0.0
+    }
+  }
+}