diff --git a/backend/src/main/java/com/opensource/docgrid/domain/embedding/client/EmbeddingProviderCircuitBreaker.java b/backend/src/main/java/com/opensource/docgrid/domain/embedding/client/EmbeddingProviderCircuitBreaker.java index cb80dd06..7428338b 100644 --- a/backend/src/main/java/com/opensource/docgrid/domain/embedding/client/EmbeddingProviderCircuitBreaker.java +++ b/backend/src/main/java/com/opensource/docgrid/domain/embedding/client/EmbeddingProviderCircuitBreaker.java @@ -16,7 +16,7 @@ * *

HTTP 실행 전 Permission만 발급하고 실제 요청·응답 계약에는 관여하지 않는다. 모든 상태 변경은 * synchronized 경계 안에서 직렬화하며, Open 전 이미 시작된 요청의 늦은 결과가 새 상태를 덮어쓰지 - * 못하도록 세대 번호를 함께 검증한다. + * 못하도록 세대 번호를 함께 검증한다. 확정된 상태 전이는 같은 경계 안에서 운영 Metric에 반영한다. */ @Slf4j @Component @@ -26,6 +26,7 @@ public class EmbeddingProviderCircuitBreaker { private final EmbeddingProviderCircuitBreakerProperties properties; private final Clock clock; + private final EmbeddingProviderCircuitMetrics metrics; private CircuitState state = CircuitState.CLOSED; private int consecutiveFailures; @@ -34,14 +35,16 @@ public class EmbeddingProviderCircuitBreaker { private boolean halfOpenProbeInProgress; /** - * Circuit 임계값·개방 시간 설정과 테스트 가능한 기준 Clock을 연결한다. + * Circuit 임계값·개방 시간 설정, 기준 Clock과 상태 전이 Metric을 연결한다. */ public EmbeddingProviderCircuitBreaker( EmbeddingProviderCircuitBreakerProperties properties, - Clock clock + Clock clock, + EmbeddingProviderCircuitMetrics metrics ) { this.properties = properties; this.clock = clock; + this.metrics = metrics; } /** @@ -61,6 +64,7 @@ public synchronized CallPermission acquirePermission() { } state = CircuitState.HALF_OPEN; halfOpenProbeInProgress = false; + metrics.recordHalfOpen(); } // 3. HALF_OPEN에서는 한 호출에만 Probe 소유권을 주고 나머지는 빠르게 거절한다. @@ -84,6 +88,8 @@ public synchronized void recordSuccess(CallPermission permission) { return; } if (permission.halfOpenProbe() && state == CircuitState.HALF_OPEN) { + // Circuit 닫힘과 함께 회복을 확인한 Probe 결과를 한 번만 기록한다. + metrics.recordProbeSuccess(); closeCircuit(); return; } @@ -122,6 +128,8 @@ public synchronized Duration recordFailure( return Duration.ZERO; } if (permission.halfOpenProbe() && state == CircuitState.HALF_OPEN) { + // 최초 OPEN과 구분해 장애 지속을 입증하는 Probe 실패를 별도로 기록한다. + metrics.recordProbeFailure(); return openCircuit(); } if (state != CircuitState.CLOSED) { @@ -146,6 +154,7 @@ private Duration openCircuit() { consecutiveFailures = 0; halfOpenProbeInProgress = false; generation++; + metrics.recordOpen(); log.warn( "Embedding Provider Circuit을 열었습니다. openDurationMs={}", properties.getOpenDuration().toMillis() @@ -162,6 +171,7 @@ private void closeCircuit() { consecutiveFailures = 0; halfOpenProbeInProgress = false; generation++; + metrics.recordClosed(); log.info("Embedding Provider Circuit이 정상 호출로 닫혔습니다."); } diff --git a/backend/src/main/java/com/opensource/docgrid/domain/embedding/client/EmbeddingProviderCircuitMetrics.java b/backend/src/main/java/com/opensource/docgrid/domain/embedding/client/EmbeddingProviderCircuitMetrics.java new file mode 100644 index 00000000..aa93fcb2 --- /dev/null +++ b/backend/src/main/java/com/opensource/docgrid/domain/embedding/client/EmbeddingProviderCircuitMetrics.java @@ -0,0 +1,96 @@ +package com.opensource.docgrid.domain.embedding.client; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.springframework.stereotype.Component; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; + +/** + * Embedding Provider Circuit의 현재 보호 활성 여부, 상태 전환과 Probe 결과를 Micrometer에 + * 기록한다. + * + *

Circuit 상태 결정은 하지 않고 {@link EmbeddingProviderCircuitBreaker}가 확정한 전이만 반영한다. + * 상태 label 세 값과 Probe 결과 label 두 값만 사용해 시계열 Cardinality가 입력 데이터에 따라 + * 증가하지 않게 한다. + */ +@Component +public class EmbeddingProviderCircuitMetrics { + + private static final String TRANSITIONS = "docgrid.embedding.provider.circuit.transitions"; + private static final String PROBES = "docgrid.embedding.provider.circuit.probe"; + + private final AtomicInteger open = new AtomicInteger(); + private final AtomicInteger probeFailed = new AtomicInteger(); + private final Counter openTransitions; + private final Counter halfOpenTransitions; + private final Counter closedTransitions; + private final Counter successfulProbes; + private final Counter failedProbes; + + /** Circuit Gauge와 상태 전환·Probe 결과 Counter를 애플리케이션 시작 시 한 번 등록한다. */ + public EmbeddingProviderCircuitMetrics(MeterRegistry meterRegistry) { + Gauge.builder("docgrid.embedding.provider.circuit.open", open, AtomicInteger::get) + .description("Whether the Embedding Provider circuit is open or half-open") + .register(meterRegistry); + Gauge.builder( + "docgrid.embedding.provider.circuit.probe.failed", + probeFailed, + AtomicInteger::get + ) + .description("Whether the current circuit protection episode has a failed probe") + .register(meterRegistry); + openTransitions = transitionCounter(meterRegistry, "open"); + halfOpenTransitions = transitionCounter(meterRegistry, "half_open"); + closedTransitions = transitionCounter(meterRegistry, "closed"); + // Counter를 0에서 미리 등록해 첫 Probe 결과도 Prometheus increase()가 관측할 수 있게 한다. + successfulProbes = probeCounter(meterRegistry, "success"); + failedProbes = probeCounter(meterRegistry, "failed"); + } + + /** Circuit OPEN 전이를 기록하고 현재 차단 상태를 게시한다. */ + void recordOpen() { + open.set(1); + openTransitions.increment(); + } + + /** Open 유예 종료 뒤 단일 Probe를 허용하는 HALF_OPEN 전이를 기록한다. */ + void recordHalfOpen() { + // Probe 성공 전까지 일반 호출은 계속 차단되므로 장기 장애 경보를 끊지 않는다. + halfOpenTransitions.increment(); + } + + /** Probe 성공으로 정상 호출을 재개한 CLOSED 전이를 기록한다. */ + void recordClosed() { + open.set(0); + closedTransitions.increment(); + } + + /** Half-open Probe 성공 횟수를 기록하고 현재 보호 주기의 실패 표시를 해제한다. */ + void recordProbeSuccess() { + successfulProbes.increment(); + probeFailed.set(0); + } + + /** Half-open Probe 실패 횟수를 기록하고 현재 보호 주기에 확인된 장애를 표시한다. */ + void recordProbeFailure() { + failedProbes.increment(); + probeFailed.set(1); + } + + private Counter transitionCounter(MeterRegistry meterRegistry, String state) { + return Counter.builder(TRANSITIONS) + .description("Embedding Provider circuit transitions by destination state") + .tag("state", state) + .register(meterRegistry); + } + + private Counter probeCounter(MeterRegistry meterRegistry, String outcome) { + return Counter.builder(PROBES) + .description("Embedding Provider half-open probe results by outcome") + .tag("outcome", outcome) + .register(meterRegistry); + } +} diff --git a/backend/src/main/java/com/opensource/docgrid/global/observability/OperationalMetrics.java b/backend/src/main/java/com/opensource/docgrid/global/observability/OperationalMetrics.java index c614f2a1..2068b2e5 100644 --- a/backend/src/main/java/com/opensource/docgrid/global/observability/OperationalMetrics.java +++ b/backend/src/main/java/com/opensource/docgrid/global/observability/OperationalMetrics.java @@ -33,6 +33,9 @@ public OperationalMetrics(MeterRegistry meterRegistry, Clock clock) { registerGauge(meterRegistry, "docgrid.embedding.claimable.jobs", "Embedding jobs that can be claimed now", snapshot -> snapshot.embeddingClaimableJobs()); + registerGauge(meterRegistry, "docgrid.embedding.delayed.retry.jobs", + "Embedding retry jobs waiting for their next retry time", + snapshot -> snapshot.embeddingDelayedRetryJobs()); registerGauge(meterRegistry, "docgrid.embedding.processing.jobs", "Embedding jobs currently processing", snapshot -> snapshot.embeddingProcessingJobs()); diff --git a/backend/src/main/java/com/opensource/docgrid/global/observability/OperationalMetricsSnapshot.java b/backend/src/main/java/com/opensource/docgrid/global/observability/OperationalMetricsSnapshot.java index 34ddec8e..c4336330 100644 --- a/backend/src/main/java/com/opensource/docgrid/global/observability/OperationalMetricsSnapshot.java +++ b/backend/src/main/java/com/opensource/docgrid/global/observability/OperationalMetricsSnapshot.java @@ -6,11 +6,12 @@ /** * 한 시점에 읽은 비동기 Queue와 Worker의 운영 상태를 불변 값으로 보관한다. * - *

DB 집계 결과와 Prometheus scrape 사이의 경계다. 오래된 항목이 없으면 해당 시각은 {@code null}이며, - * 모든 개수는 음수가 될 수 없다. + *

DB 집계 결과와 Prometheus scrape 사이의 경계다. Embedding의 즉시 실행 가능 항목과 미래 재시도 + * 항목을 분리하며, 오래된 항목이 없으면 해당 시각은 {@code null}이다. 모든 개수는 음수가 될 수 없다. */ public record OperationalMetricsSnapshot( long embeddingClaimableJobs, + long embeddingDelayedRetryJobs, long embeddingProcessingJobs, LocalDateTime embeddingOldestClaimableAt, long embeddingActiveWorkers, @@ -23,7 +24,8 @@ public record OperationalMetricsSnapshot( /** 집계 Query의 잘못된 결과가 Gauge에 게시되지 않도록 불변 조건을 확인한다. */ public OperationalMetricsSnapshot { - if (embeddingClaimableJobs < 0 || embeddingProcessingJobs < 0 || embeddingActiveWorkers < 0 + if (embeddingClaimableJobs < 0 || embeddingDelayedRetryJobs < 0 + || embeddingProcessingJobs < 0 || embeddingActiveWorkers < 0 || ragProcessingJobs < 0 || syncClaimableEvents < 0 || syncProcessingEvents < 0) { throw new IllegalArgumentException("운영 상태 개수는 음수일 수 없습니다."); } diff --git a/backend/src/main/java/com/opensource/docgrid/global/observability/OperationalMetricsSnapshotQueryService.java b/backend/src/main/java/com/opensource/docgrid/global/observability/OperationalMetricsSnapshotQueryService.java index c2dc3fc9..a3865c84 100644 --- a/backend/src/main/java/com/opensource/docgrid/global/observability/OperationalMetricsSnapshotQueryService.java +++ b/backend/src/main/java/com/opensource/docgrid/global/observability/OperationalMetricsSnapshotQueryService.java @@ -13,7 +13,7 @@ import lombok.RequiredArgsConstructor; /** - * 비동기 Pipeline의 현재 운영 상태를 행 로딩 없이 목적별 aggregate query로 읽는다. + * 비동기 Pipeline의 즉시 실행 가능 상태와 미래 재시도 대기 상태를 행 로딩 없이 aggregate query로 읽는다. * *

호출자는 이 짧은 읽기 Transaction의 결과만 메모리에 보관한다. Prometheus scrape 경로에서는 이 * 서비스를 호출하지 않아, 감시 트래픽이 DB 부하로 이어지지 않게 한다. @@ -31,6 +31,12 @@ SELECT COUNT(*) WHERE job.status = 'PENDING' AND (job.next_retry_at IS NULL OR job.next_retry_at <= ?) ) AS claimable_jobs, + ( + SELECT COUNT(*) + FROM embedding_jobs job + WHERE job.status = 'PENDING' + AND job.next_retry_at > ? + ) AS delayed_retry_jobs, ( SELECT COUNT(*) FROM embedding_jobs job @@ -103,6 +109,7 @@ public OperationalMetricsSnapshot load(LocalDateTime observedAt) { this::mapEmbeddingSnapshot, observedAt, observedAt, + observedAt, heartbeatDeadline ); RagSnapshot rag = jdbcTemplate.queryForObject(RAG_SNAPSHOT_SQL, this::mapRagSnapshot); @@ -116,6 +123,7 @@ public OperationalMetricsSnapshot load(LocalDateTime observedAt) { // 3. 세 Query가 모두 성공한 경우에만 하나의 원자적 Snapshot 후보를 반환한다. return new OperationalMetricsSnapshot( embedding.claimableJobs(), + embedding.delayedRetryJobs(), embedding.processingJobs(), embedding.oldestClaimableAt(), embedding.activeWorkers(), @@ -130,6 +138,7 @@ public OperationalMetricsSnapshot load(LocalDateTime observedAt) { private EmbeddingSnapshot mapEmbeddingSnapshot(ResultSet resultSet, int rowNumber) throws SQLException { return new EmbeddingSnapshot( resultSet.getLong("claimable_jobs"), + resultSet.getLong("delayed_retry_jobs"), resultSet.getLong("processing_jobs"), resultSet.getObject("oldest_claimable_at", LocalDateTime.class), resultSet.getLong("active_workers") @@ -151,9 +160,10 @@ private SyncSnapshot mapSyncSnapshot(ResultSet resultSet, int rowNumber) throws ); } - /** Embedding Queue와 유효 Worker를 한 SQL에서 읽은 내부 집계 값이다. */ + /** Embedding 즉시 실행·지연 재시도 Queue와 유효 Worker를 한 SQL에서 읽은 내부 집계 값이다. */ private record EmbeddingSnapshot( long claimableJobs, + long delayedRetryJobs, long processingJobs, LocalDateTime oldestClaimableAt, long activeWorkers diff --git a/backend/src/test/java/com/opensource/docgrid/domain/document/benchmark/ChunkQualityPerformanceBenchmark.java b/backend/src/test/java/com/opensource/docgrid/domain/document/benchmark/ChunkQualityPerformanceBenchmark.java index 639d92c6..437b6096 100644 --- a/backend/src/test/java/com/opensource/docgrid/domain/document/benchmark/ChunkQualityPerformanceBenchmark.java +++ b/backend/src/test/java/com/opensource/docgrid/domain/document/benchmark/ChunkQualityPerformanceBenchmark.java @@ -30,10 +30,13 @@ import com.opensource.docgrid.domain.document.benchmark.ChunkQualityBenchmarkSupport.TimingSummary; import com.opensource.docgrid.domain.embedding.client.EmbeddingClient; import com.opensource.docgrid.domain.embedding.client.EmbeddingProviderCircuitBreaker; +import com.opensource.docgrid.domain.embedding.client.EmbeddingProviderCircuitMetrics; import com.opensource.docgrid.domain.embedding.config.EmbeddingProviderCircuitBreakerProperties; import com.opensource.docgrid.domain.embedding.dto.response.EmbedBatchItemResponse; import com.opensource.docgrid.domain.embedding.dto.response.EmbedBatchServerResponse; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; + import lombok.extern.slf4j.Slf4j; /** @@ -63,7 +66,11 @@ void compareChunkSizeAndOverlapQuality() throws IOException { EmbeddingClient embeddingClient = new EmbeddingClient( restClient, restClient, - new EmbeddingProviderCircuitBreaker(circuitProperties, Clock.systemUTC()) + new EmbeddingProviderCircuitBreaker( + circuitProperties, + Clock.systemUTC(), + new EmbeddingProviderCircuitMetrics(new SimpleMeterRegistry()) + ) ); List corpus = ChunkQualityBenchmarkSupport.createCorpus(); List profiles = ChunkQualityBenchmarkSupport.profiles(); diff --git a/backend/src/test/java/com/opensource/docgrid/domain/embedding/client/EmbeddingProviderCircuitBreakerTest.java b/backend/src/test/java/com/opensource/docgrid/domain/embedding/client/EmbeddingProviderCircuitBreakerTest.java index 78dac905..f7d5f1b3 100644 --- a/backend/src/test/java/com/opensource/docgrid/domain/embedding/client/EmbeddingProviderCircuitBreakerTest.java +++ b/backend/src/test/java/com/opensource/docgrid/domain/embedding/client/EmbeddingProviderCircuitBreakerTest.java @@ -21,8 +21,11 @@ import com.opensource.docgrid.domain.embedding.config.EmbeddingProviderCircuitBreakerProperties; import com.opensource.docgrid.global.exception.ErrorCode; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; + /** - * Embedding Provider Circuit의 연속 실패, Open fast-fail과 단일 Half-open Probe 상태 전이를 검증한다. + * Embedding Provider Circuit의 연속 실패, Open fast-fail과 단일 Half-open Probe 상태 전이·결과 + * Metric을 검증한다. * *

실제 HTTP 호출과 Worker Job 재예약은 제외하고 동시 Permission 발급과 Clock 기반 복구 경계만 * 확인한다. @@ -36,7 +39,8 @@ class EmbeddingProviderCircuitBreakerTest { @DisplayName("연속 Provider 실패 3회 뒤 Circuit을 열고 실제 호출 Permission을 거절한다") void recordFailure_opensAfterThreshold() { MutableClock clock = new MutableClock(STARTED_AT); - EmbeddingProviderCircuitBreaker circuitBreaker = circuitBreaker(clock); + SimpleMeterRegistry meterRegistry = new SimpleMeterRegistry(); + EmbeddingProviderCircuitBreaker circuitBreaker = circuitBreaker(clock, meterRegistry); circuitBreaker.recordFailure(circuitBreaker.acquirePermission(), true); circuitBreaker.recordFailure(circuitBreaker.acquirePermission(), true); @@ -44,6 +48,11 @@ void recordFailure_opensAfterThreshold() { Duration circuitDelay = circuitBreaker.recordFailure(third, true); assertThat(circuitDelay).isEqualTo(Duration.ofSeconds(30)); + assertThat(gauge(meterRegistry)).isEqualTo(1.0); + assertThat(probeFailedGauge(meterRegistry)).isZero(); + assertThat(transitions(meterRegistry, "open")).isEqualTo(1.0); + assertThat(probes(meterRegistry, "success")).isZero(); + assertThat(probes(meterRegistry, "failed")).isZero(); assertThatThrownBy(circuitBreaker::acquirePermission) .isInstanceOf(EmbeddingProviderException.class) .hasFieldOrPropertyWithValue( @@ -72,7 +81,8 @@ void recordSuccess_resetsConsecutiveFailures() { @DisplayName("Open 시간이 지나면 동시 요청 중 한 건만 Half-open Probe를 획득한다") void acquirePermission_allowsSingleHalfOpenProbe() throws Exception { MutableClock clock = new MutableClock(STARTED_AT); - EmbeddingProviderCircuitBreaker circuitBreaker = openCircuit(clock); + SimpleMeterRegistry meterRegistry = new SimpleMeterRegistry(); + EmbeddingProviderCircuitBreaker circuitBreaker = openCircuit(clock, meterRegistry); clock.advance(Duration.ofSeconds(30)); CountDownLatch start = new CountDownLatch(1); @@ -108,39 +118,89 @@ void acquirePermission_allowsSingleHalfOpenProbe() throws Exception { } assertThat(circuitBreaker.acquirePermission().halfOpenProbe()).isFalse(); + assertThat(gauge(meterRegistry)).isZero(); + assertThat(probeFailedGauge(meterRegistry)).isZero(); + assertThat(transitions(meterRegistry, "half_open")).isEqualTo(1.0); + assertThat(transitions(meterRegistry, "closed")).isEqualTo(1.0); + assertThat(probes(meterRegistry, "success")).isEqualTo(1.0); + assertThat(probes(meterRegistry, "failed")).isZero(); } @Test @DisplayName("Half-open Probe 결과를 기록하지 못하면 소유권을 반환해 다음 Probe를 허용한다") void releasePermission_allowsNextHalfOpenProbe() { MutableClock clock = new MutableClock(STARTED_AT); - EmbeddingProviderCircuitBreaker circuitBreaker = openCircuit(clock); + SimpleMeterRegistry meterRegistry = new SimpleMeterRegistry(); + EmbeddingProviderCircuitBreaker circuitBreaker = openCircuit(clock, meterRegistry); clock.advance(Duration.ofSeconds(30)); CallPermission abandonedProbe = circuitBreaker.acquirePermission(); circuitBreaker.releasePermission(abandonedProbe); + assertThat(probeFailedGauge(meterRegistry)).isZero(); + assertThat(probes(meterRegistry, "success")).isZero(); + assertThat(probes(meterRegistry, "failed")).isZero(); + CallPermission nextProbe = circuitBreaker.acquirePermission(); assertThat(nextProbe.halfOpenProbe()).isTrue(); circuitBreaker.recordSuccess(nextProbe); assertThat(circuitBreaker.acquirePermission().halfOpenProbe()).isFalse(); + assertThat(probeFailedGauge(meterRegistry)).isZero(); + assertThat(probes(meterRegistry, "success")).isEqualTo(1.0); + assertThat(probes(meterRegistry, "failed")).isZero(); } @Test @DisplayName("Half-open Probe가 실패하면 Open 시간을 새로 시작한다") void recordFailure_reopensAfterProbeFailure() { MutableClock clock = new MutableClock(STARTED_AT); - EmbeddingProviderCircuitBreaker circuitBreaker = openCircuit(clock); + SimpleMeterRegistry meterRegistry = new SimpleMeterRegistry(); + EmbeddingProviderCircuitBreaker circuitBreaker = openCircuit(clock, meterRegistry); clock.advance(Duration.ofSeconds(30)); CallPermission probe = circuitBreaker.acquirePermission(); + assertThat(gauge(meterRegistry)).isEqualTo(1.0); + assertThat(transitions(meterRegistry, "half_open")).isEqualTo(1.0); + circuitBreaker.recordFailure(probe, true); + assertThat(gauge(meterRegistry)).isEqualTo(1.0); + assertThat(probeFailedGauge(meterRegistry)).isEqualTo(1.0); + assertThat(transitions(meterRegistry, "open")).isEqualTo(2.0); + assertThat(probes(meterRegistry, "success")).isZero(); + assertThat(probes(meterRegistry, "failed")).isEqualTo(1.0); assertThatThrownBy(circuitBreaker::acquirePermission) .isInstanceOf(EmbeddingProviderException.class) .hasFieldOrPropertyWithValue("minimumRetryDelay", Duration.ofSeconds(30)); } + @Test + @DisplayName("실패한 Half-open Probe 뒤 다음 Probe가 성공하면 실패 Gauge를 해제한다") + void recordSuccess_clearsFailedProbeGaugeAfterRecovery() { + MutableClock clock = new MutableClock(STARTED_AT); + SimpleMeterRegistry meterRegistry = new SimpleMeterRegistry(); + EmbeddingProviderCircuitBreaker circuitBreaker = openCircuit(clock, meterRegistry); + clock.advance(Duration.ofSeconds(30)); + + CallPermission failedProbe = circuitBreaker.acquirePermission(); + circuitBreaker.recordFailure(failedProbe, true); + + assertThat(gauge(meterRegistry)).isEqualTo(1.0); + assertThat(probeFailedGauge(meterRegistry)).isEqualTo(1.0); + + clock.advance(Duration.ofSeconds(30)); + CallPermission successfulProbe = circuitBreaker.acquirePermission(); + circuitBreaker.recordSuccess(successfulProbe); + + assertThat(gauge(meterRegistry)).isZero(); + assertThat(probeFailedGauge(meterRegistry)).isZero(); + assertThat(transitions(meterRegistry, "open")).isEqualTo(2.0); + assertThat(transitions(meterRegistry, "half_open")).isEqualTo(2.0); + assertThat(transitions(meterRegistry, "closed")).isEqualTo(1.0); + assertThat(probes(meterRegistry, "failed")).isEqualTo(1.0); + assertThat(probes(meterRegistry, "success")).isEqualTo(1.0); + } + @Test @DisplayName("Open 전 시작한 요청의 늦은 성공은 새 Circuit 상태를 닫지 못한다") void staleSuccess_doesNotCloseOpenedCircuit() { @@ -174,7 +234,14 @@ void staleFailure_returnsRemainingOpenDelay() { } private EmbeddingProviderCircuitBreaker openCircuit(MutableClock clock) { - EmbeddingProviderCircuitBreaker circuitBreaker = circuitBreaker(clock); + return openCircuit(clock, new SimpleMeterRegistry()); + } + + private EmbeddingProviderCircuitBreaker openCircuit( + MutableClock clock, + SimpleMeterRegistry meterRegistry + ) { + EmbeddingProviderCircuitBreaker circuitBreaker = circuitBreaker(clock, meterRegistry); for (int failure = 0; failure < 3; failure++) { circuitBreaker.recordFailure(circuitBreaker.acquirePermission(), true); } @@ -182,9 +249,39 @@ private EmbeddingProviderCircuitBreaker openCircuit(MutableClock clock) { } private EmbeddingProviderCircuitBreaker circuitBreaker(Clock clock) { + return circuitBreaker(clock, new SimpleMeterRegistry()); + } + + private EmbeddingProviderCircuitBreaker circuitBreaker( + Clock clock, + SimpleMeterRegistry meterRegistry + ) { EmbeddingProviderCircuitBreakerProperties properties = new EmbeddingProviderCircuitBreakerProperties(); - return new EmbeddingProviderCircuitBreaker(properties, clock); + EmbeddingProviderCircuitMetrics metrics = new EmbeddingProviderCircuitMetrics(meterRegistry); + return new EmbeddingProviderCircuitBreaker(properties, clock, metrics); + } + + private double gauge(SimpleMeterRegistry meterRegistry) { + return meterRegistry.get("docgrid.embedding.provider.circuit.open").gauge().value(); + } + + private double probeFailedGauge(SimpleMeterRegistry meterRegistry) { + return meterRegistry.get("docgrid.embedding.provider.circuit.probe.failed").gauge().value(); + } + + private double transitions(SimpleMeterRegistry meterRegistry, String state) { + return meterRegistry.get("docgrid.embedding.provider.circuit.transitions") + .tag("state", state) + .counter() + .count(); + } + + private double probes(SimpleMeterRegistry meterRegistry, String outcome) { + return meterRegistry.get("docgrid.embedding.provider.circuit.probe") + .tag("outcome", outcome) + .counter() + .count(); } private Object get(Future future) { diff --git a/backend/src/test/java/com/opensource/docgrid/domain/embedding/integration/DocumentEmbeddingIntegrationTest.java b/backend/src/test/java/com/opensource/docgrid/domain/embedding/integration/DocumentEmbeddingIntegrationTest.java index b1c85dc5..2a7a98bc 100644 --- a/backend/src/test/java/com/opensource/docgrid/domain/embedding/integration/DocumentEmbeddingIntegrationTest.java +++ b/backend/src/test/java/com/opensource/docgrid/domain/embedding/integration/DocumentEmbeddingIntegrationTest.java @@ -37,6 +37,7 @@ import com.opensource.docgrid.domain.document.enums.DocumentVersionStatus; import com.opensource.docgrid.domain.embedding.client.EmbeddingClient; import com.opensource.docgrid.domain.embedding.client.EmbeddingProviderCircuitBreaker; +import com.opensource.docgrid.domain.embedding.client.EmbeddingProviderCircuitMetrics; import com.opensource.docgrid.domain.embedding.config.EmbeddingProviderCircuitBreakerProperties; import com.opensource.docgrid.domain.embedding.dto.request.CreateDocumentEmbeddingsRequest; import com.opensource.docgrid.domain.embedding.dto.response.EmbedBatchItemResponse; @@ -46,6 +47,8 @@ import com.opensource.docgrid.global.exception.DocGridException; import com.opensource.docgrid.global.exception.ErrorCode; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; + /** * 실제 OpenSQL에서 Chunk Embedding의 Vector 저장, 실패 원자성과 동시 Transaction 수렴을 검증한다. * @@ -365,7 +368,11 @@ private static EmbeddingProviderCircuitBreaker disabledCircuitBreaker() { EmbeddingProviderCircuitBreakerProperties properties = new EmbeddingProviderCircuitBreakerProperties(); properties.setEnabled(false); - return new EmbeddingProviderCircuitBreaker(properties, Clock.systemUTC()); + return new EmbeddingProviderCircuitBreaker( + properties, + Clock.systemUTC(), + new EmbeddingProviderCircuitMetrics(new SimpleMeterRegistry()) + ); } @Override diff --git a/backend/src/test/java/com/opensource/docgrid/global/config/ManagementEndpointIntegrationTest.java b/backend/src/test/java/com/opensource/docgrid/global/config/ManagementEndpointIntegrationTest.java index a2c471b8..fa59f50d 100644 --- a/backend/src/test/java/com/opensource/docgrid/global/config/ManagementEndpointIntegrationTest.java +++ b/backend/src/test/java/com/opensource/docgrid/global/config/ManagementEndpointIntegrationTest.java @@ -2,6 +2,8 @@ import static org.assertj.core.api.Assertions.assertThat; +import java.time.Clock; + import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -13,10 +15,14 @@ import org.springframework.boot.test.web.client.TestRestTemplate; import org.springframework.boot.test.web.server.LocalManagementPort; import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import com.opensource.docgrid.domain.embedding.client.EmbeddingProviderCircuitMetrics; +import com.opensource.docgrid.global.observability.OperationalMetrics; + /** * 별도 Management 서버의 공개 범위, health group, Prometheus 출력을 실제 HTTP 경계에서 검증한다. * @@ -42,7 +48,11 @@ "spring.data.redis.timeout=100ms" } ) -@Import(ManagementEndpointSecurityConfig.class) +@Import({ + ManagementEndpointSecurityConfig.class, + EmbeddingProviderCircuitMetrics.class, + OperationalMetrics.class +}) class ManagementEndpointIntegrationTest { @Autowired @@ -96,7 +106,12 @@ void prometheus_exportsRuntimeAndDatabaseMetrics() { assertThat(metrics) .contains("jvm_memory_used_bytes") .contains("http_server_requests_seconds_count") - .contains("hikaricp_connections"); + .contains("hikaricp_connections") + .contains("docgrid_embedding_provider_circuit_open") + .contains("docgrid_embedding_provider_circuit_transitions_total") + .contains("docgrid_embedding_provider_circuit_probe_total") + .contains("docgrid_embedding_provider_circuit_probe_failed") + .contains("docgrid_embedding_delayed_retry_jobs"); } private ResponseEntity getManagement(String path) { @@ -109,5 +124,11 @@ private ResponseEntity getManagement(String path) { @SpringBootConfiguration @EnableAutoConfiguration static class TestApplication { + + /** 운영 Metric의 시간 계산 의존성을 외부 인프라 없이 제공한다. */ + @Bean + Clock clock() { + return Clock.systemUTC(); + } } } diff --git a/backend/src/test/java/com/opensource/docgrid/global/observability/OperationalMetricsSnapshotQueryServiceTest.java b/backend/src/test/java/com/opensource/docgrid/global/observability/OperationalMetricsSnapshotQueryServiceTest.java index 0be4394e..af778477 100644 --- a/backend/src/test/java/com/opensource/docgrid/global/observability/OperationalMetricsSnapshotQueryServiceTest.java +++ b/backend/src/test/java/com/opensource/docgrid/global/observability/OperationalMetricsSnapshotQueryServiceTest.java @@ -82,10 +82,11 @@ void dropSchema() { } @Test - @DisplayName("미래 backoff를 제외하고 Queue별 수·나이와 유효 Worker를 집계한다") - void load_excludesFutureBackoffAndCountsOperationalState() { + @DisplayName("즉시 실행과 미래 backoff를 분리하고 Queue별 수·나이와 유효 Worker를 집계한다") + void load_separatesFutureBackoffAndCountsOperationalState() { insertEmbeddingJob("PENDING", NOW.minusMinutes(1), null); insertEmbeddingJob("PENDING", NOW.minusDays(2), NOW.minusMinutes(2)); + insertEmbeddingJob("PENDING", NOW.minusDays(1), NOW); insertEmbeddingJob("PENDING", NOW.minusDays(3), NOW.plusMinutes(5)); insertEmbeddingJob("PROCESSING", NOW.minusMinutes(3), null); insertWorker("fresh-active", "ACTIVE", NOW.minusSeconds(10)); @@ -101,7 +102,8 @@ void load_excludesFutureBackoffAndCountsOperationalState() { OperationalMetricsSnapshot snapshot = queryService.load(NOW); - assertThat(snapshot.embeddingClaimableJobs()).isEqualTo(2); + assertThat(snapshot.embeddingClaimableJobs()).isEqualTo(3); + assertThat(snapshot.embeddingDelayedRetryJobs()).isEqualTo(1); assertThat(snapshot.embeddingProcessingJobs()).isEqualTo(1); assertThat(snapshot.embeddingOldestClaimableAt()).isEqualTo(NOW.minusMinutes(2)); assertThat(snapshot.embeddingActiveWorkers()).isEqualTo(1); @@ -118,6 +120,7 @@ void load_returnsZerosAndNullAgesWhenQueuesAreEmpty() { OperationalMetricsSnapshot snapshot = queryService.load(NOW); assertThat(snapshot.embeddingClaimableJobs()).isZero(); + assertThat(snapshot.embeddingDelayedRetryJobs()).isZero(); assertThat(snapshot.embeddingProcessingJobs()).isZero(); assertThat(snapshot.embeddingOldestClaimableAt()).isNull(); assertThat(snapshot.embeddingActiveWorkers()).isZero(); diff --git a/backend/src/test/java/com/opensource/docgrid/global/observability/OperationalMetricsSnapshotRefresherTest.java b/backend/src/test/java/com/opensource/docgrid/global/observability/OperationalMetricsSnapshotRefresherTest.java index c341b400..3e7987fe 100644 --- a/backend/src/test/java/com/opensource/docgrid/global/observability/OperationalMetricsSnapshotRefresherTest.java +++ b/backend/src/test/java/com/opensource/docgrid/global/observability/OperationalMetricsSnapshotRefresherTest.java @@ -38,7 +38,7 @@ class OperationalMetricsSnapshotRefresherTest { void refreshesSnapshotAndServesScrapesFromMemory() { LocalDateTime observedAt = LocalDateTime.now(CLOCK); OperationalMetricsSnapshot snapshot = new OperationalMetricsSnapshot( - 2, 1, observedAt.minusSeconds(30), 1, 0, null, 0, 0, null + 2, 3, 1, observedAt.minusSeconds(30), 1, 0, null, 0, 0, null ); given(queryService.load(observedAt)).willReturn(snapshot); @@ -46,7 +46,7 @@ void refreshesSnapshotAndServesScrapesFromMemory() { reset(queryService); assertThat(meterRegistry.get("docgrid.embedding.claimable.jobs").gauge().value()).isEqualTo(2.0); - assertThat(meterRegistry.get("docgrid.embedding.claimable.jobs").gauge().value()).isEqualTo(2.0); + assertThat(meterRegistry.get("docgrid.embedding.delayed.retry.jobs").gauge().value()).isEqualTo(3.0); verifyNoInteractions(queryService); } diff --git a/backend/src/test/java/com/opensource/docgrid/global/observability/OperationalMetricsTest.java b/backend/src/test/java/com/opensource/docgrid/global/observability/OperationalMetricsTest.java index 61a6792b..6b399b6d 100644 --- a/backend/src/test/java/com/opensource/docgrid/global/observability/OperationalMetricsTest.java +++ b/backend/src/test/java/com/opensource/docgrid/global/observability/OperationalMetricsTest.java @@ -31,6 +31,7 @@ void exposesLastSuccessfulSnapshotAsGauges() { LocalDateTime now = LocalDateTime.now(clock); metrics.recordRefreshSuccess(new OperationalMetricsSnapshot( 3, + 7, 2, now.minusSeconds(75), 1, @@ -42,6 +43,7 @@ void exposesLastSuccessfulSnapshotAsGauges() { ), NOW.minusSeconds(10)); assertThat(gauge("docgrid.embedding.claimable.jobs")).isEqualTo(3.0); + assertThat(gauge("docgrid.embedding.delayed.retry.jobs")).isEqualTo(7.0); assertThat(gauge("docgrid.embedding.processing.jobs")).isEqualTo(2.0); assertThat(gauge("docgrid.embedding.oldest.claimable.age.seconds")).isEqualTo(75.0); assertThat(gauge("docgrid.embedding.active.workers")).isEqualTo(1.0); @@ -59,12 +61,13 @@ void exposesLastSuccessfulSnapshotAsGauges() { void retainsLastGoodSnapshotAfterRefreshFailure() { LocalDateTime now = LocalDateTime.now(clock); metrics.recordRefreshSuccess(new OperationalMetricsSnapshot( - 2, 0, now.minusSeconds(15), 1, 0, null, 0, 0, null + 2, 4, 0, now.minusSeconds(15), 1, 0, null, 0, 0, null ), NOW); metrics.recordRefreshFailure(); assertThat(gauge("docgrid.embedding.claimable.jobs")).isEqualTo(2.0); + assertThat(gauge("docgrid.embedding.delayed.retry.jobs")).isEqualTo(4.0); assertThat(gauge("docgrid.embedding.oldest.claimable.age.seconds")).isEqualTo(15.0); assertThat(counter("success")).isEqualTo(1.0); assertThat(counter("failed")).isEqualTo(1.0); diff --git a/docs/design/gimin-#406-embedding-circuit-retry-metrics.md b/docs/design/gimin-#406-embedding-circuit-retry-metrics.md new file mode 100644 index 00000000..257fca07 --- /dev/null +++ b/docs/design/gimin-#406-embedding-circuit-retry-metrics.md @@ -0,0 +1,121 @@ +# Embedding Circuit·지연 재시도 관측성 설계 (#406) + +closes #406 + +## 1. 배경 + +Embedding Provider Circuit 상태는 Backend JVM 메모리에만 있어 로그 시점 외에는 현재 상태를 확인할 수 +없다. 기존 Queue Snapshot도 지금 Claim 가능한 Job만 집계하므로, 장애로 `next_retry_at`이 미래에 예약된 +Job이 누적돼도 `claimable_jobs=0`으로 보일 수 있다. + +## 2. 목표와 비목표 + +### 목표 + +- JVM별 Circuit 보호 활성 상태와 상태 전환·Half-open Probe 결과를 Micrometer로 노출한다. +- 즉시 Claim 가능한 Job과 미래 재시도 Job을 같은 관측 시각에서 분리 집계한다. +- Half-open Probe 실패로 Provider 장애 지속이 확인된 뒤 Circuit이 1분간 닫히지 않으면 + Prometheus가 Cluster별 경보를 한 건 생성하고, 새로운 실패 증거가 없으면 유한 시간 안에 해제한다. + +### 비목표 + +- 여러 Backend JVM이 공유하는 분산 Circuit 상태 +- delayed retry Job 존재 자체에 대한 경보 +- Grafana Dashboard와 Provider 장애 Drill +- Sync Outbox Retry Jitter + +## 3. Metric 계약 + +| Prometheus Metric | Type | 의미 | +|---|---|---| +| `docgrid_embedding_provider_circuit_open` | Gauge | OPEN 또는 HALF_OPEN이면 `1`, CLOSED이면 `0` | +| `docgrid_embedding_provider_circuit_transitions_total{state}` | Counter | 목적 상태별 전환 횟수 | +| `docgrid_embedding_provider_circuit_probe_total{outcome}` | Counter | Half-open Probe의 성공·실패 횟수 | +| `docgrid_embedding_provider_circuit_probe_failed` | Gauge | 현재 Circuit 보호 주기에서 Probe 실패가 확인됐고 아직 성공하지 못했으면 `1` | +| `docgrid_embedding_delayed_retry_jobs` | Gauge | `PENDING`이며 `next_retry_at > observedAt`인 Job 수 | + +`state`는 `open`, `half_open`, `closed`, `outcome`은 `success`, `failed`로 고정한다. Job ID, +오류 메시지, 사용자 입력은 label에 포함하지 않는다. 두 Probe Counter는 애플리케이션 +시작 시 0으로 미리 등록해 첫 결과도 Prometheus가 관측한다. + +Gauge가 HALF_OPEN에서도 `1`을 유지하는 이유는 Probe 성공 전까지 일반 호출이 계속 차단되기 때문이다. +Probe 시작 순간 Gauge를 `0`으로 바꾸면 Provider가 계속 실패하더라도 Prometheus의 `for: 1m` 시간이 +초기화될 수 있다. 실제 상태 전이는 Counter로 별도 확인한다. + +`probe_failed` Gauge는 실패 Probe에서 `1`이 되고 성공 Probe에서 `0`이 된다. 경보는 이 Gauge와 +최근 6분의 `increase(probe_total{outcome="failed"})`를 함께 사용한다. Gauge가 현재 보호 주기를 +구분하므로 이전 Circuit 주기의 실패가 새로운 최초 OPEN에 섞이지 않고, Counter 시간 창이 트래픽 +중단 뒤 오래된 실패 증거를 무기한 유지하지 않게 한다. + +## 4. 수집 흐름 + +```text +EmbeddingProviderCircuitBreaker + → synchronized 상태 전환 확정 + → EmbeddingProviderCircuitMetrics + → 현재 보호 활성 Gauge 갱신 + → 목적 상태 Counter 증가 + → Half-open Probe 결과 Counter 증가 + → Probe 실패 확인 Gauge 설정·해제 + +OperationalMetricsSnapshotRefresher (기본 15초) + → 동일 observedAt으로 aggregate SQL 실행 + → claimable: next_retry_at IS NULL OR next_retry_at <= observedAt + → delayed: next_retry_at > observedAt + → 전체 Query 성공 시 AtomicReference Snapshot 교체 + +Prometheus scrape + → JVM Circuit Metric과 메모리 Snapshot만 읽음 + → scrape 시 DB 조회 없음 +``` + +Circuit 전이와 Metric 갱신은 같은 `synchronized` 경계에서 수행한다. delayed retry 수는 새 Scheduler를 +추가하지 않고 기존 Embedding Snapshot SQL에 집계 열 하나를 추가한다. 기존 +`(status, next_retry_at)` Index가 두 조건을 지원한다. + +## 5. 경보 + +`DocGridEmbeddingProviderCircuitOpen`은 같은 Backend JVM에서 Circuit 보호가 활성화돼 있고, +현재 보호 주기의 Half-open Probe 실패가 확인됐으며, 최근 6분 안에 실제 실패 Counter 증가가 있는 +경우에만 Pending을 시작한다. 최초 OPEN 후 트래픽이 없어 Probe가 실행되지 않은 JVM은 Provider 장애 +지속의 증거가 없으므로 경보하지 않는다. 이 조건이 1분간 유지될 때 경보한다. + +```promql +max by (cluster, environment) ( + ( + (docgrid_embedding_provider_circuit_open == 1) + and on (instance, cluster, environment) + (docgrid_embedding_provider_circuit_probe_failed == 1) + ) + and on (instance, cluster, environment) + ( + increase(docgrid_embedding_provider_circuit_probe_total{ + outcome="failed" + }[6m]) > 0 + ) +) == 1 +``` + +세 시계열은 `instance`, `cluster`, `environment`로 일치시켜 다른 JVM의 OPEN·Probe 실패·Counter가 +결합되지 않게 한다. 여러 Backend가 독립 Circuit을 가지므로 `sum`으로 인스턴스 수를 세지 않는다. + +6분 시간 창은 기본 `INDEXING_WORKER_RETRY_MAX_DELAY=5m`보다 길다. Worker만 Embedding 호출을 +만드는 환경에서도 약 5분 간격의 다음 Probe 실패를 놓치지 않으면서, 단일 실패 뒤 호출이 끊긴 JVM의 +경보는 무기한 유지하지 않는다. 운영에서 Retry 최대 지연을 5분보다 늘리면 이 시간 창도 그보다 길게 +조정해야 한다. 트래픽이 끊긴 상태에서 6분 뒤 경보가 해제되는 것은 Provider 회복을 확인했다는 뜻이 +아니라 최근 실패 증거가 만료됐다는 뜻이다. 성공 Probe가 발생하면 두 Gauge가 즉시 0이 되어 시간 창을 +기다리지 않고 해제된다. + +delayed retry Gauge는 정상 재시도 중에도 증가할 수 있어 이번 범위에서는 단독 경보 조건으로 사용하지 +않는다. + +## 6. 검증 기준 + +- 실패 임계치 도달 시 Gauge `1`, `state=open` Counter 1회 증가 +- Open 기간 뒤 단일 Probe 진입 시 `state=half_open` Counter 1회 증가, Open Gauge `1` 유지 +- Probe 성공 시 `outcome=success` Counter 증가, Open·Probe 실패 Gauge `0`, `state=closed` Counter 증가 +- Probe 실패 시 `outcome=failed` Counter와 `state=open` Counter 재증가, 두 Gauge `1` 유지 +- 같은 관측 시각의 즉시 실행 Job 3건과 미래 재시도 Job 1건 분리 집계. 경계와 같은 시각은 즉시 실행에 포함 +- Prometheus rule test에서 최초 OPEN은 무경보, Probe 실패 1분 후 발생, CLOSED 전환 시 즉시 해제 +- 단일 실패 뒤 호출이 끊기면 최근 실패 시간 창 만료 후 해제 +- 5분 간격으로 Probe가 계속 실패해도 경보 유지, 이전 보호 주기의 Probe 실패는 새 OPEN에 미전파 diff --git a/monitoring/prometheus/README.md b/monitoring/prometheus/README.md index 04cb16f3..d3f303d3 100644 --- a/monitoring/prometheus/README.md +++ b/monitoring/prometheus/README.md @@ -79,6 +79,7 @@ HTTP 오류율에서는 Streamable HTTP 특성이 다른 `/mcp`를 제외한다. | 경보 | 조건 | 지속 시간 | |---|---|---:| | `DocGridEmbeddingRetryableFailureRatioHigh` | 10분간 10회 이상 실행되고 retryable 실패가 10% 초과 | 5분 | +| `DocGridEmbeddingProviderCircuitOpen` | 최근 6분 내 Half-open Probe 실패 후 Circuit이 닫히지 않음 | 1분 | | `DocGridRagProviderFallbackSpike` | 10분간 provider fallback 3회 이상 | 1분 | | `DocGridRagTimeoutSweepSpike` | 10분간 timeout 강제 종료 3회 이상 | 1분 | | `DocGridSyncOutboxTerminalFailure` | 15분간 새로운 최종 실패 1회 이상 | 즉시 | @@ -98,13 +99,17 @@ Backend는 기본 15초마다 별도 단일 Thread에서 Queue별 aggregate quer 메모리에 보관한다. `/actuator/prometheus`의 Gauge callback은 이 메모리만 읽으므로 scrape 횟수가 DB 조회 횟수를 늘리지 않는다. 주기는 `MANAGEMENT_METRICS_SNAPSHOT_INTERVAL`로 조정할 수 있다. -`PENDING` 전체 개수가 아니라 현재 시각에 실제 claim 가능한 항목만 backlog에 포함한다. Embedding -Retry는 `next_retry_at`, Sync Outbox는 `available_at`이 미래이면 제외한다. 가장 오래된 항목의 나이도 -최초 생성 시각과 Retry 실행 가능 시각을 구분해 계산한다. +즉시 실행 backlog는 `PENDING` 전체가 아니라 현재 시각에 실제 claim 가능한 항목만 포함한다. Embedding +Retry는 `next_retry_at`이 미래인 항목을 별도 delayed Gauge로 제공하고, Sync Outbox는 `available_at`이 +미래인 항목을 제외한다. 가장 오래된 항목의 나이는 최초 생성 시각과 Retry 실행 가능 시각을 구분해 +계산한다. | Metric | 의미 | |---|---| | `docgrid_embedding_claimable_jobs` | 지금 claim 가능한 Embedding Job 수 | +| `docgrid_embedding_delayed_retry_jobs` | 미래 `next_retry_at`까지 대기하는 Embedding Retry Job 수 | +| `docgrid_embedding_provider_circuit_probe_total{outcome}` | Half-open Probe 성공·실패 횟수 | +| `docgrid_embedding_provider_circuit_probe_failed` | 현재 Circuit 보호 주기에서 Probe 실패가 확인됐고 아직 성공하지 못했는지 | | `docgrid_embedding_processing_jobs` | 처리 중인 Embedding Job 수 | | `docgrid_embedding_oldest_claimable_age_seconds` | 가장 오래된 claim 가능 Job의 대기 시간 | | `docgrid_embedding_active_workers` | Heartbeat가 만료되지 않은 ACTIVE·IDLE Worker 수 | @@ -116,6 +121,19 @@ Retry는 `next_retry_at`, Sync Outbox는 `available_at`이 미래이면 제외 | `docgrid_operational_snapshot_age_seconds` | 마지막 정상 DB Snapshot 이후 경과 시간 | | `docgrid_operational_snapshot_refresh_total{outcome}` | Snapshot 갱신 성공·실패 횟수 | +Circuit 상태는 JVM별로 수집한다. `docgrid_embedding_provider_circuit_open`은 일반 호출이 차단되는 +OPEN과 단일 Probe만 허용하는 HALF_OPEN에서 `1`이고, Probe 성공으로 CLOSED가 되면 `0`이다. +`docgrid_embedding_provider_circuit_transitions_total{state}`는 `open`, `half_open`, `closed` 전환 횟수를 +제한된 label로 제공한다. Probe 결과 Counter와 실패 Gauge는 최초 OPEN 이후 실제 회복 Probe가 +실패한 경우만 구분한다. 트래픽이 없어 Probe가 실행되지 않으면 Circuit 경보는 발생하지 않고, +성공 Probe가 Circuit을 닫으면 경보 조건이 즉시 해제된다. Probe 실패 뒤 트래픽이 끊기면 성공 여부를 +확인할 수 없으므로, 최근 실패 Counter 증가가 6분 시간 창에서 빠질 때 경보를 해제한다. 이 해제는 +Provider 회복 확인이 아니라 오래된 실패 증거의 만료를 의미한다. + +6분 시간 창은 기본 `INDEXING_WORKER_RETRY_MAX_DELAY=5m`보다 길어, Worker의 Probe가 최대 재시도 +간격으로 반복돼도 지속 장애 경보가 끊기지 않게 한다. 이 환경 변수를 5분보다 늘리면 +`DocGridEmbeddingProviderCircuitOpen` 규칙의 Counter 시간 창도 더 긴 값으로 함께 조정해야 한다. + 여러 Backend가 같은 DB를 수집하면 동일 Gauge가 인스턴스 수만큼 노출된다. 번들 규칙은 이 값을 합산하지 않고 `max by (cluster, environment)`로 평가해 backlog를 중복 계산하지 않는다. 갱신 실패는 마지막 정상 값을 유지하며, 첫 성공 전과 장시간 실패는 `snapshot_age` 경보로 드러난다. diff --git a/monitoring/prometheus/rules/docgrid-pipeline-alerts.yml b/monitoring/prometheus/rules/docgrid-pipeline-alerts.yml index 914003a2..f5049b0c 100644 --- a/monitoring/prometheus/rules/docgrid-pipeline-alerts.yml +++ b/monitoring/prometheus/rules/docgrid-pipeline-alerts.yml @@ -31,6 +31,34 @@ groups: summary: DocGrid embedding retryable failure ratio is high description: More than 10% of at least 10 embedding attempts failed with a retryable cause during the last ten minutes. + - alert: DocGridEmbeddingProviderCircuitOpen + expr: | + max by (cluster, environment) ( + ( + ( + docgrid_embedding_provider_circuit_open == 1 + ) + and on (instance, cluster, environment) + ( + docgrid_embedding_provider_circuit_probe_failed == 1 + ) + ) + and on (instance, cluster, environment) + ( + increase(docgrid_embedding_provider_circuit_probe_total{ + outcome="failed" + }[6m]) > 0 + ) + ) == 1 + for: 1m + labels: + severity: warning + service: docgrid-backend + dependency: embedding-provider + annotations: + summary: DocGrid embedding provider recovery probe recently failed + description: A half-open probe failed within the last six minutes and the circuit remained unclosed for one minute. + - alert: DocGridRagProviderFallbackSpike expr: | sum by (cluster, environment) ( diff --git a/monitoring/prometheus/tests/docgrid-operational-alerts.test.yml b/monitoring/prometheus/tests/docgrid-operational-alerts.test.yml index f3e821aa..c8b1794d 100644 --- a/monitoring/prometheus/tests/docgrid-operational-alerts.test.yml +++ b/monitoring/prometheus/tests/docgrid-operational-alerts.test.yml @@ -4,6 +4,185 @@ rule_files: evaluation_interval: 15s tests: + - name: initial circuit open without a failed probe stays silent per instance + interval: 15s + input_series: + - series: 'docgrid_embedding_provider_circuit_open{instance="backend-a:8081",environment="production",cluster="docgrid-production"}' + values: '1x8' + - series: 'docgrid_embedding_provider_circuit_probe_failed{instance="backend-a:8081",environment="production",cluster="docgrid-production"}' + values: '0x8' + - series: 'docgrid_embedding_provider_circuit_open{instance="backend-b:8081",environment="production",cluster="docgrid-production"}' + values: '0x8' + - series: 'docgrid_embedding_provider_circuit_probe_failed{instance="backend-b:8081",environment="production",cluster="docgrid-production"}' + values: '1x8' + alert_rule_test: + - eval_time: 2m + alertname: DocGridEmbeddingProviderCircuitOpen + exp_alerts: [] + + - name: failed recovery probe alerts after one minute and resolves immediately when closed + interval: 15s + input_series: + - series: 'docgrid_embedding_provider_circuit_open{instance="backend-a:8081",environment="production",cluster="docgrid-production"}' + values: '1x7 0x3' + - series: 'docgrid_embedding_provider_circuit_probe_failed{instance="backend-a:8081",environment="production",cluster="docgrid-production"}' + values: '0x1 1x5 0x3' + - series: 'docgrid_embedding_provider_circuit_probe_total{instance="backend-a:8081",environment="production",cluster="docgrid-production",outcome="failed"}' + values: '0x1 1x9' + alert_rule_test: + - eval_time: 1m15s + alertname: DocGridEmbeddingProviderCircuitOpen + exp_alerts: [] + - eval_time: 1m30s + alertname: DocGridEmbeddingProviderCircuitOpen + exp_alerts: + - exp_labels: + alertname: DocGridEmbeddingProviderCircuitOpen + cluster: docgrid-production + dependency: embedding-provider + environment: production + service: docgrid-backend + severity: warning + exp_annotations: + summary: DocGrid embedding provider recovery probe recently failed + description: A half-open probe failed within the last six minutes and the circuit remained unclosed for one minute. + - eval_time: 2m + alertname: DocGridEmbeddingProviderCircuitOpen + exp_alerts: [] + + - name: circuit alert stays active when failed probes are five minutes apart + interval: 15s + input_series: + - series: 'docgrid_embedding_provider_circuit_open{instance="backend-a:8081",environment="production",cluster="docgrid-production"}' + values: '1x48' + - series: 'docgrid_embedding_provider_circuit_probe_failed{instance="backend-a:8081",environment="production",cluster="docgrid-production"}' + values: '0x1 1x46' + - series: 'docgrid_embedding_provider_circuit_probe_total{instance="backend-a:8081",environment="production",cluster="docgrid-production",outcome="failed"}' + values: '0x1 1x19 2x19 3x7' + alert_rule_test: + - eval_time: 1m30s + alertname: DocGridEmbeddingProviderCircuitOpen + exp_alerts: + - exp_labels: + alertname: DocGridEmbeddingProviderCircuitOpen + cluster: docgrid-production + dependency: embedding-provider + environment: production + service: docgrid-backend + severity: warning + exp_annotations: + summary: DocGrid embedding provider recovery probe recently failed + description: A half-open probe failed within the last six minutes and the circuit remained unclosed for one minute. + - eval_time: 5m15s + alertname: DocGridEmbeddingProviderCircuitOpen + exp_alerts: + - exp_labels: + alertname: DocGridEmbeddingProviderCircuitOpen + cluster: docgrid-production + dependency: embedding-provider + environment: production + service: docgrid-backend + severity: warning + exp_annotations: + summary: DocGrid embedding provider recovery probe recently failed + description: A half-open probe failed within the last six minutes and the circuit remained unclosed for one minute. + - eval_time: 5m30s + alertname: DocGridEmbeddingProviderCircuitOpen + exp_alerts: + - exp_labels: + alertname: DocGridEmbeddingProviderCircuitOpen + cluster: docgrid-production + dependency: embedding-provider + environment: production + service: docgrid-backend + severity: warning + exp_annotations: + summary: DocGrid embedding provider recovery probe recently failed + description: A half-open probe failed within the last six minutes and the circuit remained unclosed for one minute. + - eval_time: 10m15s + alertname: DocGridEmbeddingProviderCircuitOpen + exp_alerts: + - exp_labels: + alertname: DocGridEmbeddingProviderCircuitOpen + cluster: docgrid-production + dependency: embedding-provider + environment: production + service: docgrid-backend + severity: warning + exp_annotations: + summary: DocGrid embedding provider recovery probe recently failed + description: A half-open probe failed within the last six minutes and the circuit remained unclosed for one minute. + - eval_time: 11m30s + alertname: DocGridEmbeddingProviderCircuitOpen + exp_alerts: + - exp_labels: + alertname: DocGridEmbeddingProviderCircuitOpen + cluster: docgrid-production + dependency: embedding-provider + environment: production + service: docgrid-backend + severity: warning + exp_annotations: + summary: DocGrid embedding provider recovery probe recently failed + description: A half-open probe failed within the last six minutes and the circuit remained unclosed for one minute. + + - name: single failed probe expires after six minutes without traffic + interval: 15s + input_series: + - series: 'docgrid_embedding_provider_circuit_open{instance="backend-a:8081",environment="production",cluster="docgrid-production"}' + values: '1x120' + - series: 'docgrid_embedding_provider_circuit_probe_failed{instance="backend-a:8081",environment="production",cluster="docgrid-production"}' + values: '0x1 1x119' + - series: 'docgrid_embedding_provider_circuit_probe_total{instance="backend-a:8081",environment="production",cluster="docgrid-production",outcome="failed"}' + values: '0x1 1x119' + alert_rule_test: + - eval_time: 1m30s + alertname: DocGridEmbeddingProviderCircuitOpen + exp_alerts: + - exp_labels: + alertname: DocGridEmbeddingProviderCircuitOpen + cluster: docgrid-production + dependency: embedding-provider + environment: production + service: docgrid-backend + severity: warning + exp_annotations: + summary: DocGrid embedding provider recovery probe recently failed + description: A half-open probe failed within the last six minutes and the circuit remained unclosed for one minute. + - eval_time: 6m + alertname: DocGridEmbeddingProviderCircuitOpen + exp_alerts: + - exp_labels: + alertname: DocGridEmbeddingProviderCircuitOpen + cluster: docgrid-production + dependency: embedding-provider + environment: production + service: docgrid-backend + severity: warning + exp_annotations: + summary: DocGrid embedding provider recovery probe recently failed + description: A half-open probe failed within the last six minutes and the circuit remained unclosed for one minute. + - eval_time: 6m15s + alertname: DocGridEmbeddingProviderCircuitOpen + exp_alerts: [] + - eval_time: 30m + alertname: DocGridEmbeddingProviderCircuitOpen + exp_alerts: [] + + - name: previous probe failure does not affect a new initial circuit open + interval: 15s + input_series: + - series: 'docgrid_embedding_provider_circuit_open{instance="backend-a:8081",environment="production",cluster="docgrid-production"}' + values: '1x3 0x3 1x8' + - series: 'docgrid_embedding_provider_circuit_probe_failed{instance="backend-a:8081",environment="production",cluster="docgrid-production"}' + values: '1x3 0x12' + - series: 'docgrid_embedding_provider_circuit_probe_total{instance="backend-a:8081",environment="production",cluster="docgrid-production",outcome="failed"}' + values: '0 1x15' + alert_rule_test: + - eval_time: 3m15s + alertname: DocGridEmbeddingProviderCircuitOpen + exp_alerts: [] + - name: claimable embedding work without workers alerts once per cluster interval: 15s input_series: