From aca0feb2af6b36b459a73804fd5be984d34ba3fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EA=B8=B0=EB=AF=BC?= Date: Sun, 4 Oct 2026 02:59:53 +0900 Subject: [PATCH 1/6] feat: provision bundled grafana service --- .env.example | 4 +++ docker-compose.yml | 31 +++++++++++++++++++ .../provisioning/dashboards/docgrid.yml | 14 +++++++++ .../provisioning/datasources/prometheus.yml | 17 ++++++++++ 4 files changed, 66 insertions(+) create mode 100644 monitoring/grafana/provisioning/dashboards/docgrid.yml create mode 100644 monitoring/grafana/provisioning/datasources/prometheus.yml diff --git a/.env.example b/.env.example index 1c00552e..a6f5b3e0 100644 --- a/.env.example +++ b/.env.example @@ -74,6 +74,10 @@ MINIO_SECRET_KEY=minioadmin1234 # ALERTMANAGER_MEMORY_LIMIT=128m # ALERTMANAGER_CPU_LIMIT=0.25 # ALERTMANAGER_CONFIG_FILE=./monitoring/alertmanager/alertmanager.yml +# Grafana는 127.0.0.1에만 열리고 익명 Viewer만 허용하며 관리자 계정을 만들지 않습니다. +# GRAFANA_PORT=3000 +# GRAFANA_MEMORY_LIMIT=384m +# GRAFANA_CPU_LIMIT=0.5 # 벡터 검색 최소 유사도 - 기본값 0.30을 쓰면 설정하지 않아도 됩니다. # 검색 품질 평가로 기준을 조정하거나 이전 Top-K 동작을 확인할 때만 변경합니다. diff --git a/docker-compose.yml b/docker-compose.yml index 4bafefe5..e6efc3a7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -118,6 +118,36 @@ services: networks: - docgrid-local + grafana: + image: grafana/grafana:13.2.3 + container_name: docgrid-grafana + profiles: ["monitoring"] + restart: unless-stopped + mem_limit: "${GRAFANA_MEMORY_LIMIT:-384m}" + cpus: "${GRAFANA_CPU_LIMIT:-0.5}" + environment: + # 1. Bundled Grafana는 Loopback 전용 읽기 화면이며 저장소에 관리자 Credential을 두지 않는다. + GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION: "true" + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + GF_AUTH_DISABLE_LOGIN_FORM: "true" + GF_USERS_ALLOW_SIGN_UP: "false" + ports: + - "127.0.0.1:${GRAFANA_PORT:-3000}:3000" + volumes: + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - ./monitoring/grafana/dashboards:/etc/grafana/dashboards:ro + - grafana-data:/var/lib/grafana + healthcheck: + test: ["CMD", "wget", "--spider", "--quiet", "http://localhost:3000/api/health"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 20s + # 2. Prometheus가 잠시 중단돼도 Grafana가 먼저 기동하고 연결 복구를 기다린다. + networks: + - docgrid-local + valkey: image: valkey/valkey:9.1-alpine container_name: docgrid-valkey @@ -145,3 +175,4 @@ volumes: valkey-data: prometheus-data: alertmanager-data: + grafana-data: diff --git a/monitoring/grafana/provisioning/dashboards/docgrid.yml b/monitoring/grafana/provisioning/dashboards/docgrid.yml new file mode 100644 index 00000000..74d0b248 --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/docgrid.yml @@ -0,0 +1,14 @@ +apiVersion: 1 + +providers: + - name: DocGrid + orgId: 1 + folder: DocGrid + folderUid: docgrid + type: file + disableDeletion: false + allowUiUpdates: false + # Docker Bind Mount에서도 파일 변경을 확실히 감지하도록 Watch 대신 Polling을 사용한다. + updateIntervalSeconds: 30 + options: + path: /etc/grafana/dashboards diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yml b/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 00000000..ef60651c --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,17 @@ +apiVersion: 1 + +# 제거된 Provisioning 파일의 Data Source가 로컬 DB에 남지 않게 한다. +prune: true + +datasources: + - name: DocGrid Prometheus + uid: docgrid-prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + version: 1 + jsonData: + httpMethod: POST + timeInterval: 15s From fd74bde1aab439a337b428eb20b5ecd19252e466 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EA=B8=B0=EB=AF=BC?= Date: Sun, 4 Oct 2026 03:00:01 +0900 Subject: [PATCH 2/6] feat: add docgrid operations dashboard --- .../dashboards/docgrid-operations.json | 2611 +++++++++++++++++ 1 file changed, 2611 insertions(+) create mode 100644 monitoring/grafana/dashboards/docgrid-operations.json diff --git a/monitoring/grafana/dashboards/docgrid-operations.json b/monitoring/grafana/dashboards/docgrid-operations.json new file mode 100644 index 00000000..e023fc34 --- /dev/null +++ b/monitoring/grafana/dashboards/docgrid-operations.json @@ -0,0 +1,2611 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "DocGrid Backend와 비동기 Pipeline의 Queue, Circuit, Provider, RAG, Sync Outbox 상태를 함께 해석하는 운영 Dashboard", + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "DocGrid Monitoring Guide", + "tooltip": "Prometheus와 Alertmanager 운영 가이드", + "type": "link", + "url": "https://github.com/DocGrid/docgrid/tree/develop/monitoring" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "전체 상태", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "선택한 Backend Target의 마지막 Scrape 성공 여부입니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 0, + "text": "DOWN" + }, + "1": { + "color": "green", + "index": 1, + "text": "UP" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": false, + "expr": "max(up{job=\"docgrid-backend\",cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"})", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "title": "Backend", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "Embedding Provider Metric Endpoint의 마지막 Scrape 성공 여부입니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 0, + "text": "DOWN" + }, + "1": { + "color": "green", + "index": 1, + "text": "UP" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": false, + "expr": "max(up{job=\"embedding-provider\",cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"})", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "title": "Embedding Provider", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "Prometheus가 현재 firing으로 평가한 경보 수입니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": false, + "expr": "sum(ALERTS{alertstate=\"firing\",cluster=~\"$cluster\",environment=~\"$environment\"}) or vector(0)", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "title": "발생 중 경보", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "마지막 정상 DB 운영 Snapshot 갱신 이후 경과 시간입니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 30 + }, + { + "color": "red", + "value": 60 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": false, + "expr": "max(docgrid_operational_snapshot_age_seconds{cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"})", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "title": "운영 Snapshot 나이", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 6, + "panels": [], + "title": "Embedding Queue", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "현재 시각에 Worker가 바로 claim할 수 있는 PENDING Job 수입니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 7, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": false, + "expr": "max(docgrid_embedding_claimable_jobs{cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"})", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "title": "즉시 처리 가능 Job", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "미래 next_retry_at까지 대기 중인 PENDING Job 수입니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 7 + }, + "id": 8, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": false, + "expr": "max(docgrid_embedding_delayed_retry_jobs{cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"})", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "title": "지연 Retry Job", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "PROCESSING 상태의 Embedding Job 수입니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 7 + }, + "id": 9, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": false, + "expr": "max(docgrid_embedding_processing_jobs{cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"})", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "title": "처리 중 Job", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "Heartbeat가 만료되지 않은 ACTIVE 또는 IDLE Worker 수입니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 7 + }, + "id": 10, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": false, + "expr": "max(docgrid_embedding_active_workers{cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"})", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "title": "활성 Worker", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "여러 Backend가 같은 DB 상태를 노출하므로 합산하지 않고 cluster별 최댓값을 사용합니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": true, + "expr": "max by (cluster) (docgrid_embedding_claimable_jobs{cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"})", + "legendFormat": "claimable", + "refId": "A", + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": true, + "expr": "max by (cluster) (docgrid_embedding_delayed_retry_jobs{cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"})", + "legendFormat": "delayed retry", + "refId": "B", + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": true, + "expr": "max by (cluster) (docgrid_embedding_processing_jobs{cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"})", + "legendFormat": "processing", + "refId": "C", + "instant": false + } + ], + "title": "Embedding Queue 추이", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "대기 시간과 최근 실행 중 재시도 가능한 실패 비율을 함께 봅니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": true, + "expr": "max by (cluster) (docgrid_embedding_oldest_claimable_age_seconds{cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"})", + "legendFormat": "oldest claimable age", + "refId": "A", + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": true, + "expr": "100 * sum by (cluster) (rate(docgrid_embedding_job_attempts_total{cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\",outcome=~\"retry_scheduled|terminal_failure\",retryable=\"true\"}[$__rate_interval])) / clamp_min(sum by (cluster) (rate(docgrid_embedding_job_attempts_total{cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"}[$__rate_interval])), 0.001)", + "legendFormat": "retryable failure %", + "refId": "B", + "instant": false + } + ], + "title": "Embedding 지연과 Retry 실패율", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 13, + "panels": [], + "title": "Embedding Circuit", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "OPEN 또는 HALF_OPEN이면 1이며 일반 호출이 제한됩니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "green", + "index": 0, + "text": "정상" + }, + "1": { + "color": "red", + "index": 1, + "text": "보호 중" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 21 + }, + "id": 14, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": false, + "expr": "max(docgrid_embedding_provider_circuit_open{cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"})", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "title": "Circuit 보호 상태", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "현재 Circuit 보호 주기에서 실패한 Half-open Probe가 있고 아직 성공하지 못했는지 나타냅니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "green", + "index": 0, + "text": "정상" + }, + "1": { + "color": "red", + "index": 1, + "text": "보호 중" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 21 + }, + "id": 15, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": false, + "expr": "max(docgrid_embedding_provider_circuit_probe_failed{cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"})", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "title": "현재 보호 주기 Probe 실패", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "선택 구간 내 OPEN, HALF_OPEN, CLOSED 전환 횟수입니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": true, + "expr": "sum by (state) (increase(docgrid_embedding_provider_circuit_transitions_total{cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{state}}", + "refId": "A", + "instant": false + } + ], + "title": "Circuit 상태 전환", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "실제 Provider 회복을 확인한 Half-open Probe의 성공·실패 횟수입니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": true, + "expr": "sum by (outcome) (increase(docgrid_embedding_provider_circuit_probe_total{cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{outcome}}", + "refId": "A", + "instant": false + } + ], + "title": "Half-open Probe 결과", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 18, + "panels": [], + "title": "Embedding Provider", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "Provider Process가 모델을 메모리에 올리고 요청을 받을 준비가 됐는지 나타냅니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 0, + "text": "DOWN" + }, + "1": { + "color": "green", + "index": 1, + "text": "UP" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 37 + }, + "id": 19, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": false, + "expr": "max(embedding_provider_model_loaded{job=\"embedding-provider\",cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"})", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "title": "Model Loaded", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "Provider 내부 FIFO Queue에서 대기 중인 요청 수입니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 37 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": false, + "expr": "max(embedding_provider_waiting_requests{job=\"embedding-provider\",cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"})", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "title": "대기 요청", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "Provider가 허용하는 내부 대기 Queue의 최대 크기입니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 37 + }, + "id": 21, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": false, + "expr": "max(embedding_provider_max_queue_size{job=\"embedding-provider\",cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"})", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "title": "Queue 상한", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "Provider RSS를 Cgroup Memory Limit으로 나눈 비율입니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 37 + }, + "id": 22, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": false, + "expr": "100 * max(process_resident_memory_bytes{job=\"embedding-provider\",cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"}) / clamp_min(max(embedding_provider_memory_limit_bytes{job=\"embedding-provider\",cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"}), 1)", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "title": "메모리 사용률", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "Provider 요청 결과별 초당 처리율입니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 23, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": true, + "expr": "sum by (outcome) (rate(embedding_provider_requests_total{job=\"embedding-provider\",cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{outcome}}", + "refId": "A", + "instant": false + } + ], + "title": "Provider 요청 결과", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "성공한 Embedding 요청의 p99 처리 시간입니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": true, + "expr": "histogram_quantile(0.99, sum by (cluster, le) (rate(embedding_provider_request_duration_seconds_bucket{job=\"embedding-provider\",cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\",outcome=\"success\"}[$__rate_interval])))", + "legendFormat": "p99", + "refId": "A", + "instant": false + } + ], + "title": "Provider 성공 요청 p99", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 25, + "panels": [], + "title": "RAG Queue", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "PROCESSING 상태의 RAG 응답 수입니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 51 + }, + "id": 26, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": false, + "expr": "max(docgrid_rag_processing_jobs{cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"})", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "title": "처리 중 RAG", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "가장 오래 PROCESSING에 머문 RAG 응답의 나이입니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 120 + }, + { + "color": "red", + "value": 150 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 51 + }, + "id": 27, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": false, + "expr": "max(docgrid_rag_oldest_processing_age_seconds{cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"})", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "title": "가장 오래된 RAG", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "정상 완료, Provider Fallback, Timeout Sweep 결과를 구분합니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 28, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": true, + "expr": "sum by (outcome) (increase(docgrid_rag_job_completions_total{cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{outcome}}", + "refId": "A", + "instant": false + } + ], + "title": "RAG 완료 결과", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 29, + "panels": [], + "title": "Sync Outbox", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "available_at이 현재 시각 이하인 PENDING Outbox Event 수입니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 60 + }, + "id": 30, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": false, + "expr": "max(docgrid_sync_outbox_claimable_events{cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"})", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "title": "즉시 처리 가능 Event", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "PROCESSING 상태의 Sync Outbox Event 수입니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 60 + }, + "id": 31, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": false, + "expr": "max(docgrid_sync_outbox_processing_events{cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"})", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "title": "처리 중 Event", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "가장 오래된 claim 가능 Sync Event의 대기 시간입니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 90 + }, + { + "color": "red", + "value": 120 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 60 + }, + "id": 32, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": false, + "expr": "max(docgrid_sync_outbox_oldest_claimable_age_seconds{cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"})", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "title": "가장 오래된 Event", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "성공, Retry 예약, 최종 실패 횟수를 구분합니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 60 + }, + "id": 33, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": true, + "expr": "sum by (outcome) (increase(docgrid_sync_event_attempts_total{cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{outcome}}", + "refId": "A", + "instant": false + } + ], + "title": "Sync Outbox 처리 결과", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 65 + }, + "id": 34, + "panels": [], + "title": "Metric 수집 상태", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "DB Aggregate Snapshot 갱신이 멈추면 계속 증가합니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 66 + }, + "id": 35, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": true, + "expr": "max by (cluster) (docgrid_operational_snapshot_age_seconds{cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"})", + "legendFormat": "{{cluster}}", + "refId": "A", + "instant": false + } + ], + "title": "운영 Snapshot 나이", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "description": "Snapshot 갱신 성공과 실패 횟수입니다.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 66 + }, + "id": 36, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "editorMode": "code", + "range": true, + "expr": "sum by (outcome) (increase(docgrid_operational_snapshot_refresh_total{cluster=~\"$cluster\",environment=~\"$environment\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{outcome}}", + "refId": "A", + "instant": false + } + ], + "title": "운영 Snapshot 갱신 결과", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "15s", + "schemaVersion": 41, + "tags": [ + "docgrid", + "operations", + "prometheus" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "definition": "label_values(up{job=~\"docgrid-backend|embedding-provider\"}, cluster)", + "includeAll": true, + "allValue": ".*", + "label": "Cluster", + "name": "cluster", + "options": [], + "query": { + "query": "label_values(up{job=~\"docgrid-backend|embedding-provider\"}, cluster)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "definition": "label_values(up{cluster=~\"$cluster\"}, environment)", + "includeAll": true, + "allValue": ".*", + "label": "Environment", + "name": "environment", + "options": [], + "query": { + "query": "label_values(up{cluster=~\"$cluster\"}, environment)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "docgrid-prometheus" + }, + "definition": "label_values(up{cluster=~\"$cluster\",environment=~\"$environment\"}, instance)", + "includeAll": true, + "allValue": ".*", + "label": "Instance", + "name": "instance", + "options": [], + "query": { + "query": "label_values(up{cluster=~\"$cluster\",environment=~\"$environment\"}, instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "15s", + "30s", + "1m", + "5m", + "15m" + ] + }, + "timezone": "browser", + "title": "DocGrid Operations", + "uid": "docgrid-operations", + "version": 1, + "weekStart": "" +} From 9f8d762dabf7d15c1442843da4ba8f863be21a15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EA=B8=B0=EB=AF=BC?= Date: Sun, 4 Oct 2026 03:00:08 +0900 Subject: [PATCH 3/6] test: verify grafana dashboard provisioning --- monitoring/grafana/tests/assert_dashboard.py | 118 ++++++++++++++++++ monitoring/grafana/tests/backend-metrics.txt | 47 +++++++ monitoring/grafana/tests/docker-compose.yml | 36 ++++++ monitoring/grafana/tests/prometheus.yml | 25 ++++ monitoring/grafana/tests/provider-metrics.txt | 20 +++ monitoring/grafana/tests/run-e2e.sh | 33 +++++ monitoring/grafana/validate_dashboard.py | 108 ++++++++++++++++ monitoring/verify.sh | 22 +++- 8 files changed, 407 insertions(+), 2 deletions(-) create mode 100755 monitoring/grafana/tests/assert_dashboard.py create mode 100644 monitoring/grafana/tests/backend-metrics.txt create mode 100644 monitoring/grafana/tests/docker-compose.yml create mode 100644 monitoring/grafana/tests/prometheus.yml create mode 100644 monitoring/grafana/tests/provider-metrics.txt create mode 100755 monitoring/grafana/tests/run-e2e.sh create mode 100755 monitoring/grafana/validate_dashboard.py diff --git a/monitoring/grafana/tests/assert_dashboard.py b/monitoring/grafana/tests/assert_dashboard.py new file mode 100755 index 00000000..e7127ffe --- /dev/null +++ b/monitoring/grafana/tests/assert_dashboard.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Verify Grafana provisioning and execute every dashboard PromQL expression.""" + +from __future__ import annotations + +import json +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + + +EXPECTED_UID = "docgrid-operations" +EXPECTED_DATASOURCE_UID = "docgrid-prometheus" + + +def request_json(url: str) -> dict: + with urllib.request.urlopen(url, timeout=5) as response: + return json.load(response) + + +def wait_json(url: str, predicate, description: str, timeout_seconds: int = 60) -> dict: + deadline = time.monotonic() + timeout_seconds + last_error: Exception | None = None + while time.monotonic() < deadline: + try: + payload = request_json(url) + if predicate(payload): + return payload + except (OSError, urllib.error.HTTPError, json.JSONDecodeError) as error: + last_error = error + time.sleep(1) + raise AssertionError(f"{description} 대기 실패: {last_error}") + + +def prometheus_query(base_url: str, expression: str) -> list[dict]: + encoded = urllib.parse.urlencode({"query": expression}) + url = ( + f"{base_url}/api/datasources/proxy/uid/{EXPECTED_DATASOURCE_UID}" + f"/api/v1/query?{encoded}" + ) + payload = request_json(url) + if payload.get("status") != "success": + raise AssertionError(f"PromQL 실행 실패: {expression}: {payload}") + return payload.get("data", {}).get("result", []) + + +def concrete_expression(expression: str) -> str: + return ( + expression.replace('$cluster', 'docgrid-test') + .replace('$environment', 'test') + .replace('$instance', '.*') + .replace('$__rate_interval', '1m') + ) + + +def main(base_url: str) -> None: + # 1. Grafana DB와 HTTP Server가 준비될 때까지 실제 Health API를 확인한다. + health = wait_json( + f"{base_url}/api/health", + lambda payload: payload.get("database") == "ok", + "Grafana Health", + ) + print(f"[성공] Grafana Health: database={health['database']}") + + # 2. File Provisioning이 Data Source와 Dashboard를 실제 DB에 등록했는지 확인한다. + datasource = wait_json( + f"{base_url}/api/datasources/uid/{EXPECTED_DATASOURCE_UID}", + lambda payload: payload.get("uid") == EXPECTED_DATASOURCE_UID, + "Prometheus Data Source Provisioning", + ) + if datasource.get("url") != "http://prometheus:9090": + raise AssertionError(f"예상하지 않은 Prometheus URL: {datasource.get('url')}") + print(f"[성공] Data Source Provisioning: uid={datasource['uid']}") + + response = wait_json( + f"{base_url}/api/dashboards/uid/{EXPECTED_UID}", + lambda payload: payload.get("dashboard", {}).get("uid") == EXPECTED_UID, + "Dashboard Provisioning", + ) + dashboard = response["dashboard"] + panels = dashboard.get("panels", []) + variables = dashboard.get("templating", {}).get("list", []) + if {variable.get("name") for variable in variables} != {"cluster", "environment", "instance"}: + raise AssertionError("Dashboard 변수 Provisioning 결과가 예상과 다릅니다.") + print(f"[성공] Dashboard Provisioning: uid={dashboard['uid']}, panels={len(panels)}") + + # 3. Grafana Data Source Proxy를 경유해 실제 Fixture 시계열이 조회되는지 확인한다. + claimable = wait_json( + f"{base_url}/api/datasources/proxy/uid/{EXPECTED_DATASOURCE_UID}/api/v1/query?" + + urllib.parse.urlencode( + {"query": 'docgrid_embedding_claimable_jobs{cluster="docgrid-test",environment="test"}'} + ), + lambda payload: bool(payload.get("data", {}).get("result")), + "Prometheus Fixture Scrape", + ) + value = float(claimable["data"]["result"][0]["value"][1]) + if value != 4: + raise AssertionError(f"claimable fixture 값이 4가 아닙니다: {value}") + print(f"[성공] Grafana→Prometheus 실제 조회: claimable_jobs={value:.0f}") + + # 4. Dashboard의 모든 PromQL을 변수 치환 후 실행해 문법과 Label Matcher를 검증한다. + expressions = [ + target["expr"] + for panel in panels + for target in panel.get("targets", []) + if target.get("expr") + ] + for expression in expressions: + prometheus_query(base_url, concrete_expression(expression)) + print(f"[성공] Dashboard PromQL 실행: {len(expressions)}개") + + +if __name__ == "__main__": + if len(sys.argv) != 2: + raise SystemExit("usage: assert_dashboard.py http://127.0.0.1:PORT") + main(sys.argv[1].rstrip("/")) diff --git a/monitoring/grafana/tests/backend-metrics.txt b/monitoring/grafana/tests/backend-metrics.txt new file mode 100644 index 00000000..5d86745f --- /dev/null +++ b/monitoring/grafana/tests/backend-metrics.txt @@ -0,0 +1,47 @@ +# HELP docgrid_embedding_claimable_jobs Claimable embedding jobs. +# TYPE docgrid_embedding_claimable_jobs gauge +docgrid_embedding_claimable_jobs 4 +# TYPE docgrid_embedding_delayed_retry_jobs gauge +docgrid_embedding_delayed_retry_jobs 2 +# TYPE docgrid_embedding_processing_jobs gauge +docgrid_embedding_processing_jobs 1 +# TYPE docgrid_embedding_oldest_claimable_age_seconds gauge +docgrid_embedding_oldest_claimable_age_seconds 70 +# TYPE docgrid_embedding_active_workers gauge +docgrid_embedding_active_workers 1 +# TYPE docgrid_embedding_provider_circuit_open gauge +docgrid_embedding_provider_circuit_open 1 +# TYPE docgrid_embedding_provider_circuit_probe_failed gauge +docgrid_embedding_provider_circuit_probe_failed 1 +# TYPE docgrid_embedding_provider_circuit_transitions_total counter +docgrid_embedding_provider_circuit_transitions_total{state="open"} 2 +docgrid_embedding_provider_circuit_transitions_total{state="half_open"} 1 +docgrid_embedding_provider_circuit_transitions_total{state="closed"} 0 +# TYPE docgrid_embedding_provider_circuit_probe_total counter +docgrid_embedding_provider_circuit_probe_total{outcome="failed"} 1 +docgrid_embedding_provider_circuit_probe_total{outcome="success"} 0 +# TYPE docgrid_embedding_job_attempts_total counter +docgrid_embedding_job_attempts_total{outcome="success",failure_type="NONE",retryable="false"} 20 +docgrid_embedding_job_attempts_total{outcome="retry_scheduled",failure_type="EMBEDDING_PROVIDER_TIMEOUT",retryable="true"} 2 +# TYPE docgrid_rag_processing_jobs gauge +docgrid_rag_processing_jobs 1 +# TYPE docgrid_rag_oldest_processing_age_seconds gauge +docgrid_rag_oldest_processing_age_seconds 20 +# TYPE docgrid_rag_job_completions_total counter +docgrid_rag_job_completions_total{outcome="success"} 10 +docgrid_rag_job_completions_total{outcome="provider_fallback"} 1 +# TYPE docgrid_sync_outbox_claimable_events gauge +docgrid_sync_outbox_claimable_events 2 +# TYPE docgrid_sync_outbox_processing_events gauge +docgrid_sync_outbox_processing_events 1 +# TYPE docgrid_sync_outbox_oldest_claimable_age_seconds gauge +docgrid_sync_outbox_oldest_claimable_age_seconds 30 +# TYPE docgrid_sync_event_attempts_total counter +docgrid_sync_event_attempts_total{outcome="success"} 8 +docgrid_sync_event_attempts_total{outcome="retry_scheduled"} 1 +docgrid_sync_event_attempts_total{outcome="terminal_failure"} 0 +# TYPE docgrid_operational_snapshot_age_seconds gauge +docgrid_operational_snapshot_age_seconds 12 +# TYPE docgrid_operational_snapshot_refresh_total counter +docgrid_operational_snapshot_refresh_total{outcome="success"} 12 +docgrid_operational_snapshot_refresh_total{outcome="failed"} 1 diff --git a/monitoring/grafana/tests/docker-compose.yml b/monitoring/grafana/tests/docker-compose.yml new file mode 100644 index 00000000..f48c07f9 --- /dev/null +++ b/monitoring/grafana/tests/docker-compose.yml @@ -0,0 +1,36 @@ +services: + metrics-fixture: + image: python:3.14-alpine + command: ["python3", "-m", "http.server", "8000", "--directory", "/fixture"] + volumes: + - ./backend-metrics.txt:/fixture/backend-metrics.txt:ro + - ./provider-metrics.txt:/fixture/provider-metrics.txt:ro + + prometheus: + image: prom/prometheus:v3.5.5 + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ../../prometheus/rules:/etc/prometheus/rules:ro + + grafana: + image: grafana/grafana:13.2.3 + environment: + GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION: "true" + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + GF_AUTH_DISABLE_LOGIN_FORM: "true" + GF_USERS_ALLOW_SIGN_UP: "false" + ports: + - "127.0.0.1:${GRAFANA_TEST_PORT}:3000" + volumes: + - ../provisioning:/etc/grafana/provisioning:ro + - ../dashboards:/etc/grafana/dashboards:ro + healthcheck: + test: ["CMD", "wget", "--spider", "--quiet", "http://localhost:3000/api/health"] + interval: 2s + timeout: 2s + retries: 30 + start_period: 5s diff --git a/monitoring/grafana/tests/prometheus.yml b/monitoring/grafana/tests/prometheus.yml new file mode 100644 index 00000000..d607a4e1 --- /dev/null +++ b/monitoring/grafana/tests/prometheus.yml @@ -0,0 +1,25 @@ +global: + scrape_interval: 1s + evaluation_interval: 1s + +rule_files: + - /etc/prometheus/rules/*.yml + +scrape_configs: + - job_name: docgrid-backend + metrics_path: /backend-metrics.txt + static_configs: + - targets: + - metrics-fixture:8000 + labels: + environment: test + cluster: docgrid-test + + - job_name: embedding-provider + metrics_path: /provider-metrics.txt + static_configs: + - targets: + - metrics-fixture:8000 + labels: + environment: test + cluster: docgrid-test diff --git a/monitoring/grafana/tests/provider-metrics.txt b/monitoring/grafana/tests/provider-metrics.txt new file mode 100644 index 00000000..e0ca2fdb --- /dev/null +++ b/monitoring/grafana/tests/provider-metrics.txt @@ -0,0 +1,20 @@ +# TYPE embedding_provider_model_loaded gauge +embedding_provider_model_loaded 1 +# TYPE embedding_provider_waiting_requests gauge +embedding_provider_waiting_requests 1 +# TYPE embedding_provider_max_queue_size gauge +embedding_provider_max_queue_size 2 +# TYPE embedding_provider_memory_limit_bytes gauge +embedding_provider_memory_limit_bytes 4294967296 +# TYPE process_resident_memory_bytes gauge +process_resident_memory_bytes 1073741824 +# TYPE embedding_provider_requests_total counter +embedding_provider_requests_total{outcome="success"} 10 +embedding_provider_requests_total{outcome="overloaded"} 1 +# TYPE embedding_provider_request_duration_seconds histogram +embedding_provider_request_duration_seconds_bucket{outcome="success",le="1"} 4 +embedding_provider_request_duration_seconds_bucket{outcome="success",le="5"} 9 +embedding_provider_request_duration_seconds_bucket{outcome="success",le="25"} 10 +embedding_provider_request_duration_seconds_bucket{outcome="success",le="+Inf"} 10 +embedding_provider_request_duration_seconds_count{outcome="success"} 10 +embedding_provider_request_duration_seconds_sum{outcome="success"} 22 diff --git a/monitoring/grafana/tests/run-e2e.sh b/monitoring/grafana/tests/run-e2e.sh new file mode 100755 index 00000000..ba371387 --- /dev/null +++ b/monitoring/grafana/tests/run-e2e.sh @@ -0,0 +1,33 @@ +#!/bin/sh +set -eu + +ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/../../.." && pwd) +COMPOSE_FILE="$ROOT_DIR/monitoring/grafana/tests/docker-compose.yml" +RUN_ID=$(date -u +%Y%m%dt%H%M%Sz)-$$ +PROJECT_NAME="docgrid-grafana-$RUN_ID" +GRAFANA_TEST_PORT=$(python3 - <<'PY' +import socket + +with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + print(sock.getsockname()[1]) +PY +) +export GRAFANA_TEST_PORT + +cleanup() { + docker compose -p "$PROJECT_NAME" -f "$COMPOSE_FILE" down --volumes --remove-orphans >/dev/null 2>&1 || true +} +trap cleanup EXIT INT TERM + +echo "[시작] run_id=$RUN_ID grafana_url=http://127.0.0.1:$GRAFANA_TEST_PORT" + +# 1. 실제 Prometheus와 Grafana를 격리 Project로 기동한다. +docker compose -p "$PROJECT_NAME" -f "$COMPOSE_FILE" up -d --wait + +# 2. Provisioning 결과와 모든 Dashboard PromQL을 실제 HTTP API로 검증한다. +python3 "$ROOT_DIR/monitoring/grafana/tests/assert_dashboard.py" \ + "http://127.0.0.1:$GRAFANA_TEST_PORT" + +# 3. 종료 Trap이 Container와 익명 Volume을 회수한다. +echo "[완료] run_id=$RUN_ID result=SUCCESS" diff --git a/monitoring/grafana/validate_dashboard.py b/monitoring/grafana/validate_dashboard.py new file mode 100755 index 00000000..48f3c427 --- /dev/null +++ b/monitoring/grafana/validate_dashboard.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Validate invariants shared by DocGrid's provisioned Grafana dashboard.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + + +DASHBOARD_UID = "docgrid-operations" +DATASOURCE_UID = "docgrid-prometheus" +EXPECTED_VARIABLES = {"cluster", "environment", "instance"} +EXPECTED_ROWS = { + "전체 상태", + "Embedding Queue", + "Embedding Circuit", + "Embedding Provider", + "RAG Queue", + "Sync Outbox", + "Metric 수집 상태", +} +REQUIRED_METRICS = { + "docgrid_embedding_claimable_jobs", + "docgrid_embedding_delayed_retry_jobs", + "docgrid_embedding_provider_circuit_open", + "docgrid_embedding_provider_circuit_probe_failed", + "docgrid_rag_processing_jobs", + "docgrid_sync_outbox_claimable_events", + "docgrid_operational_snapshot_age_seconds", + "embedding_provider_model_loaded", +} + + +def fail(message: str) -> None: + raise SystemExit(f"Grafana dashboard validation failed: {message}") + + +def validate(path: Path) -> None: + dashboard = json.loads(path.read_text(encoding="utf-8")) + if dashboard.get("uid") != DASHBOARD_UID: + fail(f"expected dashboard uid {DASHBOARD_UID}") + if dashboard.get("editable") is not False: + fail("provisioned dashboard must be read-only") + + # 1. 선택 범위를 제한하는 세 변수는 모든 운영 Panel이 공유한다. + variables = dashboard.get("templating", {}).get("list", []) + variable_names = {variable.get("name") for variable in variables} + if variable_names != EXPECTED_VARIABLES: + fail(f"unexpected variables: {sorted(variable_names)}") + for variable in variables: + if variable.get("datasource", {}).get("uid") != DATASOURCE_UID: + fail(f"variable {variable.get('name')} uses another data source") + + # 2. Panel ID와 Grid 영역이 겹치면 Provisioning은 성공해도 화면을 읽기 어렵다. + panels = dashboard.get("panels", []) + panel_ids = [panel.get("id") for panel in panels] + if len(panel_ids) != len(set(panel_ids)): + fail("panel ids must be unique") + row_titles = {panel.get("title") for panel in panels if panel.get("type") == "row"} + if row_titles != EXPECTED_ROWS: + fail(f"unexpected rows: {sorted(row_titles)}") + + occupied: set[tuple[int, int]] = set() + expressions: list[str] = [] + for panel in panels: + grid = panel.get("gridPos", {}) + for x in range(grid.get("x", 0), grid.get("x", 0) + grid.get("w", 0)): + for y in range(grid.get("y", 0), grid.get("y", 0) + grid.get("h", 0)): + coordinate = (x, y) + if coordinate in occupied: + fail(f"panel {panel.get('title')} overlaps another panel at {coordinate}") + occupied.add(coordinate) + + if panel.get("type") == "row": + continue + if not panel.get("description"): + fail(f"panel {panel.get('title')} has no interpretation description") + if panel.get("datasource", {}).get("uid") != DATASOURCE_UID: + fail(f"panel {panel.get('title')} uses another data source") + targets = panel.get("targets", []) + if not targets: + fail(f"panel {panel.get('title')} has no Prometheus query") + for target in targets: + if target.get("datasource", {}).get("uid") != DATASOURCE_UID: + fail(f"target in {panel.get('title')} uses another data source") + expression = target.get("expr", "") + if not expression: + fail(f"target in {panel.get('title')} has an empty expression") + expressions.append(expression) + + # 3. 핵심 장애 판단 Metric이 빠지면 Dashboard가 일부 정상처럼 보일 수 있다. + joined_expressions = "\n".join(expressions) + missing_metrics = sorted(metric for metric in REQUIRED_METRICS if metric not in joined_expressions) + if missing_metrics: + fail(f"required metrics are missing: {', '.join(missing_metrics)}") + + print( + "Grafana dashboard validation: SUCCESS " + f"({len(panels)} panels, {len(expressions)} PromQL expressions)" + ) + + +if __name__ == "__main__": + dashboard_path = Path(sys.argv[1]) if len(sys.argv) > 1 else Path( + "monitoring/grafana/dashboards/docgrid-operations.json" + ) + validate(dashboard_path) diff --git a/monitoring/verify.sh b/monitoring/verify.sh index f85d52a0..5c092470 100755 --- a/monitoring/verify.sh +++ b/monitoring/verify.sh @@ -58,7 +58,14 @@ for config_file in "$ROOT_DIR"/monitoring/alertmanager/examples/*.yml; do "$ALERTMANAGER_IMAGE" check-config /etc/alertmanager/alertmanager.yml done -# 4. 장시간 실험을 다시 실행하지 않고도 Drill 실행기 문법과 격리 Compose를 검증한다. +# 4. Grafana Dashboard 불변식과 격리 E2E Compose를 검증한다. +python3 "$ROOT_DIR/monitoring/grafana/validate_dashboard.py" \ + "$ROOT_DIR/monitoring/grafana/dashboards/docgrid-operations.json" +sh -n "$ROOT_DIR/monitoring/grafana/tests/run-e2e.sh" +GRAFANA_TEST_PORT=13000 docker compose \ + -f "$ROOT_DIR/monitoring/grafana/tests/docker-compose.yml" config --quiet + +# 5. 장시간 실험을 다시 실행하지 않고도 Drill 실행기 문법과 격리 Compose를 검증한다. python3 - "$ROOT_DIR/monitoring/drills/run_drill.py" <<'PY' import ast import pathlib @@ -70,7 +77,7 @@ sh -n "$ROOT_DIR/monitoring/drills/run.sh" DRILL_TMP_DIR="$SECRET_DIR" docker compose \ -f "$ROOT_DIR/monitoring/drills/docker-compose.yml" config --quiet -# 5. 사용자가 실행할 두 Compose 형태가 모두 정상 렌더링되는지 확인한다. +# 6. 사용자가 실행할 두 Compose 형태가 모두 정상 렌더링되는지 확인한다. docker compose -f "$ROOT_DIR/docker-compose.yml" config --quiet MONITORING_COMPOSE_CONFIG="$SECRET_DIR/monitoring-compose.json" docker compose -f "$ROOT_DIR/docker-compose.yml" --profile monitoring \ @@ -86,10 +93,21 @@ monitored_dependencies = {"alertmanager", "embedding-server"} & set(prometheus_d if monitored_dependencies: names = ", ".join(sorted(monitored_dependencies)) raise SystemExit(f"Prometheus must start independently from monitored services: {names}") + +grafana = config["services"]["grafana"] +if grafana.get("depends_on"): + raise SystemExit("Grafana must start independently and reconnect to Prometheus") +published = grafana["ports"][0] +if published.get("host_ip") != "127.0.0.1": + raise SystemExit("Bundled Grafana must only publish to the loopback interface") +environment = grafana.get("environment", {}) +if environment.get("GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION") != "true": + raise SystemExit("Bundled Grafana must not create a default admin account") PY if [ "${1:-}" = "--e2e" ]; then "$ROOT_DIR/monitoring/alertmanager/tests/run-e2e.sh" + "$ROOT_DIR/monitoring/grafana/tests/run-e2e.sh" fi echo "Monitoring configuration validation: SUCCESS" From b16946ee0abe7b1e1ffef52767cc81d50dabfb52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EA=B8=B0=EB=AF=BC?= Date: Sun, 4 Oct 2026 03:02:34 +0900 Subject: [PATCH 4/6] docs: record grafana operations verification --- README.md | 2 +- .../run-01-compose-project-name-failure.log | 15 + .../evidence/issue-416/run-02-grafana-e2e.log | 334 ++++++++++++++++++ .../run-03-monitoring-validation.log | 168 +++++++++ ...gimin-#416-grafana-operations-dashboard.md | 150 ++++++++ monitoring/grafana/README.md | 89 +++++ monitoring/prometheus/README.md | 7 +- 7 files changed, 762 insertions(+), 3 deletions(-) create mode 100644 docs/test-results/evidence/issue-416/run-01-compose-project-name-failure.log create mode 100644 docs/test-results/evidence/issue-416/run-02-grafana-e2e.log create mode 100644 docs/test-results/evidence/issue-416/run-03-monitoring-validation.log create mode 100644 docs/test-results/gimin-#416-grafana-operations-dashboard.md create mode 100644 monitoring/grafana/README.md diff --git a/README.md b/README.md index 9ae80d4e..3887cebe 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ SQL을 균등하게 나누는 설정이 아닙니다. Patroni는 각 노드의 | AI | BAAI/bge-m3, Ollama, qwen2.5:7b | | Storage | GCS (GCP), MinIO (로컬), Local Filesystem, 선택형 AWS S3 Adapter | | Cache | Valkey 9.1 (로컬 Compose), Spring Data Redis 클라이언트; GCP 공용 VM은 전환 전 Redis 6.2 확인 | -| Observability | RAGOps Dashboard, Prometheus, Alertmanager (로컬 monitoring profile) | +| Observability | RAGOps Dashboard, Prometheus, Alertmanager, Grafana (로컬 monitoring profile) | | Test | JUnit 5, Gradle, Node.js Test Runner | ## 빠른 실행 diff --git a/docs/test-results/evidence/issue-416/run-01-compose-project-name-failure.log b/docs/test-results/evidence/issue-416/run-01-compose-project-name-failure.log new file mode 100644 index 00000000..1ce9352b --- /dev/null +++ b/docs/test-results/evidence/issue-416/run-01-compose-project-name-failure.log @@ -0,0 +1,15 @@ +테스트 목적: Grafana Provisioning·Prometheus 연동 E2E +실행 위치: 로컬 Docker Desktop +실행 명령: ./monitoring/grafana/tests/run-e2e.sh +실행 ID: 20261003T175627Z-62642 +시각대: UTC +Revision: feature/416 작업 트리(커밋 전) +성공 기준: Grafana·Prometheus·Metric Fixture 기동 후 Dashboard Provisioning 및 32개 PromQL 성공 +캡처 방식: 실행 직후 Codex 도구 출력을 그대로 보존했으며 Live tee Log는 아님 + +[시작] run_id=20261003T175627Z-62642 grafana_url=http://127.0.0.1:61846 +invalid project name "docgrid-grafana-20261003T175627Z-62642": must consist only of lowercase alphanumeric characters, hyphens, and underscores as well as start with a letter or number + +결과: FAIL +실패 원인: UTC Run ID의 대문자 T와 Z를 Docker Compose Project 이름에 포함함 +실행 범위: Compose 입력 검증 단계에서 종료되어 Container와 Volume은 생성되지 않음 diff --git a/docs/test-results/evidence/issue-416/run-02-grafana-e2e.log b/docs/test-results/evidence/issue-416/run-02-grafana-e2e.log new file mode 100644 index 00000000..93a6f17f --- /dev/null +++ b/docs/test-results/evidence/issue-416/run-02-grafana-e2e.log @@ -0,0 +1,334 @@ +테스트 목적: 실제 Grafana Provisioning·Prometheus Metric 조회·Dashboard PromQL 검증 +실행 위치: 로컬 Docker Desktop, 격리 Compose Project와 동적 Loopback Port +실행 명령: ./monitoring/grafana/tests/run-e2e.sh +시각대: Asia/Seoul(테스트 Run ID는 UTC) +Revision: feature/416 작업 트리(커밋 전) +Workload: Backend Fixture 27개 Sample 계열, Provider Fixture 13개 Sample 계열, Dashboard PromQL 32개 +성공 기준: Health·Data Source·Dashboard·Fixture 값·PromQL 32개 통과 및 Container·Volume 정리 + +[시작] run_id=20261003t175700z-63206 grafana_url=http://127.0.0.1:61878 + Image python:3.14-alpine Pulling + Image grafana/grafana:13.2.3 Pulling + a9986cd6f37d Already exists 0B + 541523bae70c Pulling fs layer 0B + 9625030534d1 Pulling fs layer 0B + de6af1da8f48 Pulling fs layer 0B + a9986cd6f37d Already exists 0B + 1efc728fcb3e Pulling fs layer 0B + 70c8dc1ba068 Pulling fs layer 0B + 4f4fb700ef54 Pulling fs layer 0B + 8998fa436a65 Pulling fs layer 0B + 1efc728fcb3e Waiting 0B + 70c8dc1ba068 Waiting 0B + ade4e896e587 Pulling fs layer 0B + df1d4782a40b Pulling fs layer 0B + 4f4fb700ef54 Waiting 0B + 947ef31db38e Pulling fs layer 0B + ade4e896e587 Waiting 0B + 8998fa436a65 Waiting 0B + df1d4782a40b Waiting 0B + bb9dbccd4085 Pulling fs layer 0B + 947ef31db38e Waiting 0B + 37bc7484dfb5 Pulling fs layer 0B + bb9dbccd4085 Waiting 0B + 37bc7484dfb5 Waiting 0B + 9625030534d1 Downloading 168.2kB + 9625030534d1 Downloading 4.083MB + 9625030534d1 Downloading 7.824MB + 9625030534d1 Downloading 11.35MB + 541523bae70c Downloading 16.38kB + 9625030534d1 Downloading 14.61MB + 541523bae70c Downloading 77.94kB + 9625030534d1 Verifying Checksum 0B + 9625030534d1 Download complete 0B + de6af1da8f48 Downloading 248B + de6af1da8f48 Verifying Checksum 0B + de6af1da8f48 Download complete 0B + 541523bae70c Downloading 419.1kB + 541523bae70c Verifying Checksum 0B + 541523bae70c Download complete 0B + 541523bae70c Extracting 32.77kB + 541523bae70c Extracting 419.1kB + 541523bae70c Extracting 419.1kB + 541523bae70c Pull complete 0B + 9625030534d1 Extracting 196.6kB + 1efc728fcb3e Downloading 139B + 1efc728fcb3e Download complete 0B + 1efc728fcb3e Extracting 139B + 1efc728fcb3e Extracting 139B + 1efc728fcb3e Pull complete 0B + 9625030534d1 Extracting 5.308MB + 9625030534d1 Extracting 8.651MB + 70c8dc1ba068 Downloading 80.34kB + 9625030534d1 Extracting 12.78MB + 70c8dc1ba068 Downloading 2.61MB + 9625030534d1 Extracting 14.75MB + 70c8dc1ba068 Downloading 6.078MB + 9625030534d1 Extracting 16.5MB + 9625030534d1 Extracting 16.5MB + 4f4fb700ef54 Downloading 32B + 70c8dc1ba068 Downloading 6.573MB + 4f4fb700ef54 Verifying Checksum 0B + 4f4fb700ef54 Download complete 0B + 70c8dc1ba068 Verifying Checksum 0B + 70c8dc1ba068 Download complete 0B + 70c8dc1ba068 Extracting 98.3kB + 9625030534d1 Pull complete 0B + de6af1da8f48 Extracting 248B + de6af1da8f48 Extracting 248B + de6af1da8f48 Pull complete 0B + Image python:3.14-alpine Pulled + 70c8dc1ba068 Extracting 3.637MB + 8998fa436a65 Downloading 16.38kB + 8998fa436a65 Downloading 76.02kB + 8998fa436a65 Verifying Checksum 0B + 8998fa436a65 Download complete 0B + 70c8dc1ba068 Extracting 6.873MB + ade4e896e587 Downloading 16.38kB + ade4e896e587 Downloading 32.55kB + ade4e896e587 Verifying Checksum 0B + ade4e896e587 Download complete 0B + 70c8dc1ba068 Pull complete 0B + 4f4fb700ef54 Extracting 32B + 4f4fb700ef54 Extracting 32B + 4f4fb700ef54 Pull complete 0B + 8998fa436a65 Extracting 32.77kB + 8998fa436a65 Extracting 76.02kB + 8998fa436a65 Pull complete 0B + ade4e896e587 Extracting 32.55kB + ade4e896e587 Extracting 32.55kB + ade4e896e587 Pull complete 0B + df1d4782a40b Downloading 539.4kB + df1d4782a40b Downloading 4.8MB + 947ef31db38e Downloading 526.4kB + df1d4782a40b Downloading 7.985MB + 947ef31db38e Downloading 1.051MB + bb9dbccd4085 Downloading 182B + bb9dbccd4085 Verifying Checksum 0B + bb9dbccd4085 Download complete 0B + df1d4782a40b Downloading 12.8MB + 947ef31db38e Downloading 3.162MB + df1d4782a40b Downloading 17.6MB + 947ef31db38e Downloading 5.305MB + df1d4782a40b Downloading 22.4MB + 37bc7484dfb5 Downloading 1.333kB + 37bc7484dfb5 Verifying Checksum 0B + 37bc7484dfb5 Download complete 0B + 947ef31db38e Downloading 7.438MB + df1d4782a40b Downloading 26.67MB + df1d4782a40b Downloading 27.21MB + 947ef31db38e Downloading 9.587MB + df1d4782a40b Downloading 31.49MB + 947ef31db38e Downloading 11.73MB + df1d4782a40b Downloading 35.22MB + 947ef31db38e Downloading 13.87MB + df1d4782a40b Downloading 39.5MB + 947ef31db38e Downloading 16.01MB + df1d4782a40b Downloading 43.76MB + 947ef31db38e Downloading 18.16MB + df1d4782a40b Downloading 47.49MB + 947ef31db38e Downloading 20.3MB + df1d4782a40b Downloading 51.74MB + 947ef31db38e Downloading 22.96MB + df1d4782a40b Downloading 55.45MB + 947ef31db38e Downloading 25.09MB + df1d4782a40b Downloading 59.69MB + df1d4782a40b Downloading 60.22MB + 947ef31db38e Downloading 27.21MB + df1d4782a40b Downloading 63.95MB + 947ef31db38e Downloading 29.34MB + df1d4782a40b Downloading 68.21MB + 947ef31db38e Downloading 31.99MB + 947ef31db38e Downloading 34.13MB + df1d4782a40b Downloading 72.48MB + df1d4782a40b Downloading 76.23MB + 947ef31db38e Downloading 36.81MB + df1d4782a40b Downloading 77.3MB + 947ef31db38e Downloading 39.46MB + df1d4782a40b Downloading 81.53MB + 947ef31db38e Downloading 42.13MB + df1d4782a40b Downloading 85.29MB + 947ef31db38e Downloading 44.79MB + df1d4782a40b Downloading 89.02MB + 947ef31db38e Downloading 47.45MB + df1d4782a40b Downloading 93.24MB + 947ef31db38e Downloading 47.99MB + df1d4782a40b Downloading 96.97MB + 947ef31db38e Downloading 50.64MB + df1d4782a40b Downloading 101.2MB + 947ef31db38e Downloading 53.31MB + df1d4782a40b Downloading 105.5MB + 947ef31db38e Downloading 55.98MB + df1d4782a40b Downloading 109.7MB + 947ef31db38e Downloading 58.09MB + df1d4782a40b Downloading 114MB + 947ef31db38e Downloading 60.76MB + 947ef31db38e Downloading 63.44MB + df1d4782a40b Downloading 118.2MB + 947ef31db38e Downloading 66.1MB + df1d4782a40b Downloading 122.5MB + 947ef31db38e Downloading 68.76MB + df1d4782a40b Downloading 126.8MB + 947ef31db38e Downloading 69.29MB + df1d4782a40b Downloading 130MB + 947ef31db38e Downloading 72.49MB + df1d4782a40b Downloading 134.2MB + 947ef31db38e Downloading 75.16MB + df1d4782a40b Downloading 136.9MB + 947ef31db38e Downloading 77.83MB + df1d4782a40b Downloading 140.7MB + df1d4782a40b Downloading 141.2MB + 947ef31db38e Downloading 80.5MB + df1d4782a40b Downloading 144.9MB + 947ef31db38e Downloading 83.67MB + df1d4782a40b Downloading 149.2MB + 947ef31db38e Downloading 86.87MB + df1d4782a40b Downloading 152.9MB + 947ef31db38e Downloading 90.07MB + df1d4782a40b Downloading 156.7MB + 947ef31db38e Downloading 92.72MB + df1d4782a40b Downloading 157.7MB + 947ef31db38e Downloading 95.93MB + df1d4782a40b Downloading 161.5MB + 947ef31db38e Downloading 99.12MB + df1d4782a40b Downloading 165.3MB + df1d4782a40b Downloading 168.4MB + 947ef31db38e Downloading 101.8MB + df1d4782a40b Downloading 171.7MB + 947ef31db38e Downloading 104.5MB + df1d4782a40b Downloading 174.9MB + 947ef31db38e Downloading 107.6MB + df1d4782a40b Downloading 178MB + 947ef31db38e Downloading 110.3MB + df1d4782a40b Downloading 181.8MB + 947ef31db38e Downloading 113.5MB + df1d4782a40b Downloading 185MB + 947ef31db38e Downloading 115.1MB + df1d4782a40b Downloading 185.5MB + 947ef31db38e Downloading 118.3MB + df1d4782a40b Downloading 189.2MB + 947ef31db38e Downloading 121MB + df1d4782a40b Downloading 193MB + 947ef31db38e Downloading 124.1MB + df1d4782a40b Downloading 197.3MB + 947ef31db38e Downloading 127.3MB + df1d4782a40b Downloading 201MB + df1d4782a40b Downloading 204.7MB + 947ef31db38e Downloading 130.5MB + df1d4782a40b Downloading 209MB + 947ef31db38e Downloading 133.7MB + 947ef31db38e Downloading 136.4MB + df1d4782a40b Downloading 212.7MB + df1d4782a40b Downloading 215.9MB + 947ef31db38e Downloading 139.6MB + df1d4782a40b Downloading 219.7MB + 947ef31db38e Downloading 142.8MB + 947ef31db38e Verifying Checksum 0B + 947ef31db38e Download complete 0B + df1d4782a40b Downloading 223.9MB + df1d4782a40b Downloading 229.2MB + df1d4782a40b Downloading 233.5MB + df1d4782a40b Downloading 239.9MB + df1d4782a40b Downloading 245.8MB + df1d4782a40b Downloading 251.1MB + df1d4782a40b Downloading 256.4MB + df1d4782a40b Downloading 257.5MB + df1d4782a40b Downloading 263.9MB + df1d4782a40b Downloading 269.2MB + df1d4782a40b Downloading 275.1MB + df1d4782a40b Downloading 279.3MB + df1d4782a40b Downloading 280.4MB + df1d4782a40b Downloading 285.2MB + df1d4782a40b Download complete 0B + df1d4782a40b Extracting 557.1kB + df1d4782a40b Extracting 7.799MB + df1d4782a40b Extracting 11.7MB + df1d4782a40b Extracting 19.5MB + df1d4782a40b Extracting 28.97MB + df1d4782a40b Extracting 40.11MB + df1d4782a40b Extracting 45.12MB + df1d4782a40b Extracting 55.71MB + df1d4782a40b Extracting 59.05MB + df1d4782a40b Extracting 61.28MB + df1d4782a40b Extracting 72.42MB + df1d4782a40b Extracting 81.33MB + df1d4782a40b Extracting 86.9MB + df1d4782a40b Extracting 97.48MB + df1d4782a40b Extracting 120.3MB + df1d4782a40b Extracting 152.6MB + df1d4782a40b Extracting 184.9MB + df1d4782a40b Extracting 189.4MB + df1d4782a40b Extracting 192.7MB + df1d4782a40b Extracting 193.9MB + df1d4782a40b Extracting 195MB + df1d4782a40b Extracting 196.1MB + df1d4782a40b Extracting 198.3MB + df1d4782a40b Extracting 199.4MB + df1d4782a40b Extracting 202.8MB + df1d4782a40b Extracting 204.4MB + df1d4782a40b Extracting 205MB + df1d4782a40b Extracting 208.9MB + df1d4782a40b Extracting 214.5MB + df1d4782a40b Extracting 221.2MB + df1d4782a40b Extracting 230.6MB + df1d4782a40b Extracting 238.4MB + df1d4782a40b Extracting 245.1MB + df1d4782a40b Extracting 251.2MB + df1d4782a40b Extracting 261.3MB + df1d4782a40b Extracting 269.6MB + df1d4782a40b Extracting 276.9MB + df1d4782a40b Extracting 281.9MB + df1d4782a40b Extracting 287.4MB + df1d4782a40b Extracting 288.4MB + df1d4782a40b Pull complete 0B + 947ef31db38e Extracting 557.1kB + 947ef31db38e Extracting 12.26MB + 947ef31db38e Extracting 23.95MB + 947ef31db38e Extracting 36.77MB + 947ef31db38e Extracting 49.02MB + 947ef31db38e Extracting 61.28MB + 947ef31db38e Extracting 73.53MB + 947ef31db38e Extracting 85.23MB + 947ef31db38e Extracting 94.14MB + 947ef31db38e Extracting 95.81MB + 947ef31db38e Extracting 98.04MB + 947ef31db38e Extracting 100.3MB + 947ef31db38e Extracting 108.1MB + 947ef31db38e Extracting 119.2MB + 947ef31db38e Extracting 127.6MB + 947ef31db38e Extracting 140.4MB + 947ef31db38e Extracting 145.3MB + 947ef31db38e Pull complete 0B + bb9dbccd4085 Extracting 182B + bb9dbccd4085 Extracting 182B + bb9dbccd4085 Pull complete 0B + 37bc7484dfb5 Extracting 1.333kB + 37bc7484dfb5 Extracting 1.333kB + 37bc7484dfb5 Pull complete 0B + Image grafana/grafana:13.2.3 Pulled + Network docgrid-grafana-20261003t175700z-63206_default Creating + Network docgrid-grafana-20261003t175700z-63206_default Created + Container docgrid-grafana-20261003t175700z-63206-grafana-1 Creating + Container docgrid-grafana-20261003t175700z-63206-prometheus-1 Creating + Container docgrid-grafana-20261003t175700z-63206-metrics-fixture-1 Creating + Container docgrid-grafana-20261003t175700z-63206-grafana-1 Created + Container docgrid-grafana-20261003t175700z-63206-metrics-fixture-1 Created + Container docgrid-grafana-20261003t175700z-63206-prometheus-1 Created + Container docgrid-grafana-20261003t175700z-63206-grafana-1 Starting + Container docgrid-grafana-20261003t175700z-63206-metrics-fixture-1 Starting + Container docgrid-grafana-20261003t175700z-63206-prometheus-1 Starting + Container docgrid-grafana-20261003t175700z-63206-grafana-1 Started + Container docgrid-grafana-20261003t175700z-63206-prometheus-1 Started + Container docgrid-grafana-20261003t175700z-63206-metrics-fixture-1 Started + Container docgrid-grafana-20261003t175700z-63206-grafana-1 Waiting + Container docgrid-grafana-20261003t175700z-63206-metrics-fixture-1 Waiting + Container docgrid-grafana-20261003t175700z-63206-prometheus-1 Waiting + Container docgrid-grafana-20261003t175700z-63206-metrics-fixture-1 Healthy + Container docgrid-grafana-20261003t175700z-63206-prometheus-1 Healthy + Container docgrid-grafana-20261003t175700z-63206-grafana-1 Healthy +[성공] Grafana Health: database=ok +[성공] Data Source Provisioning: uid=docgrid-prometheus +[성공] Dashboard Provisioning: uid=docgrid-operations, panels=36 +[성공] Grafana→Prometheus 실제 조회: claimable_jobs=4 +[성공] Dashboard PromQL 실행: 32개 +[완료] run_id=20261003t175700z-63206 result=SUCCESS diff --git a/docs/test-results/evidence/issue-416/run-03-monitoring-validation.log b/docs/test-results/evidence/issue-416/run-03-monitoring-validation.log new file mode 100644 index 00000000..5f473fcf --- /dev/null +++ b/docs/test-results/evidence/issue-416/run-03-monitoring-validation.log @@ -0,0 +1,168 @@ +테스트 목적: Grafana 변경을 포함한 전체 Monitoring 회귀 검증 +실행 위치: 로컬 Docker Desktop +실행 명령: ./monitoring/verify.sh --e2e +시각대: Asia/Seoul(내부 E2E Run ID는 UTC) +Revision: 22e41b2 +Workload: Prometheus 설정 1개·Rule 3개·Rule Test 3개, Alertmanager 설정과 채널 예시, Compose 3종, Alertmanager E2E, Grafana Dashboard 36개 Panel·PromQL 32개 +성공 기준: 모든 정적 검사·Rule Test·Routing E2E·Grafana E2E 통과 및 격리 자원 정리 + +Checking /etc/prometheus/prometheus.yml + SUCCESS: 3 rule files found + SUCCESS: /etc/prometheus/prometheus.yml is valid prometheus config file syntax + +Checking /etc/prometheus/rules/docgrid-backend-alerts.yml + SUCCESS: 3 rules found + +Checking /etc/prometheus/rules/docgrid-pipeline-alerts.yml + SUCCESS: 10 rules found + +Checking /etc/prometheus/rules/embedding-provider-alerts.yml + SUCCESS: 7 rules found + +Checking /etc/prometheus/rules/embedding-provider-alerts.yml + SUCCESS: 7 rules found + +Checking /etc/prometheus/rules/docgrid-backend-alerts.yml + SUCCESS: 3 rules found + +Checking /etc/prometheus/rules/docgrid-pipeline-alerts.yml + SUCCESS: 10 rules found + + SUCCESS + + SUCCESS + + SUCCESS + +Checking /test/rules-firing.yml + SUCCESS: 3 rules found + +Checking /test/rules-resolved.yml + SUCCESS: 3 rules found + +Checking '/etc/alertmanager/alertmanager.yml' SUCCESS +Found: + - global config + - route + - 2 inhibit rules + - 1 receivers + - 0 templates + +Checking '/etc/alertmanager/alertmanager.yml' SUCCESS +Found: + - global config + - route + - 1 inhibit rules + - 1 receivers + - 0 templates + +Checking '/etc/alertmanager/alertmanager.yml' SUCCESS +Found: + - global config + - route + - 2 inhibit rules + - 1 receivers + - 0 templates + +Checking '/etc/alertmanager/alertmanager.yml' SUCCESS +Found: + - global config + - route + - 2 inhibit rules + - 1 receivers + - 0 templates + +Checking '/etc/alertmanager/alertmanager.yml' SUCCESS +Found: + - global config + - route + - 2 inhibit rules + - 1 receivers + - 0 templates + +Checking '/etc/alertmanager/alertmanager.yml' SUCCESS +Found: + - global config + - route + - 2 inhibit rules + - 1 receivers + - 0 templates + +Checking '/etc/alertmanager/alertmanager.yml' SUCCESS +Found: + - global config + - route + - 2 inhibit rules + - 1 receivers + - 0 templates + +Checking '/etc/alertmanager/alertmanager.yml' SUCCESS +Found: + - global config + - route + - 2 inhibit rules + - 1 receivers + - 0 templates + +Checking '/etc/alertmanager/alertmanager.yml' SUCCESS +Found: + - global config + - route + - 2 inhibit rules + - 1 receivers + - 0 templates + +Grafana dashboard validation: SUCCESS (36 panels, 32 PromQL expressions) + Network docgrid-alertmanager-e2e-65857_default Creating + Network docgrid-alertmanager-e2e-65857_default Created + Container docgrid-alertmanager-e2e-65857-webhook-receiver-1 Creating + Container docgrid-alertmanager-e2e-65857-webhook-receiver-1 Created + Container docgrid-alertmanager-e2e-65857-alertmanager-1 Creating + Container docgrid-alertmanager-e2e-65857-alertmanager-1 Created + Container docgrid-alertmanager-e2e-65857-prometheus-1 Creating + Container docgrid-alertmanager-e2e-65857-prometheus-1 Created + Container docgrid-alertmanager-e2e-65857-webhook-receiver-1 Starting + Container docgrid-alertmanager-e2e-65857-webhook-receiver-1 Started + Container docgrid-alertmanager-e2e-65857-webhook-receiver-1 Waiting + Container docgrid-alertmanager-e2e-65857-webhook-receiver-1 Healthy + Container docgrid-alertmanager-e2e-65857-alertmanager-1 Starting + Container docgrid-alertmanager-e2e-65857-alertmanager-1 Started + Container docgrid-alertmanager-e2e-65857-alertmanager-1 Waiting + Container docgrid-alertmanager-e2e-65857-alertmanager-1 Healthy + Container docgrid-alertmanager-e2e-65857-prometheus-1 Starting + Container docgrid-alertmanager-e2e-65857-prometheus-1 Started + Container docgrid-alertmanager-e2e-65857-alertmanager-1 Waiting + Container docgrid-alertmanager-e2e-65857-prometheus-1 Waiting + Container docgrid-alertmanager-e2e-65857-webhook-receiver-1 Waiting + Container docgrid-alertmanager-e2e-65857-alertmanager-1 Healthy + Container docgrid-alertmanager-e2e-65857-webhook-receiver-1 Healthy + Container docgrid-alertmanager-e2e-65857-prometheus-1 Healthy +Alertmanager routing E2E: SUCCESS (firing 2s, resolved 1s) +[시작] run_id=20261003t180051z-65994 grafana_url=http://127.0.0.1:62346 + Network docgrid-grafana-20261003t180051z-65994_default Creating + Network docgrid-grafana-20261003t180051z-65994_default Created + Container docgrid-grafana-20261003t180051z-65994-prometheus-1 Creating + Container docgrid-grafana-20261003t180051z-65994-grafana-1 Creating + Container docgrid-grafana-20261003t180051z-65994-metrics-fixture-1 Creating + Container docgrid-grafana-20261003t180051z-65994-prometheus-1 Created + Container docgrid-grafana-20261003t180051z-65994-metrics-fixture-1 Created + Container docgrid-grafana-20261003t180051z-65994-grafana-1 Created + Container docgrid-grafana-20261003t180051z-65994-grafana-1 Starting + Container docgrid-grafana-20261003t180051z-65994-prometheus-1 Starting + Container docgrid-grafana-20261003t180051z-65994-metrics-fixture-1 Starting + Container docgrid-grafana-20261003t180051z-65994-metrics-fixture-1 Started + Container docgrid-grafana-20261003t180051z-65994-prometheus-1 Started + Container docgrid-grafana-20261003t180051z-65994-grafana-1 Started + Container docgrid-grafana-20261003t180051z-65994-prometheus-1 Waiting + Container docgrid-grafana-20261003t180051z-65994-grafana-1 Waiting + Container docgrid-grafana-20261003t180051z-65994-metrics-fixture-1 Waiting + Container docgrid-grafana-20261003t180051z-65994-prometheus-1 Healthy + Container docgrid-grafana-20261003t180051z-65994-metrics-fixture-1 Healthy + Container docgrid-grafana-20261003t180051z-65994-grafana-1 Healthy +[성공] Grafana Health: database=ok +[성공] Data Source Provisioning: uid=docgrid-prometheus +[성공] Dashboard Provisioning: uid=docgrid-operations, panels=36 +[성공] Grafana→Prometheus 실제 조회: claimable_jobs=4 +[성공] Dashboard PromQL 실행: 32개 +[완료] run_id=20261003t180051z-65994 result=SUCCESS +Monitoring configuration validation: SUCCESS diff --git a/docs/test-results/gimin-#416-grafana-operations-dashboard.md b/docs/test-results/gimin-#416-grafana-operations-dashboard.md new file mode 100644 index 00000000..87e29e3d --- /dev/null +++ b/docs/test-results/gimin-#416-grafana-operations-dashboard.md @@ -0,0 +1,150 @@ +# #416 DocGrid Grafana 운영 Dashboard 검증 결과 + +## 1. 검증 대상 + +| 항목 | 값 | +|---|---| +| Issue | `#416` | +| Branch | `feature/416` | +| 실행 Revision | `22e41b2` | +| 실행 일시 | 2026-10-04 02:56~03:01 KST | +| Host | macOS arm64, Docker Engine 29.4.1, Docker Compose 5.1.3 | +| Runtime | Grafana 13.2.3, Prometheus 3.5.5, Alertmanager 0.33.1, Python 3.14.6 | + +검증 대상은 Grafana Container, Prometheus Data Source와 Dashboard File Provisioning, 운영 Dashboard, +정적 검증기, 실제 Prometheus 연동 E2E다. Java Production Code를 변경하지 않아 Backend Gradle Test는 +실행하지 않았다. + +## 2. 성공 기준 + +1. 번들 Grafana가 관리자 Credential 없이 Loopback 전용 읽기 화면으로 기동한다. +2. `docgrid-prometheus` Data Source와 `docgrid-operations` Dashboard가 UI 조작 없이 등록된다. +3. Dashboard는 7개 Row, 29개 시각 Panel, 3개 조회 변수를 제공한다. +4. Fixture Metric을 Prometheus가 실제 Scrape하고 Grafana Data Source Proxy가 조회한다. +5. Dashboard에 포함된 PromQL 32개가 실제 Prometheus에서 오류 없이 실행된다. +6. 기존 Prometheus Rule과 Alertmanager Routing 검증에 회귀가 없다. +7. 격리 E2E가 생성한 Container와 Volume을 종료 후 모두 회수한다. + +## 3. Test Fixture + +### 3.1 Backend Fixture + +Backend Target은 다음 상태를 포함한 27개 Sample 계열을 1초마다 노출했다. + +- Claim 가능 Embedding Job 4개 +- 지연 Retry Job 2개 +- 처리 중 Embedding Job 1개 +- 활성 Worker 1개 +- Circuit 보호 상태 1, Probe 실패 상태 1 +- 처리 중 RAG 1개 +- Claim 가능 Sync Outbox Event 2개 +- 운영 Snapshot 나이 12초 + +### 3.2 Embedding Provider Fixture + +Provider Target은 다음 상태를 포함한 13개 Sample 계열을 노출했다. + +- Model Loaded 1 +- 대기 요청 1개, Queue 상한 2개 +- RSS 1 GiB, Memory Limit 4 GiB +- 성공 요청 10건, Overload 요청 1건 +- 성공 요청 Histogram 4개 Bucket과 Count·Sum + +Fixture는 운영 환경 측정치가 아니다. Dashboard의 Data Source, Label Matcher, PromQL 실행 경로를 +반복 가능하게 검증하기 위한 결정적 입력이다. + +## 4. 실행 결과 + +| Run | 목적 | 실행 방법 | 주요 결과 | 판정 | +|---|---|---|---|---| +| 01 | 최초 Grafana E2E | `./monitoring/grafana/tests/run-e2e.sh` | Compose Project 이름의 대문자 `T/Z`를 거부, Container 생성 전 종료 | 실패 후 수정 | +| 02 | Grafana 단독 E2E | `./monitoring/grafana/tests/run-e2e.sh` | Health OK, Data Source 1개, Dashboard 36개 Panel, Fixture 값 4, PromQL 32개 성공 | PASS | +| 03 | 전체 Monitoring 회귀 | `./monitoring/verify.sh --e2e` | Rule 20개, Rule Test 3개, Alertmanager 설정 9개, Routing E2E, Grafana E2E 성공 | PASS | +| 시각 검토 | 실제 Browser Rendering | 격리 Grafana에서 `docgrid-operations` 조회 | 세 변수와 7개 Row 표시, 상태 색상·Queue 시계열 렌더링 확인 | PASS | +| 정리 확인 | 격리 자원 누수 검사 | 이름 기준 Container·Volume 조회 | 일치하는 Container 0개, Volume 0개 | PASS | + +### 4.1 최초 실패와 수정 + +첫 실행 ID는 `20261003T175627Z-62642`였다. Run ID를 그대로 Compose Project 이름에 사용하면서 +대문자 `T`와 `Z`가 포함됐고, Docker Compose가 소문자 영문·숫자·하이픈·밑줄 규칙 위반으로 +거부했다. Container와 Volume은 생성되지 않았다. + +Run ID 형식을 `20261003t175700z-63206`처럼 소문자로 바꾼 뒤 같은 E2E를 다시 실행했다. 이 수정은 +Metric이나 Dashboard 동작을 바꾸지 않고 실행 격리 이름만 유효하게 만든다. + +### 4.2 Grafana E2E + +Run 02는 최초 Image Pull을 포함해 2026-10-04 02:56:51~02:57:28 KST에 실행됐다. + +```text +[성공] Grafana Health: database=ok +[성공] Data Source Provisioning: uid=docgrid-prometheus +[성공] Dashboard Provisioning: uid=docgrid-operations, panels=36 +[성공] Grafana→Prometheus 실제 조회: claimable_jobs=4 +[성공] Dashboard PromQL 실행: 32개 +[완료] result=SUCCESS +``` + +36개 Panel에는 구분용 Row 7개와 실제 시각 Panel 29개가 포함된다. `claimable_jobs=4`는 정적 JSON을 +읽은 값이 아니라 다음 실제 경로로 조회한 값이다. + +```text +backend-metrics.txt + → Python HTTP Metric Fixture + → Prometheus 1초 Scrape + → Grafana Provisioned Data Source + → Grafana Data Source Proxy + → Prometheus Query API + → claimable_jobs=4 확인 +``` + +그다음 Dashboard JSON의 Target 32개를 순회하고 `cluster=docgrid-test`, `environment=test`, +`instance=.*`, `$__rate_interval=1m`으로 치환해 전부 실행했다. 이 검사는 JSON 파싱만으로 찾을 수 없는 +PromQL 문법 오류와 Label Matcher 오류를 검출한다. + +### 4.3 전체 Monitoring 회귀 + +Run 03은 2026-10-04 03:00:25~03:01:00 KST에 실행됐다. + +- Prometheus 설정 1개와 Rule 파일 3개 파싱 성공 +- Backend 3개, 비동기 Pipeline 10개, Provider 7개로 총 20개 Rule 검증 성공 +- Prometheus Rule Test 파일 3개 성공 +- 기본·Test·Drill·채널 예시를 합쳐 Alertmanager 설정 9개 검증 성공 +- Alertmanager Routing E2E: Firing 전달 2초, Resolved 전달 1초 +- Grafana 정적 검증: Panel 36개, PromQL 32개 +- Grafana E2E 재실행 성공 +- 최종 출력 `Monitoring configuration validation: SUCCESS` + +## 5. 시각 검토 + +Grafana 13.2.3 Browser 화면에서 다음을 직접 확인했다. + +- `Cluster`, `Environment`, `Instance`가 모두 `All` 기본값으로 표시됨 +- Backend와 Provider가 `UP`, 발생 중 경보가 `0`, Snapshot 나이가 `12s`로 표시됨 +- Claim 가능 Job 4개와 지연 Retry Job 2개는 노란색, 처리 중 Job·활성 Worker는 정상 색상으로 표시됨 +- 수동 Refresh 후 Gauge 시계열이 실제 값으로 렌더링됨 +- 초기 Fixture Scrape 전 `No data`가 남지 않도록 Dashboard Refresh 선택 목록에 `15s`를 명시함 + +Browser Screenshot은 Codex 화면에서 확인했지만 파일 저장에는 성공하지 않아 Repository 증거로 +남기지 않았다. 시각 판정은 보조 검증이며, 재현 가능한 근거는 API E2E Log와 32개 PromQL 실행 +결과다. + +## 6. 보안과 정리 결과 + +- 번들 Grafana Port는 `127.0.0.1`에만 Bind된다. +- 초기 관리자 계정을 생성하지 않고 익명 Viewer만 허용한다. +- Provisioning Data Source와 Dashboard는 UI에서 수정할 수 없다. +- Test Log와 Grafana 파일에서 Password·Token·Authorization Header·Access Key를 검색했고 결과는 0건이었다. +- Run 02·03과 시각 검토가 생성한 격리 Container·Network·Volume을 모두 제거했다. + +공유 서버 또는 외부 Network에서는 조직 Grafana의 인증·TLS·권한 설정을 사용해야 한다. 번들 설정은 +로컬 Loopback 운영 화면의 편의 구성을 검증한 결과다. + +## 7. 증거 파일 + +- `docs/test-results/evidence/issue-416/run-01-compose-project-name-failure.log` +- `docs/test-results/evidence/issue-416/run-02-grafana-e2e.log` +- `docs/test-results/evidence/issue-416/run-03-monitoring-validation.log` + +각 실행 목적과 반복 실행의 Raw Output을 별도 파일로 보존했다. Run 01은 Live File Capture 이전의 +도구 출력을 실행 직후 그대로 옮긴 기록이며, Run 02·03은 `tee`로 실행 중 수집한 Log다. diff --git a/monitoring/grafana/README.md b/monitoring/grafana/README.md new file mode 100644 index 00000000..8f042160 --- /dev/null +++ b/monitoring/grafana/README.md @@ -0,0 +1,89 @@ +# DocGrid Grafana + +DocGrid Grafana는 Prometheus에 저장된 Backend·Embedding Provider·비동기 Pipeline Metric을 한 화면에서 +해석하는 읽기 전용 운영 Dashboard다. 경보의 평가와 외부 전송은 계속 Prometheus와 Alertmanager가 +담당한다. + +## 번들 Dashboard 실행 + +Backend Management Endpoint가 `localhost:8081`에서 실행 중인 상태로 세 Monitoring 서비스를 +기동한다. + +```bash +docker compose --profile monitoring up -d prometheus alertmanager grafana +``` + +- Grafana: +- Prometheus Target: +- Prometheus Alert: +- Alertmanager: + +Grafana는 Prometheus의 준비 여부에 의존하지 않고 시작한다. Prometheus가 나중에 준비되면 +Provisioning된 Data Source가 다시 연결된다. + +## 접근 경계 + +번들 Grafana는 `127.0.0.1`에만 Port를 공개한다. 초기 관리자 계정을 만들지 않고 익명 Viewer만 +허용하므로 Dashboard를 조회할 수 있지만 UI에서 Data Source나 Dashboard를 변경할 수 없다. 설정은 +Git에서 다음 파일을 수정한다. + +- Data Source: `monitoring/grafana/provisioning/datasources/prometheus.yml` +- Dashboard Provider: `monitoring/grafana/provisioning/dashboards/docgrid.yml` +- Dashboard: `monitoring/grafana/dashboards/docgrid-operations.json` + +공유 서버나 외부 Network에 배포할 때는 번들 접근 설정을 그대로 사용하지 않는다. 조직 Grafana의 +인증·TLS·권한 정책을 적용하고 Dashboard JSON을 가져온다. 이때 Prometheus Data Source UID를 +`docgrid-prometheus`로 만들거나 가져오기 과정에서 조직 Data Source로 교체한다. + +## 화면 구성 + +Dashboard는 `cluster`, `environment`, `instance` 순서로 조회 범위를 좁힌다. 기본값 `All`은 같은 +Prometheus가 수집하는 모든 DocGrid 배포를 표시한다. + +| Row | 확인하는 상태 | 주요 판단 | +|---|---|---| +| 전체 상태 | Backend·Provider Scrape, 발생 중 경보, Snapshot 나이 | 수집 중단과 실제 서비스 상태를 먼저 구분한다. | +| Embedding Queue | Claim 가능·지연 Retry·처리 중 Job, Worker, 대기 시간 | Worker 부재, Retry 대기, 처리 정체를 구분한다. | +| Embedding Circuit | OPEN/HALF_OPEN 보호, Probe 실패, 상태 전환 | Provider 요청 차단과 실제 회복 Probe 결과를 확인한다. | +| Embedding Provider | Model, 내부 Queue, Memory, 처리율, p99 | Circuit 원인이 Provider 자원·Queue·지연인지 확인한다. | +| RAG Queue | PROCESSING 수·나이, 완료 결과 | Provider Fallback과 Timeout Sweep 증가를 확인한다. | +| Sync Outbox | Claim 가능·처리 중 Event, 대기 시간, 처리 결과 | 재시도와 최종 실패를 확인한다. | +| Metric 수집 상태 | Snapshot 나이와 갱신 결과 | Dashboard 값 자체가 오래됐는지 확인한다. | + +여러 Backend가 같은 DB를 수집하면 Queue Gauge가 인스턴스마다 같은 상태를 노출한다. Dashboard는 +해당 값을 합산하지 않고 `max by (cluster)`로 표시한다. Counter 기반 Panel은 선택 시간 범위에 맞춘 +`$__rate_interval`을 사용한다. + +## 장애 해석 순서 + +1. 전체 상태에서 Backend·Provider Scrape와 현재 경보를 확인한다. +2. 운영 Snapshot 나이가 계속 증가하면 Queue 값이 최신이 아니므로 Snapshot 갱신 실패부터 조사한다. +3. Embedding Queue에서 Claim 가능 Job, 지연 Retry Job, 활성 Worker를 비교한다. +4. Circuit이 보호 중이면 Probe 실패와 최근 상태 전환을 확인한다. +5. Provider Row에서 Model, 대기 Queue, Memory, p99로 원인을 좁힌다. +6. 복구 후 Circuit과 Probe 실패가 `0`, 지연 Job이 감소하고 처리 성공 Counter가 증가하는지 확인한다. + +Circuit 경보의 정확한 시간 창과 Queue 경보 기준은 +[Prometheus 가이드](../prometheus/README.md)를 따른다. + +## 설정 검증 + +정적 검증은 Dashboard 구조, 필수 Row·변수·Metric, Panel ID·배치, Data Source 참조를 확인한다. + +```bash +python3 monitoring/grafana/validate_dashboard.py +``` + +격리 E2E는 Fixture Metric을 실제 Prometheus가 Scrape하게 하고 Grafana File Provisioning을 거쳐 +Dashboard의 모든 PromQL을 실행한다. 실행마다 동적 Loopback Port와 별도 Compose Project를 사용하며 +종료 시 Container와 Volume을 회수한다. + +```bash +./monitoring/grafana/tests/run-e2e.sh +``` + +Prometheus·Alertmanager 검증까지 포함한 전체 Monitoring 검증은 다음 명령을 사용한다. + +```bash +./monitoring/verify.sh --e2e +``` diff --git a/monitoring/prometheus/README.md b/monitoring/prometheus/README.md index d3f303d3..09f21ac7 100644 --- a/monitoring/prometheus/README.md +++ b/monitoring/prometheus/README.md @@ -15,20 +15,23 @@ Management endpoint는 기본적으로 Host의 `8081` 포트에서 열린다. 실행한다. ```bash -docker compose --profile monitoring up -d prometheus alertmanager +docker compose --profile monitoring up -d prometheus alertmanager grafana ``` Prometheus는 컨테이너에서 `host.docker.internal:8081`을 수집하고 firing·resolved 경보를 같은 profile의 Alertmanager로 전달한다. Compose의 `extra_hosts`가 Linux의 host gateway를 같은 이름으로 연결하며 Docker Desktop도 같은 주소를 지원한다. -두 monitoring 서비스를 명시하는 이유는 Prometheus가 Alertmanager와 Embedding Provider의 health +세 monitoring 서비스를 명시하는 이유는 Prometheus가 Alertmanager와 Embedding Provider의 health 상태에 의존하지 않고 시작하기 때문이다. Embedding Provider가 아직 준비되지 않았거나 중단됐어도 Prometheus는 먼저 기동하고 해당 target을 `DOWN`으로 기록해 장애 경보를 평가한다. - Target 상태: - Alert 상태: - Alertmanager 상태: +- Grafana 운영 Dashboard: + +Grafana의 Panel 구성과 접근 경계는 [Grafana 가이드](../grafana/README.md)를 따른다. 기본 Backend target과 배포 식별 label은 `monitoring/prometheus/targets/docgrid-backend.yml`에서 변경한다. From 8f2a3bddd89b30c1b9884149ef2d18aacc0433d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EA=B8=B0=EB=AF=BC?= Date: Sun, 4 Oct 2026 03:03:22 +0900 Subject: [PATCH 5/6] docs: normalize grafana test evidence --- .../evidence/issue-416/run-02-grafana-e2e.log | 48 +++++----- .../run-03-monitoring-validation.log | 88 +++++++++---------- 2 files changed, 68 insertions(+), 68 deletions(-) diff --git a/docs/test-results/evidence/issue-416/run-02-grafana-e2e.log b/docs/test-results/evidence/issue-416/run-02-grafana-e2e.log index 93a6f17f..d7f18e8f 100644 --- a/docs/test-results/evidence/issue-416/run-02-grafana-e2e.log +++ b/docs/test-results/evidence/issue-416/run-02-grafana-e2e.log @@ -7,8 +7,8 @@ Workload: Backend Fixture 27개 Sample 계열, Provider Fixture 13개 Sample 계 성공 기준: Health·Data Source·Dashboard·Fixture 값·PromQL 32개 통과 및 Container·Volume 정리 [시작] run_id=20261003t175700z-63206 grafana_url=http://127.0.0.1:61878 - Image python:3.14-alpine Pulling - Image grafana/grafana:13.2.3 Pulling + Image python:3.14-alpine Pulling + Image grafana/grafana:13.2.3 Pulling a9986cd6f37d Already exists 0B 541523bae70c Pulling fs layer 0B 9625030534d1 Pulling fs layer 0B @@ -77,7 +77,7 @@ Workload: Backend Fixture 27개 Sample 계열, Provider Fixture 13개 Sample 계 de6af1da8f48 Extracting 248B de6af1da8f48 Extracting 248B de6af1da8f48 Pull complete 0B - Image python:3.14-alpine Pulled + Image python:3.14-alpine Pulled 70c8dc1ba068 Extracting 3.637MB 8998fa436a65 Downloading 16.38kB 8998fa436a65 Downloading 76.02kB @@ -305,27 +305,27 @@ Workload: Backend Fixture 27개 Sample 계열, Provider Fixture 13개 Sample 계 37bc7484dfb5 Extracting 1.333kB 37bc7484dfb5 Extracting 1.333kB 37bc7484dfb5 Pull complete 0B - Image grafana/grafana:13.2.3 Pulled - Network docgrid-grafana-20261003t175700z-63206_default Creating - Network docgrid-grafana-20261003t175700z-63206_default Created - Container docgrid-grafana-20261003t175700z-63206-grafana-1 Creating - Container docgrid-grafana-20261003t175700z-63206-prometheus-1 Creating - Container docgrid-grafana-20261003t175700z-63206-metrics-fixture-1 Creating - Container docgrid-grafana-20261003t175700z-63206-grafana-1 Created - Container docgrid-grafana-20261003t175700z-63206-metrics-fixture-1 Created - Container docgrid-grafana-20261003t175700z-63206-prometheus-1 Created - Container docgrid-grafana-20261003t175700z-63206-grafana-1 Starting - Container docgrid-grafana-20261003t175700z-63206-metrics-fixture-1 Starting - Container docgrid-grafana-20261003t175700z-63206-prometheus-1 Starting - Container docgrid-grafana-20261003t175700z-63206-grafana-1 Started - Container docgrid-grafana-20261003t175700z-63206-prometheus-1 Started - Container docgrid-grafana-20261003t175700z-63206-metrics-fixture-1 Started - Container docgrid-grafana-20261003t175700z-63206-grafana-1 Waiting - Container docgrid-grafana-20261003t175700z-63206-metrics-fixture-1 Waiting - Container docgrid-grafana-20261003t175700z-63206-prometheus-1 Waiting - Container docgrid-grafana-20261003t175700z-63206-metrics-fixture-1 Healthy - Container docgrid-grafana-20261003t175700z-63206-prometheus-1 Healthy - Container docgrid-grafana-20261003t175700z-63206-grafana-1 Healthy + Image grafana/grafana:13.2.3 Pulled + Network docgrid-grafana-20261003t175700z-63206_default Creating + Network docgrid-grafana-20261003t175700z-63206_default Created + Container docgrid-grafana-20261003t175700z-63206-grafana-1 Creating + Container docgrid-grafana-20261003t175700z-63206-prometheus-1 Creating + Container docgrid-grafana-20261003t175700z-63206-metrics-fixture-1 Creating + Container docgrid-grafana-20261003t175700z-63206-grafana-1 Created + Container docgrid-grafana-20261003t175700z-63206-metrics-fixture-1 Created + Container docgrid-grafana-20261003t175700z-63206-prometheus-1 Created + Container docgrid-grafana-20261003t175700z-63206-grafana-1 Starting + Container docgrid-grafana-20261003t175700z-63206-metrics-fixture-1 Starting + Container docgrid-grafana-20261003t175700z-63206-prometheus-1 Starting + Container docgrid-grafana-20261003t175700z-63206-grafana-1 Started + Container docgrid-grafana-20261003t175700z-63206-prometheus-1 Started + Container docgrid-grafana-20261003t175700z-63206-metrics-fixture-1 Started + Container docgrid-grafana-20261003t175700z-63206-grafana-1 Waiting + Container docgrid-grafana-20261003t175700z-63206-metrics-fixture-1 Waiting + Container docgrid-grafana-20261003t175700z-63206-prometheus-1 Waiting + Container docgrid-grafana-20261003t175700z-63206-metrics-fixture-1 Healthy + Container docgrid-grafana-20261003t175700z-63206-prometheus-1 Healthy + Container docgrid-grafana-20261003t175700z-63206-grafana-1 Healthy [성공] Grafana Health: database=ok [성공] Data Source Provisioning: uid=docgrid-prometheus [성공] Dashboard Provisioning: uid=docgrid-operations, panels=36 diff --git a/docs/test-results/evidence/issue-416/run-03-monitoring-validation.log b/docs/test-results/evidence/issue-416/run-03-monitoring-validation.log index 5f473fcf..2d6cb894 100644 --- a/docs/test-results/evidence/issue-416/run-03-monitoring-validation.log +++ b/docs/test-results/evidence/issue-416/run-03-monitoring-validation.log @@ -113,52 +113,52 @@ Found: - 0 templates Grafana dashboard validation: SUCCESS (36 panels, 32 PromQL expressions) - Network docgrid-alertmanager-e2e-65857_default Creating - Network docgrid-alertmanager-e2e-65857_default Created - Container docgrid-alertmanager-e2e-65857-webhook-receiver-1 Creating - Container docgrid-alertmanager-e2e-65857-webhook-receiver-1 Created - Container docgrid-alertmanager-e2e-65857-alertmanager-1 Creating - Container docgrid-alertmanager-e2e-65857-alertmanager-1 Created - Container docgrid-alertmanager-e2e-65857-prometheus-1 Creating - Container docgrid-alertmanager-e2e-65857-prometheus-1 Created - Container docgrid-alertmanager-e2e-65857-webhook-receiver-1 Starting - Container docgrid-alertmanager-e2e-65857-webhook-receiver-1 Started - Container docgrid-alertmanager-e2e-65857-webhook-receiver-1 Waiting - Container docgrid-alertmanager-e2e-65857-webhook-receiver-1 Healthy - Container docgrid-alertmanager-e2e-65857-alertmanager-1 Starting - Container docgrid-alertmanager-e2e-65857-alertmanager-1 Started - Container docgrid-alertmanager-e2e-65857-alertmanager-1 Waiting - Container docgrid-alertmanager-e2e-65857-alertmanager-1 Healthy - Container docgrid-alertmanager-e2e-65857-prometheus-1 Starting - Container docgrid-alertmanager-e2e-65857-prometheus-1 Started - Container docgrid-alertmanager-e2e-65857-alertmanager-1 Waiting - Container docgrid-alertmanager-e2e-65857-prometheus-1 Waiting - Container docgrid-alertmanager-e2e-65857-webhook-receiver-1 Waiting - Container docgrid-alertmanager-e2e-65857-alertmanager-1 Healthy - Container docgrid-alertmanager-e2e-65857-webhook-receiver-1 Healthy - Container docgrid-alertmanager-e2e-65857-prometheus-1 Healthy + Network docgrid-alertmanager-e2e-65857_default Creating + Network docgrid-alertmanager-e2e-65857_default Created + Container docgrid-alertmanager-e2e-65857-webhook-receiver-1 Creating + Container docgrid-alertmanager-e2e-65857-webhook-receiver-1 Created + Container docgrid-alertmanager-e2e-65857-alertmanager-1 Creating + Container docgrid-alertmanager-e2e-65857-alertmanager-1 Created + Container docgrid-alertmanager-e2e-65857-prometheus-1 Creating + Container docgrid-alertmanager-e2e-65857-prometheus-1 Created + Container docgrid-alertmanager-e2e-65857-webhook-receiver-1 Starting + Container docgrid-alertmanager-e2e-65857-webhook-receiver-1 Started + Container docgrid-alertmanager-e2e-65857-webhook-receiver-1 Waiting + Container docgrid-alertmanager-e2e-65857-webhook-receiver-1 Healthy + Container docgrid-alertmanager-e2e-65857-alertmanager-1 Starting + Container docgrid-alertmanager-e2e-65857-alertmanager-1 Started + Container docgrid-alertmanager-e2e-65857-alertmanager-1 Waiting + Container docgrid-alertmanager-e2e-65857-alertmanager-1 Healthy + Container docgrid-alertmanager-e2e-65857-prometheus-1 Starting + Container docgrid-alertmanager-e2e-65857-prometheus-1 Started + Container docgrid-alertmanager-e2e-65857-alertmanager-1 Waiting + Container docgrid-alertmanager-e2e-65857-prometheus-1 Waiting + Container docgrid-alertmanager-e2e-65857-webhook-receiver-1 Waiting + Container docgrid-alertmanager-e2e-65857-alertmanager-1 Healthy + Container docgrid-alertmanager-e2e-65857-webhook-receiver-1 Healthy + Container docgrid-alertmanager-e2e-65857-prometheus-1 Healthy Alertmanager routing E2E: SUCCESS (firing 2s, resolved 1s) [시작] run_id=20261003t180051z-65994 grafana_url=http://127.0.0.1:62346 - Network docgrid-grafana-20261003t180051z-65994_default Creating - Network docgrid-grafana-20261003t180051z-65994_default Created - Container docgrid-grafana-20261003t180051z-65994-prometheus-1 Creating - Container docgrid-grafana-20261003t180051z-65994-grafana-1 Creating - Container docgrid-grafana-20261003t180051z-65994-metrics-fixture-1 Creating - Container docgrid-grafana-20261003t180051z-65994-prometheus-1 Created - Container docgrid-grafana-20261003t180051z-65994-metrics-fixture-1 Created - Container docgrid-grafana-20261003t180051z-65994-grafana-1 Created - Container docgrid-grafana-20261003t180051z-65994-grafana-1 Starting - Container docgrid-grafana-20261003t180051z-65994-prometheus-1 Starting - Container docgrid-grafana-20261003t180051z-65994-metrics-fixture-1 Starting - Container docgrid-grafana-20261003t180051z-65994-metrics-fixture-1 Started - Container docgrid-grafana-20261003t180051z-65994-prometheus-1 Started - Container docgrid-grafana-20261003t180051z-65994-grafana-1 Started - Container docgrid-grafana-20261003t180051z-65994-prometheus-1 Waiting - Container docgrid-grafana-20261003t180051z-65994-grafana-1 Waiting - Container docgrid-grafana-20261003t180051z-65994-metrics-fixture-1 Waiting - Container docgrid-grafana-20261003t180051z-65994-prometheus-1 Healthy - Container docgrid-grafana-20261003t180051z-65994-metrics-fixture-1 Healthy - Container docgrid-grafana-20261003t180051z-65994-grafana-1 Healthy + Network docgrid-grafana-20261003t180051z-65994_default Creating + Network docgrid-grafana-20261003t180051z-65994_default Created + Container docgrid-grafana-20261003t180051z-65994-prometheus-1 Creating + Container docgrid-grafana-20261003t180051z-65994-grafana-1 Creating + Container docgrid-grafana-20261003t180051z-65994-metrics-fixture-1 Creating + Container docgrid-grafana-20261003t180051z-65994-prometheus-1 Created + Container docgrid-grafana-20261003t180051z-65994-metrics-fixture-1 Created + Container docgrid-grafana-20261003t180051z-65994-grafana-1 Created + Container docgrid-grafana-20261003t180051z-65994-grafana-1 Starting + Container docgrid-grafana-20261003t180051z-65994-prometheus-1 Starting + Container docgrid-grafana-20261003t180051z-65994-metrics-fixture-1 Starting + Container docgrid-grafana-20261003t180051z-65994-metrics-fixture-1 Started + Container docgrid-grafana-20261003t180051z-65994-prometheus-1 Started + Container docgrid-grafana-20261003t180051z-65994-grafana-1 Started + Container docgrid-grafana-20261003t180051z-65994-prometheus-1 Waiting + Container docgrid-grafana-20261003t180051z-65994-grafana-1 Waiting + Container docgrid-grafana-20261003t180051z-65994-metrics-fixture-1 Waiting + Container docgrid-grafana-20261003t180051z-65994-prometheus-1 Healthy + Container docgrid-grafana-20261003t180051z-65994-metrics-fixture-1 Healthy + Container docgrid-grafana-20261003t180051z-65994-grafana-1 Healthy [성공] Grafana Health: database=ok [성공] Data Source Provisioning: uid=docgrid-prometheus [성공] Dashboard Provisioning: uid=docgrid-operations, panels=36 From afb0bc75e9e75d1090e42c2f5aca320ad763e9b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EA=B8=B0=EB=AF=BC?= Date: Sun, 4 Oct 2026 03:05:07 +0900 Subject: [PATCH 6/6] docs: record post-rebase monitoring validation --- .../run-04-post-rebase-validation.log | 168 ++++++++++++++++++ ...gimin-#416-grafana-operations-dashboard.md | 27 ++- 2 files changed, 191 insertions(+), 4 deletions(-) create mode 100644 docs/test-results/evidence/issue-416/run-04-post-rebase-validation.log diff --git a/docs/test-results/evidence/issue-416/run-04-post-rebase-validation.log b/docs/test-results/evidence/issue-416/run-04-post-rebase-validation.log new file mode 100644 index 00000000..b41d624f --- /dev/null +++ b/docs/test-results/evidence/issue-416/run-04-post-rebase-validation.log @@ -0,0 +1,168 @@ +테스트 목적: 최신 develop 재배치 후 전체 Monitoring 최종 회귀 검증 +실행 위치: 로컬 Docker Desktop +실행 명령: ./monitoring/verify.sh --e2e +시각대: Asia/Seoul(내부 E2E Run ID는 UTC) +Revision: 8f2a3bd +Workload: Prometheus 설정·Rule·Rule Test, Alertmanager 설정·Routing E2E, Grafana Dashboard 36개 Panel·PromQL 32개 +성공 기준: 최신 develop과 결합된 최종 Branch에서 전체 Monitoring 검증 통과 및 격리 자원 정리 + +Checking /etc/prometheus/prometheus.yml + SUCCESS: 3 rule files found + SUCCESS: /etc/prometheus/prometheus.yml is valid prometheus config file syntax + +Checking /etc/prometheus/rules/docgrid-backend-alerts.yml + SUCCESS: 3 rules found + +Checking /etc/prometheus/rules/docgrid-pipeline-alerts.yml + SUCCESS: 10 rules found + +Checking /etc/prometheus/rules/embedding-provider-alerts.yml + SUCCESS: 7 rules found + +Checking /etc/prometheus/rules/embedding-provider-alerts.yml + SUCCESS: 7 rules found + +Checking /etc/prometheus/rules/docgrid-backend-alerts.yml + SUCCESS: 3 rules found + +Checking /etc/prometheus/rules/docgrid-pipeline-alerts.yml + SUCCESS: 10 rules found + + SUCCESS + + SUCCESS + + SUCCESS + +Checking /test/rules-firing.yml + SUCCESS: 3 rules found + +Checking /test/rules-resolved.yml + SUCCESS: 3 rules found + +Checking '/etc/alertmanager/alertmanager.yml' SUCCESS +Found: + - global config + - route + - 2 inhibit rules + - 1 receivers + - 0 templates + +Checking '/etc/alertmanager/alertmanager.yml' SUCCESS +Found: + - global config + - route + - 1 inhibit rules + - 1 receivers + - 0 templates + +Checking '/etc/alertmanager/alertmanager.yml' SUCCESS +Found: + - global config + - route + - 2 inhibit rules + - 1 receivers + - 0 templates + +Checking '/etc/alertmanager/alertmanager.yml' SUCCESS +Found: + - global config + - route + - 2 inhibit rules + - 1 receivers + - 0 templates + +Checking '/etc/alertmanager/alertmanager.yml' SUCCESS +Found: + - global config + - route + - 2 inhibit rules + - 1 receivers + - 0 templates + +Checking '/etc/alertmanager/alertmanager.yml' SUCCESS +Found: + - global config + - route + - 2 inhibit rules + - 1 receivers + - 0 templates + +Checking '/etc/alertmanager/alertmanager.yml' SUCCESS +Found: + - global config + - route + - 2 inhibit rules + - 1 receivers + - 0 templates + +Checking '/etc/alertmanager/alertmanager.yml' SUCCESS +Found: + - global config + - route + - 2 inhibit rules + - 1 receivers + - 0 templates + +Checking '/etc/alertmanager/alertmanager.yml' SUCCESS +Found: + - global config + - route + - 2 inhibit rules + - 1 receivers + - 0 templates + +Grafana dashboard validation: SUCCESS (36 panels, 32 PromQL expressions) + Network docgrid-alertmanager-e2e-68913_default Creating + Network docgrid-alertmanager-e2e-68913_default Created + Container docgrid-alertmanager-e2e-68913-webhook-receiver-1 Creating + Container docgrid-alertmanager-e2e-68913-webhook-receiver-1 Created + Container docgrid-alertmanager-e2e-68913-alertmanager-1 Creating + Container docgrid-alertmanager-e2e-68913-alertmanager-1 Created + Container docgrid-alertmanager-e2e-68913-prometheus-1 Creating + Container docgrid-alertmanager-e2e-68913-prometheus-1 Created + Container docgrid-alertmanager-e2e-68913-webhook-receiver-1 Starting + Container docgrid-alertmanager-e2e-68913-webhook-receiver-1 Started + Container docgrid-alertmanager-e2e-68913-webhook-receiver-1 Waiting + Container docgrid-alertmanager-e2e-68913-webhook-receiver-1 Healthy + Container docgrid-alertmanager-e2e-68913-alertmanager-1 Starting + Container docgrid-alertmanager-e2e-68913-alertmanager-1 Started + Container docgrid-alertmanager-e2e-68913-alertmanager-1 Waiting + Container docgrid-alertmanager-e2e-68913-alertmanager-1 Healthy + Container docgrid-alertmanager-e2e-68913-prometheus-1 Starting + Container docgrid-alertmanager-e2e-68913-prometheus-1 Started + Container docgrid-alertmanager-e2e-68913-alertmanager-1 Waiting + Container docgrid-alertmanager-e2e-68913-prometheus-1 Waiting + Container docgrid-alertmanager-e2e-68913-webhook-receiver-1 Waiting + Container docgrid-alertmanager-e2e-68913-alertmanager-1 Healthy + Container docgrid-alertmanager-e2e-68913-webhook-receiver-1 Healthy + Container docgrid-alertmanager-e2e-68913-prometheus-1 Healthy +Alertmanager routing E2E: SUCCESS (firing 4s, resolved 2s) +[시작] run_id=20261003t180419z-69064 grafana_url=http://127.0.0.1:62749 + Network docgrid-grafana-20261003t180419z-69064_default Creating + Network docgrid-grafana-20261003t180419z-69064_default Created + Container docgrid-grafana-20261003t180419z-69064-grafana-1 Creating + Container docgrid-grafana-20261003t180419z-69064-prometheus-1 Creating + Container docgrid-grafana-20261003t180419z-69064-metrics-fixture-1 Creating + Container docgrid-grafana-20261003t180419z-69064-prometheus-1 Created + Container docgrid-grafana-20261003t180419z-69064-metrics-fixture-1 Created + Container docgrid-grafana-20261003t180419z-69064-grafana-1 Created + Container docgrid-grafana-20261003t180419z-69064-metrics-fixture-1 Starting + Container docgrid-grafana-20261003t180419z-69064-prometheus-1 Starting + Container docgrid-grafana-20261003t180419z-69064-grafana-1 Starting + Container docgrid-grafana-20261003t180419z-69064-metrics-fixture-1 Started + Container docgrid-grafana-20261003t180419z-69064-grafana-1 Started + Container docgrid-grafana-20261003t180419z-69064-prometheus-1 Started + Container docgrid-grafana-20261003t180419z-69064-grafana-1 Waiting + Container docgrid-grafana-20261003t180419z-69064-metrics-fixture-1 Waiting + Container docgrid-grafana-20261003t180419z-69064-prometheus-1 Waiting + Container docgrid-grafana-20261003t180419z-69064-prometheus-1 Healthy + Container docgrid-grafana-20261003t180419z-69064-metrics-fixture-1 Healthy + Container docgrid-grafana-20261003t180419z-69064-grafana-1 Healthy +[성공] Grafana Health: database=ok +[성공] Data Source Provisioning: uid=docgrid-prometheus +[성공] Dashboard Provisioning: uid=docgrid-operations, panels=36 +[성공] Grafana→Prometheus 실제 조회: claimable_jobs=4 +[성공] Dashboard PromQL 실행: 32개 +[완료] run_id=20261003t180419z-69064 result=SUCCESS +Monitoring configuration validation: SUCCESS diff --git a/docs/test-results/gimin-#416-grafana-operations-dashboard.md b/docs/test-results/gimin-#416-grafana-operations-dashboard.md index 87e29e3d..7ad86ac1 100644 --- a/docs/test-results/gimin-#416-grafana-operations-dashboard.md +++ b/docs/test-results/gimin-#416-grafana-operations-dashboard.md @@ -6,8 +6,8 @@ |---|---| | Issue | `#416` | | Branch | `feature/416` | -| 실행 Revision | `22e41b2` | -| 실행 일시 | 2026-10-04 02:56~03:01 KST | +| 최종 실행 Revision | `8f2a3bd` | +| 실행 일시 | 2026-10-04 02:56~03:04 KST | | Host | macOS arm64, Docker Engine 29.4.1, Docker Compose 5.1.3 | | Runtime | Grafana 13.2.3, Prometheus 3.5.5, Alertmanager 0.33.1, Python 3.14.6 | @@ -60,6 +60,7 @@ Fixture는 운영 환경 측정치가 아니다. Dashboard의 Data Source, Label | 01 | 최초 Grafana E2E | `./monitoring/grafana/tests/run-e2e.sh` | Compose Project 이름의 대문자 `T/Z`를 거부, Container 생성 전 종료 | 실패 후 수정 | | 02 | Grafana 단독 E2E | `./monitoring/grafana/tests/run-e2e.sh` | Health OK, Data Source 1개, Dashboard 36개 Panel, Fixture 값 4, PromQL 32개 성공 | PASS | | 03 | 전체 Monitoring 회귀 | `./monitoring/verify.sh --e2e` | Rule 20개, Rule Test 3개, Alertmanager 설정 9개, Routing E2E, Grafana E2E 성공 | PASS | +| 04 | 최신 develop 재배치 후 최종 회귀 | `./monitoring/verify.sh --e2e` | Run 03과 같은 전체 범위 통과, Routing Firing 4초·Resolved 2초 | PASS | | 시각 검토 | 실제 Browser Rendering | 격리 Grafana에서 `docgrid-operations` 조회 | 세 변수와 7개 Row 표시, 상태 색상·Queue 시계열 렌더링 확인 | PASS | | 정리 확인 | 격리 자원 누수 검사 | 이름 기준 Container·Volume 조회 | 일치하는 Container 0개, Volume 0개 | PASS | @@ -115,6 +116,23 @@ Run 03은 2026-10-04 03:00:25~03:01:00 KST에 실행됐다. - Grafana E2E 재실행 성공 - 최종 출력 `Monitoring configuration validation: SUCCESS` +### 4.4 최신 develop 재배치 후 최종 검증 + +작업 중 `develop`에 PR #415가 머지되어 Grafana 커밋을 최신 `develop` 위로 재배치했다. 재배치 후 +Revision `8f2a3bd`에서 Run 03과 같은 `./monitoring/verify.sh --e2e`를 다시 실행했다. + +- 실행 시간: 2026-10-04 03:03:47~03:04:33 KST +- Prometheus 설정·Rule·Rule Test 전부 성공 +- Alertmanager 설정 검증 성공 +- Alertmanager Routing E2E: Firing 전달 4초, Resolved 전달 2초 +- Grafana Health·Data Source·Dashboard Provisioning 성공 +- Fixture `claimable_jobs=4` 조회 성공 +- Dashboard PromQL 32개 실행 성공 +- 최종 출력 `Monitoring configuration validation: SUCCESS` + +두 Routing 실행의 2~4초 차이는 1초 Polling과 Container 기동 시점에 따른 범위이며, 두 실행 모두 +E2E의 제한 시간 안에서 Firing과 Resolved를 순서대로 전달했다. + ## 5. 시각 검토 Grafana 13.2.3 Browser 화면에서 다음을 직접 확인했다. @@ -135,7 +153,7 @@ Browser Screenshot은 Codex 화면에서 확인했지만 파일 저장에는 성 - 초기 관리자 계정을 생성하지 않고 익명 Viewer만 허용한다. - Provisioning Data Source와 Dashboard는 UI에서 수정할 수 없다. - Test Log와 Grafana 파일에서 Password·Token·Authorization Header·Access Key를 검색했고 결과는 0건이었다. -- Run 02·03과 시각 검토가 생성한 격리 Container·Network·Volume을 모두 제거했다. +- Run 02·03·04와 시각 검토가 생성한 격리 Container·Network·Volume을 모두 제거했다. 공유 서버 또는 외부 Network에서는 조직 Grafana의 인증·TLS·권한 설정을 사용해야 한다. 번들 설정은 로컬 Loopback 운영 화면의 편의 구성을 검증한 결과다. @@ -145,6 +163,7 @@ Browser Screenshot은 Codex 화면에서 확인했지만 파일 저장에는 성 - `docs/test-results/evidence/issue-416/run-01-compose-project-name-failure.log` - `docs/test-results/evidence/issue-416/run-02-grafana-e2e.log` - `docs/test-results/evidence/issue-416/run-03-monitoring-validation.log` +- `docs/test-results/evidence/issue-416/run-04-post-rebase-validation.log` 각 실행 목적과 반복 실행의 Raw Output을 별도 파일로 보존했다. Run 01은 Live File Capture 이전의 -도구 출력을 실행 직후 그대로 옮긴 기록이며, Run 02·03은 `tee`로 실행 중 수집한 Log다. +도구 출력을 실행 직후 그대로 옮긴 기록이며, Run 02·03·04는 `tee`로 실행 중 수집한 Log다.