feat(phase-14): observability — trace propagation, Prometheus/Grafana, metrics and Loki

.env.example

@@ -53,3 +53,9 @@ METADATA_PORT=8002
 UPLOAD_PORT=8003
 NGINX_VOD_PORT=8081
 FRONTEND_PORT=3000
+PROMETHEUS_PORT=9090
+GRAFANA_PORT=3001
+GRAFANA_USER=admin
+GRAFANA_PASSWORD=admin
+TRANSCODER_METRICS_PORT=8004
+LOKI_PORT=3100

Makefile

@@ -3,11 +3,11 @@
 # .env — единственный в корне, compose явно указывает на него (--env-file), deploy/.env не нужен
 COMPOSE=docker compose --env-file .env -f deploy/docker-compose.yml
 
-.PHONY: up down logs ps build lint fmt test e2e e2e-file swagger swagger-install sync-py migrate-up migrate-down migrate-create migrate-alembic-up
+.PHONY: up down logs ps build lint fmt test e2e e2e-file swagger swagger-install sync-py migrate-up migrate-down migrate-create migrate-alembic-up metrics loki-logs grafana prometheus
 
 up:
 	$(COMPOSE) up --build -d
-	@echo "infra: postgres :5432, minio :9000/:9001, rabbit :5672/:15672, nginx-vod :8081, gateway :8080"
+	@echo "infra: postgres :5432, minio :9000/:9001, rabbit :5672/:15672, nginx-vod :8081, gateway :8080, prometheus :9090, grafana :3001, loki :3100, transcoder :8004/metrics"
 
 up-frontend:
 	$(COMPOSE) up --build -d frontend
@@ -21,6 +21,22 @@ logs:
 ps:
 	$(COMPOSE) ps
 
+# Observability (фаза 14) — Prometheus/Grafana/Loki
+metrics:
+	curl -s http://localhost:9090/api/v1/query?query=up | jq 2>/dev/null | head -n 50 || curl -s http://localhost:9090/api/v1/query?query=up | head -n 50
+	@echo "--- gateway/metrics ---"
+	curl -s http://localhost:8080/metrics | grep -E 'rabbitmq_queue_depth|upload_bytes|vod_cache_hit|ffmpeg_duration' | head -n 20 || true
+	curl -s http://localhost:3100/ready 2>/dev/null && echo "loki ready" || echo "loki not ready (http://localhost:3100/ready)"
+
+loki-logs:
+	curl -G -s "http://localhost:3100/loki/api/v1/query" --data-urlencode 'query={service="gateway"}' 2>/dev/null | jq 2>/dev/null | head -n 100 || curl -G -s "http://localhost:3100/loki/api/v1/query" --data-urlencode 'query={service="gateway"}' | head -n 100
+
+grafana:
+	open http://localhost:3001/d/flowix-overview 2>/dev/null || xdg-open http://localhost:3001/d/flowix-overview 2>/dev/null || echo "open http://localhost:3001/d/flowix-overview (admin/admin)"
+
+prometheus:
+	open http://localhost:9090/targets 2>/dev/null || xdg-open http://localhost:9090/targets 2>/dev/null || echo "open http://localhost:9090/targets"
+
 build:
 	$(COMPOSE) build
 

deploy/docker-compose.yml

@@ -198,6 +198,9 @@ services:
       ENCODE_MODE: ${ENCODE_MODE:-cbr}
       FFMPEG_HWACCEL: ${FFMPEG_HWACCEL:-auto}
       TRANSCODER_PIPE_INPUT: ${TRANSCODER_PIPE_INPUT:-true}
+      METRICS_PORT: ${TRANSCODER_METRICS_PORT:-8004}
+    ports:
+      - "${TRANSCODER_METRICS_PORT:-8004}:8004"
     depends_on:
       rabbitmq:
         condition: service_healthy
@@ -257,7 +260,81 @@ services:
       - gateway
     restart: unless-stopped
 
+  prometheus:
+    image: prom/prometheus:v2.55.1
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.path=/prometheus
+      - --web.console.libraries=/usr/share/prometheus/console_libraries
+      - --web.console.templates=/usr/share/prometheus/consoles
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./prometheus/alert.rules.yml:/etc/prometheus/alert.rules.yml:ro
+      - promdata:/prometheus
+    ports:
+      - "${PROMETHEUS_PORT:-9090}:9090"
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://localhost:9090/-/healthy"]
+      interval: 15s
+      timeout: 5s
+      retries: 5
+
+  grafana:
+    image: grafana/grafana:11.4.0
+    environment:
+      GF_SECURITY_ADMIN_USER: ${GRAFANA_USER:-admin}
+      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD:-admin}
+      GF_SERVER_HTTP_PORT: 3000
+    volumes:
+      - grafanadata:/var/lib/grafana
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/etc/grafana/dashboards:ro
+    ports:
+      - "${GRAFANA_PORT:-3001}:3000"
+    depends_on:
+      prometheus:
+        condition: service_healthy
+      loki:
+        condition: service_healthy
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- http://localhost:3000/api/health | grep -q ok || curl -f http://localhost:3000/api/health"]
+      interval: 15s
+      timeout: 5s
+      retries: 10
+
+  loki:
+    image: grafana/loki:3.0.0
+    command: -config.file=/etc/loki/loki.yml
+    volumes:
+      - ./loki/loki.yml:/etc/loki/loki.yml:ro
+      - lokidata:/loki
+    ports:
+      - "${LOKI_PORT:-3100}:3100"
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- http://localhost:3100/ready | grep -q ready || curl -f http://localhost:3100/ready"]
+      interval: 15s
+      timeout: 5s
+      retries: 10
+
+  promtail:
+    image: grafana/promtail:3.0.0
+    command: -config.file=/etc/promtail/promtail.yml
+    volumes:
+      - ./promtail/promtail.yml:/etc/promtail/promtail.yml:ro
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    depends_on:
+      loki:
+        condition: service_healthy
+    restart: unless-stopped
+
 volumes:
   pgdata:
   miniodata:
   rabbitdata:
+  promdata:
+  grafanadata:
+  lokidata:

deploy/grafana/dashboards/flowix.json

@@ -0,0 +1,45 @@
+{
+  "title": "Flowix — Overview",
+  "uid": "flowix-overview",
+  "tags": ["flowix"],
+  "timezone": "browser",
+  "schemaVersion": 30,
+  "version": 1,
+  "panels": [
+    {
+      "type": "stat",
+      "title": "RabbitMQ queue depth",
+      "datasource": "Prometheus",
+      "targets": [{ "expr": "rabbitmq_queue_depth", "instant": true }],
+      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 }
+    },
+    {
+      "type": "graph",
+      "title": "FFmpeg duration (histogram count)",
+      "datasource": "Prometheus",
+      "targets": [{ "expr": "sum by (quality) (increase(ffmpeg_duration_seconds_count[5m]))", "legendFormat": "{{quality}}" }],
+      "gridPos": { "h": 8, "w": 12, "x": 6, "y": 0 }
+    },
+    {
+      "type": "stat",
+      "title": "Upload bytes (total)",
+      "datasource": "Prometheus",
+      "targets": [{ "expr": "sum(upload_bytes)", "instant": true }],
+      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 4 }
+    },
+    {
+      "type": "stat",
+      "title": "VOD cache hits (total)",
+      "datasource": "Prometheus",
+      "targets": [{ "expr": "sum(vod_cache_hit)", "instant": true }],
+      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 4 }
+    },
+    {
+      "type": "graph",
+      "title": "HTTP requests by service (from metrics if available)",
+      "datasource": "Prometheus",
+      "targets": [{ "expr": "sum by (job) (rate(http_requests_total[5m]))", "legendFormat": "{{job}}" }],
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }
+    }
+  ]
+}

deploy/grafana/provisioning/dashboards/dashboard.yml

@@ -0,0 +1,10 @@
+apiVersion: 1
+providers:
+  - name: 'Flowix'
+    orgId: 1
+    folder: ''
+    type: file
+    disableDeletion: false
+    editable: true
+    options:
+      path: /etc/grafana/dashboards

deploy/grafana/provisioning/datasources/datasource.yml

@@ -0,0 +1,13 @@
+apiVersion: 1
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: true
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    editable: true

deploy/loki/loki.yml

@@ -0,0 +1,55 @@
+auth_enabled: false
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+
+common:
+  path_prefix: /loki
+  storage:
+    filesystem:
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+ingester:
+  lifecycler:
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+    final_sleep: 0s
+  chunk_idle_period: 5m
+  chunk_retain_period: 30s
+  wal:
+    enabled: false
+
+schema_config:
+  configs:
+    - from: 2020-05-15
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v11
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  boltdb_shipper:
+    active_index_directory: /loki/index
+    cache_location: /loki/cache
+    cache_ttl: 24h
+  filesystem:
+    directory: /loki/chunks
+
+limits_config:
+  allow_structured_metadata: false
+  volume_enabled: true
+  retention_period: 744h
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+
+analytics:
+  reporting_enabled: false

deploy/prometheus/alert.rules.yml

@@ -0,0 +1,25 @@
+groups:
+  - name: flowix
+    interval: 30s
+    rules:
+      - alert: QueueDepthHigh
+        expr: rabbitmq_queue_messages > 100
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: "RabbitMQ queue depth >100"
+      - alert: TranscoderOOM
+        expr: increase(container_memory_failures_total{name=~"transcoder.*"}[5m]) > 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Transcoder OOM detected"
+      - alert: MinIODiskHigh
+        expr: (minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) < 0.2
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "MinIO disk >80% used"

deploy/prometheus/prometheus.yml

@@ -0,0 +1,45 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: 'gateway'
+    static_configs:
+      - targets: ['gateway:8080']
+    metrics_path: /metrics
+
+  - job_name: 'metadata'
+    static_configs:
+      - targets: ['metadata:8002']
+    metrics_path: /metrics
+
+  - job_name: 'upload'
+    static_configs:
+      - targets: ['upload:8003']
+    metrics_path: /metrics
+
+  - job_name: 'auth'
+    static_configs:
+      - targets: ['auth:8001']
+    metrics_path: /metrics
+
+  - job_name: 'transcoder'
+    static_configs:
+      - targets: ['transcoder:8004']
+    metrics_path: /metrics
+
+  - job_name: 'rabbitmq'
+    static_configs:
+      - targets: ['rabbitmq:15672']
+
+  - job_name: 'minio'
+    static_configs:
+      - targets: ['minio:9000']
+    metrics_path: /minio/v2/metrics/cluster
+
+rule_files:
+  - /etc/prometheus/alert.rules.yml

deploy/promtail/promtail.yml

@@ -0,0 +1,37 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: docker
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 5s
+    relabel_configs:
+      - source_labels: ['__meta_docker_container_name']
+        regex: '/(.*)'
+        target_label: 'container'
+      - source_labels: ['__meta_docker_container_label_com_docker_compose_service']
+        target_label: 'service'
+      - source_labels: ['__meta_docker_container_label_com_docker_compose_project']
+        target_label: 'project'
+      - source_labels: ['__meta_docker_container_log_stream']
+        target_label: 'stream'
+    pipeline_stages:
+      - json:
+          expressions:
+            level: level
+            msg: msg
+            service: service
+            trace_id: trace_id
+            request_id: request_id
+      - labels:
+          level:
+          service:
+          trace_id:

services/auth/pyproject.toml

@@ -17,6 +17,7 @@ dependencies = [
   "pydantic-settings>=2.6,<3",
   "python-multipart>=0.0.9",
   "httpx>=0.27,<0.28",
+  "prometheus_client>=0.21",
 ]
 
 [dependency-groups]