ux/qol audit sweep: data-correctness fixes, consistency, and QOL

packages/backend/migrations/054_exception_service_attribution.sql

@@ -0,0 +1,71 @@
+-- ============================================================================
+-- Migration 054: Engine-independent service attribution for error groups
+-- The error-group trigger resolved a group's affected service with
+-- `SELECT service FROM logs WHERE id = NEW.log_id`, but on ClickHouse and
+-- MongoDB reservoir backends the ingested logs never land in the Postgres
+-- `logs` table, so the lookup always returned NULL and every error group was
+-- attributed to 'unknown'. The ingestion path already knows the service, so we
+-- carry it on the exception row and have the trigger prefer it, falling back to
+-- the logs lookup (TimescaleDB) and only then to 'unknown'.
+-- ============================================================================
+
+ALTER TABLE exceptions ADD COLUMN IF NOT EXISTS service TEXT;
+
+CREATE OR REPLACE FUNCTION update_error_group_on_exception()
+RETURNS TRIGGER AS $$
+DECLARE
+  v_service TEXT;
+BEGIN
+  -- Prefer the service carried on the exception (known at ingestion, works on
+  -- every storage engine); fall back to the Postgres logs table (TimescaleDB)
+  -- and finally to 'unknown'.
+  v_service := NEW.service;
+
+  IF v_service IS NULL THEN
+    SELECT service INTO v_service
+    FROM logs
+    WHERE id = NEW.log_id
+    LIMIT 1;
+  END IF;
+
+  v_service := COALESCE(v_service, 'unknown');
+
+  -- Insert or update error group
+  INSERT INTO error_groups (
+    organization_id,
+    project_id,
+    fingerprint,
+    exception_type,
+    exception_message,
+    language,
+    occurrence_count,
+    first_seen,
+    last_seen,
+    affected_services,
+    sample_log_id
+  )
+  VALUES (
+    NEW.organization_id,
+    NEW.project_id,
+    NEW.fingerprint,
+    NEW.exception_type,
+    NEW.exception_message,
+    NEW.language,
+    1,
+    NEW.created_at,
+    NEW.created_at,
+    ARRAY[v_service],
+    NEW.log_id
+  )
+  ON CONFLICT (organization_id, COALESCE(project_id, '00000000-0000-0000-0000-000000000000'::UUID), fingerprint)
+  DO UPDATE SET
+    occurrence_count = error_groups.occurrence_count + 1,
+    last_seen = NEW.created_at,
+    affected_services = (
+      SELECT ARRAY(SELECT DISTINCT unnest(array_cat(error_groups.affected_services, ARRAY[v_service])))
+    ),
+    updated_at = NOW();
+
+  RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;

packages/backend/migrations/055_error_group_auto_merge.sql

@@ -0,0 +1,123 @@
+-- ============================================================================
+-- Migration 055: Auto-merge error groups at ingestion
+-- Stack fingerprints split one logical error into several groups when the deep
+-- stack varies (source maps, async internal frames, different callers). We add a
+-- coarser "merge key" (exception type + normalized message + the top application
+-- frame) so a new occurrence folds into an existing group instead of creating a
+-- duplicate. The key is computed by ONE Postgres function used by the backfill,
+-- the trigger, and the runtime fold lookup, so there is no SQL/JS normalization
+-- to keep in sync. See docs/superpowers/specs/2026-07-21-error-group-auto-merge-design.md
+-- ============================================================================
+
+-- Single source of truth for the merge key. IMMUTABLE so it can be used in a
+-- WHERE clause against the merge_key index. Returns NULL when there is no app
+-- frame, so library-only errors are never auto-merged. md5 (not a security
+-- digest, just an internal grouping key) avoids a pgcrypto dependency.
+CREATE OR REPLACE FUNCTION logtide_merge_key(p_type TEXT, p_message TEXT, p_top_frame TEXT)
+RETURNS TEXT AS $$
+DECLARE
+  m TEXT;
+BEGIN
+  IF p_top_frame IS NULL THEN
+    RETURN NULL;
+  END IF;
+
+  m := COALESCE(p_message, '');
+  -- UUIDs first (they contain hex runs the next step would otherwise match).
+  m := regexp_replace(m, '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}', '<uuid>', 'gi');
+  -- Long hex runs (request ids, hashes, addresses).
+  m := regexp_replace(m, '\y[0-9a-f]{8,}\y', '<hex>', 'gi');
+  m := regexp_replace(m, '[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}', '<email>', 'gi');
+  m := regexp_replace(m, '''[^'']*''|"[^"]*"', '<str>', 'g');
+  m := regexp_replace(m, '\d+(\.\d+)?', '<n>', 'g');
+  m := btrim(regexp_replace(m, '\s+', ' ', 'g'));
+
+  RETURN md5(p_type || E'\n' || m || E'\n' || p_top_frame);
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+ALTER TABLE error_groups ADD COLUMN IF NOT EXISTS merge_key TEXT;
+CREATE INDEX IF NOT EXISTS idx_error_groups_merge_key ON error_groups (organization_id, merge_key);
+
+-- Raw "file:function" of the first app frame, carried from the app (which has the
+-- parsed frames) so the trigger can compute the group's merge key.
+ALTER TABLE exceptions ADD COLUMN IF NOT EXISTS top_frame TEXT;
+
+CREATE OR REPLACE FUNCTION update_error_group_on_exception()
+RETURNS TRIGGER AS $$
+DECLARE
+  v_service TEXT;
+  v_merge_key TEXT;
+BEGIN
+  -- Service: prefer the value carried on the exception (works on every storage
+  -- engine), fall back to the Postgres logs table (TimescaleDB), then 'unknown'.
+  v_service := NEW.service;
+  IF v_service IS NULL THEN
+    SELECT service INTO v_service
+    FROM logs
+    WHERE id = NEW.log_id
+    LIMIT 1;
+  END IF;
+  v_service := COALESCE(v_service, 'unknown');
+
+  v_merge_key := logtide_merge_key(NEW.exception_type, NEW.exception_message, NEW.top_frame);
+
+  INSERT INTO error_groups (
+    organization_id,
+    project_id,
+    fingerprint,
+    exception_type,
+    exception_message,
+    language,
+    occurrence_count,
+    first_seen,
+    last_seen,
+    affected_services,
+    sample_log_id,
+    merge_key
+  )
+  VALUES (
+    NEW.organization_id,
+    NEW.project_id,
+    NEW.fingerprint,
+    NEW.exception_type,
+    NEW.exception_message,
+    NEW.language,
+    1,
+    NEW.created_at,
+    NEW.created_at,
+    ARRAY[v_service],
+    NEW.log_id,
+    v_merge_key
+  )
+  ON CONFLICT (organization_id, COALESCE(project_id, '00000000-0000-0000-0000-000000000000'::UUID), fingerprint)
+  DO UPDATE SET
+    occurrence_count = error_groups.occurrence_count + 1,
+    last_seen = NEW.created_at,
+    affected_services = (
+      SELECT ARRAY(SELECT DISTINCT unnest(array_cat(error_groups.affected_services, ARRAY[v_service])))
+    ),
+    merge_key = COALESCE(error_groups.merge_key, v_merge_key),
+    updated_at = NOW();
+
+  RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Backfill existing groups so historical duplicates can fold forward.
+UPDATE error_groups g
+SET merge_key = logtide_merge_key(g.exception_type, g.exception_message, tf.top_frame)
+FROM (
+  SELECT
+    eg.id AS group_id,
+    (
+      SELECT sf.file_path || ':' || COALESCE(sf.function_name, '<anonymous>')
+      FROM exceptions e
+      JOIN stack_frames sf ON sf.exception_id = e.id
+      WHERE e.log_id = eg.sample_log_id AND sf.is_app_code = TRUE
+      ORDER BY sf.frame_index ASC
+      LIMIT 1
+    ) AS top_frame
+  FROM error_groups eg
+) tf
+WHERE g.id = tf.group_id;

packages/backend/src/database/types.ts

@@ -641,6 +641,8 @@ export interface ExceptionsTable {
   fingerprint: string;
   raw_stack_trace: string;
   frame_count: number;
+  service: string | null;
+  top_frame: string | null;
   created_at: Generated<Timestamp>;
 }
 
@@ -689,6 +691,7 @@ export interface ErrorGroupsTable {
   resolved_by: string | null;
   affected_services: string[] | null;
   sample_log_id: string | null;
+  merge_key: string | null;
   last_notified_at: Timestamp | null;
   created_at: Generated<Timestamp>;
   updated_at: Generated<Timestamp>;

packages/backend/src/modules/dashboard/service.ts

@@ -49,6 +49,20 @@ export interface TimelineEvent {
   detectionsBySeverity: { critical: number; high: number; medium: number; low: number };
 }
 
+/**
+ * Below this recent (last-hour) log volume, dashboard "today" stats are computed
+ * from raw logs instead of the hourly continuous aggregate. The aggregate only
+ * reflects data below its materialization watermark that its refresh policy has
+ * actually processed (start_offset = 3h); on instances where the policy is not
+ * keeping the aggregate warm, "today" collapses to roughly the last hour (the
+ * only part read from raw), which is what makes "Total Logs Today" and the error
+ * rate read far too low. Verified live on the hosted demo: 24h = ~1,500 logs,
+ * last hour = ~67, yet "Total Logs Today" showed ~60. A raw count over the
+ * bounded today/yesterday window is exact and cheap at low volume; higher-volume
+ * instances keep the fast aggregate path (their live ingestion keeps it warm).
+ */
+const RAW_STATS_MAX_RECENT_VOLUME = 5000;
+
 class DashboardService {
   /**
    * Resolve project IDs from org or single project.
@@ -163,6 +177,13 @@ class DashboardService {
     });
     const recentVolume = recentTotalResult.count;
 
+    // Low-volume instances (fresh installs, backfilled/seeded data, or a stale
+    // continuous aggregate) are exactly where the aggregate undercounts "today",
+    // so count today/yesterday from raw for an exact, cheap result.
+    if (recentVolume <= RAW_STATS_MAX_RECENT_VOLUME) {
+      return this.getStatsFromRawLogs(projectIds, todayStart, yesterdayStart, lastHourStart, prevHourStart);
+    }
+
     // 2. Query aggregate for historical data and reservoir for recent data in parallel
     const [todayAggregateStats, recentTotal, recentErrors, recentServices, yesterdayAggregateStats, prevHourCount] = await Promise.all([
       // Today's historical stats from aggregate (today start to 1 hour ago)

packages/backend/src/modules/exceptions/detection.ts

@@ -13,6 +13,11 @@ import type { ExceptionLanguage, StructuredException } from '@logtide/shared';
 // Library patterns for detecting vendor/library code
 const LIBRARY_PATTERNS = [
   /node_modules/,
+  // Node.js runtime frames (node:internal/..., node:events, ...). These are not
+  // application code, and because async stack traces vary in which internal
+  // frames they include, treating them as app code makes the same logical error
+  // fingerprint differently and split into several error groups.
+  /^node:/,
   /vendor\//,
   /site-packages/,
   /\.cargo/,

packages/backend/src/modules/exceptions/fingerprint-service.ts

@@ -43,6 +43,20 @@ export class FingerprintService {
     return crypto.createHash('sha256').update(input).digest('hex');
   }
 
+  /**
+   * The throw site: raw "file:function" of the first application frame, or null
+   * when there is none. Used as the coarse-grouping key's frame component (see
+   * migration 055's logtide_merge_key). Kept raw (not path-normalized) so the app
+   * and the SQL function agree without duplicating normalization.
+   */
+  static topAppFrame(parsedException: ParsedException): string | null {
+    const frame = parsedException.frames.find((f) => f.isAppCode);
+    if (!frame) return null;
+    const file = frame.originalFile || frame.filePath;
+    const func = frame.originalFunction || frame.functionName || '<anonymous>';
+    return `${file}:${func}`;
+  }
+
   /**
    * Default normalization when parser is not available
    */

packages/backend/src/modules/exceptions/routes.ts

@@ -567,6 +567,85 @@ export async function exceptionsRoutes(fastify: FastifyInstance) {
     }
   );
 
+  /**
+   * GET /api/v1/error-groups/:id/duplicates
+   * Other groups with the same exception type and message (merge candidates).
+   */
+  fastify.get(
+    '/api/v1/error-groups/:id/duplicates',
+    async (request: any, reply) => {
+      try {
+        const { id } = z.object({ id: z.string().uuid() }).parse(request.params);
+        const { organizationId } = z
+          .object({ organizationId: z.string().uuid() })
+          .parse(request.query);
+
+        const isMember = await checkOrganizationMembership(request.user.id, organizationId);
+        if (!isMember) {
+          return reply.status(403).send({ error: 'You are not a member of this organization' });
+        }
+
+        const group = await exceptionService.getErrorGroupById(id);
+        if (!group || group.organizationId !== organizationId) {
+          return reply.status(404).send({ error: 'Error group not found' });
+        }
+
+        const duplicates = await exceptionService.findDuplicateErrorGroups(id, organizationId);
+        return reply.send({ duplicates });
+      } catch (error: any) {
+        if (error instanceof z.ZodError) {
+          return reply.status(400).send({ error: 'Validation error', details: error.errors });
+        }
+        console.error('Error finding duplicate error groups:', error);
+        return reply.status(500).send({ error: 'Failed to find duplicate error groups' });
+      }
+    }
+  );
+
+  /**
+   * POST /api/v1/error-groups/:id/merge
+   * Merge the given source groups into this group.
+   */
+  fastify.post(
+    '/api/v1/error-groups/:id/merge',
+    {
+      config: { rateLimit: { max: 20, timeWindow: '1 minute' } },
+    },
+    async (request: any, reply) => {
+      try {
+        const { id } = z.object({ id: z.string().uuid() }).parse(request.params);
+        const body = z
+          .object({
+            organizationId: z.string().uuid(),
+            sourceIds: z.array(z.string().uuid()).min(1).max(100),
+          })
+          .parse(request.body);
+
+        const isMember = await checkOrganizationMembership(request.user.id, body.organizationId);
+        if (!isMember) {
+          return reply.status(403).send({ error: 'You are not a member of this organization' });
+        }
+
+        const group = await exceptionService.getErrorGroupById(id);
+        if (!group || group.organizationId !== body.organizationId) {
+          return reply.status(404).send({ error: 'Error group not found' });
+        }
+
+        const merged = await exceptionService.mergeErrorGroups(id, body.sourceIds, body.organizationId);
+        if (!merged) {
+          return reply.status(404).send({ error: 'Error group not found' });
+        }
+        return reply.send(merged);
+      } catch (error: any) {
+        if (error instanceof z.ZodError) {
+          return reply.status(400).send({ error: 'Validation error', details: error.errors });
+        }
+        console.error('Error merging error groups:', error);
+        return reply.status(500).send({ error: 'Failed to merge error groups' });
+      }
+    }
+  );
+
   /**
    * GET /api/v1/error-groups/:id/trend
    * Get error group occurrence trend (time-series)