From 8ec8e023b167da80aaae7b361895fe229a3f897e Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 15 Sep 2026 00:15:13 -0700 Subject: [PATCH] fix: keep large message listings within SQLite parameter limits --- CHANGELOG.md | 1 + docs/commands/messages.md | 1 + internal/store/messages.go | 37 +++++++------ internal/store/messages_large_test.go | 80 +++++++++++++++++++++++++++ 4 files changed, 102 insertions(+), 17 deletions(-) create mode 100644 internal/store/messages_large_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 51d16655..7a69cd22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Fix large message listings and TUI reply/mention hydration failing with SQLite's parameter limit, preserving all requested rows and display names. - Update Crawlkit to v0.16.3, retaining the existing Go minimum and SQLite runtime pairing. ## 0.15.1 - 2026-09-13 diff --git a/docs/commands/messages.md b/docs/commands/messages.md index 22b286f3..1058bb45 100644 --- a/docs/commands/messages.md +++ b/docs/commands/messages.md @@ -35,6 +35,7 @@ discrawl --json messages --channel maintainers --days 3 - if `tail` is already running, plain `messages` reads the local archive without waiting; `messages --sync` fails fast instead of waiting behind the tail lock - `--dm` skips Git snapshot auto-update because DMs are never imported from the shared mirror - use either `--last` for the newest matching rows or `--all` for an uncapped oldest-to-newest slice +- uncapped and large listings retain mention display names even beyond 32,766 messages; reply and mention lookups do not impose a separate row cap - ambiguous channel names fail with candidate guild/channel ids; resolve once with `discrawl channels resolve --json` and reuse the numeric id ## Empty results diff --git a/internal/store/messages.go b/internal/store/messages.go index 1748a2ca..88158919 100644 --- a/internal/store/messages.go +++ b/internal/store/messages.go @@ -2,7 +2,11 @@ package store import ( "context" + "database/sql" + "encoding/json" + "maps" "regexp" + "slices" "strings" "time" ) @@ -198,7 +202,7 @@ func (s *Store) hydrateMessageThreadContext(ctx context.Context, rows []MessageR if len(rows) == 0 { return rows, nil } - rootIDs := make([]any, 0, len(rows)) + rootIDs := make([]string, 0, len(rows)) seenRoots := map[string]struct{}{} visible := map[string]struct{}{} for _, row := range rows { @@ -256,9 +260,9 @@ func (s *Store) hydrateMessageThreadContext(ctx context.Context, rows []MessageR left join guilds g on g.id = m.guild_id and g.deleted_at is null left join channels c on c.id = m.channel_id left join members mem on mem.guild_id = m.guild_id and mem.user_id = m.author_id and mem.deleted_at is null - where m.id in (` + placeholders(len(rootIDs)) + `) + where m.id in (select value from json_each(?)) order by m.created_at asc, m.id asc` - contextRows, err := s.db.QueryContext(ctx, query, rootIDs...) + contextRows, err := s.queryMessageIDs(ctx, query, rootIDs) if err != nil { return nil, err } @@ -349,7 +353,7 @@ func (s *Store) resolveMessageDisplayMentions(ctx context.Context, rows []Messag if len(rows) == 0 { return nil } - ids := make([]any, 0, len(rows)) + ids := make([]string, 0, len(rows)) indexByID := make(map[string]int, len(rows)) for index, row := range rows { id := strings.TrimSpace(row.MessageID) @@ -362,8 +366,8 @@ func (s *Store) resolveMessageDisplayMentions(ctx context.Context, rows []Messag if len(ids) == 0 { return nil } - query := `select message_id, target_type, target_id, target_name from mention_events where message_id in (` + placeholders(len(ids)) + `)` - mentionRows, err := s.db.QueryContext(ctx, query, ids...) + query := `select message_id, target_type, target_id, target_name from mention_events where message_id in (select value from json_each(?))` + mentionRows, err := s.queryMessageIDs(ctx, query, ids) if err != nil { return err } @@ -458,7 +462,6 @@ func (s *Store) discordMemberDisplayNames(ctx context.Context, ids map[string]st if len(ids) == 0 { return nil, nil } - args := mapKeysAsAny(ids) query := ` select guild_id, user_id, coalesce( @@ -469,9 +472,9 @@ func (s *Store) discordMemberDisplayNames(ctx context.Context, ids map[string]st '' ) from members - where deleted_at is null and user_id in (` + placeholders(len(args)) + `) + where deleted_at is null and user_id in (select value from json_each(?)) ` - rows, err := s.db.QueryContext(ctx, query, args...) + rows, err := s.queryMessageIDs(ctx, query, slices.Collect(maps.Keys(ids))) if err != nil { return nil, err } @@ -491,9 +494,8 @@ func (s *Store) discordChannelNames(ctx context.Context, ids map[string]struct{} if len(ids) == 0 { return nil, nil } - args := mapKeysAsAny(ids) - query := `select guild_id, id, coalesce(nullif(name, ''), '') from channels where id in (` + placeholders(len(args)) + `)` - rows, err := s.db.QueryContext(ctx, query, args...) + query := `select guild_id, id, coalesce(nullif(name, ''), '') from channels where id in (select value from json_each(?))` + rows, err := s.queryMessageIDs(ctx, query, slices.Collect(maps.Keys(ids))) if err != nil { return nil, err } @@ -509,12 +511,13 @@ func (s *Store) discordChannelNames(ctx context.Context, ids map[string]struct{} return out, rows.Err() } -func mapKeysAsAny(values map[string]struct{}) []any { - out := make([]any, 0, len(values)) - for value := range values { - out = append(out, value) +func (s *Store) queryMessageIDs(ctx context.Context, query string, ids []string) (*sql.Rows, error) { + // Bind one JSON array so uncapped listings do not exhaust SQLite parameters. + encoded, err := json.Marshal(ids) + if err != nil { + return nil, err } - return out + return s.db.QueryContext(ctx, query, string(encoded)) } func rememberResolvedDiscordName(out map[string]string, guildID, id, name string) { diff --git a/internal/store/messages_large_test.go b/internal/store/messages_large_test.go new file mode 100644 index 00000000..fb335aed --- /dev/null +++ b/internal/store/messages_large_test.go @@ -0,0 +1,80 @@ +package store + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMessageHydrationBeyondSQLiteVariableLimit(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + + // SQLite permits 32766 bound parameters; uncapped archive slices can exceed it. + const count = 32767 + _, err = s.db.ExecContext(ctx, ` + with recursive sequence(n) as (select 1 union all select n + 1 from sequence where n < ?) + insert into messages (id, guild_id, channel_id, message_type, created_at, content, normalized_content, raw_json, updated_at) + select printf('m%05d', n), 'g1', 'archive', 0, '2026-09-01T00:00:00.000000000Z', + 'ping <@&role>', 'ping <@&role>', '{}', '2026-09-01T00:00:00.000000000Z' from sequence`, count) + require.NoError(t, err) + _, err = s.db.ExecContext(ctx, ` + insert into mention_events (message_id, guild_id, channel_id, target_type, target_id, target_name, event_at) + select id, guild_id, channel_id, 'role', 'role', 'Readers', created_at from messages`) + require.NoError(t, err) + require.NoError(t, s.UpsertMember(ctx, MemberRecord{GuildID: "g1", UserID: "u32767", Username: "last-user", RoleIDsJSON: `[]`, RawJSON: `{}`})) + require.NoError(t, s.UpsertChannel(ctx, ChannelRecord{ID: "c32767", GuildID: "g1", Kind: "text", Name: "last-channel", RawJSON: `{}`})) + + t.Run("uncapped listing", func(t *testing.T) { + rows, err := s.ListMessages(ctx, MessageListOptions{Channel: "archive"}) + require.NoError(t, err) + require.Len(t, rows, count) + require.Equal(t, "m00001", rows[0].MessageID) + require.Equal(t, "m32767", rows[count-1].MessageID) + for _, row := range rows { + require.Equal(t, "ping @Readers", row.DisplayContent) + } + }) + + t.Run("reply roots", func(t *testing.T) { + replies := make([]MessageRow, count) + for i := range replies { + replies[i] = MessageRow{MessageID: fmt.Sprintf("reply%d", i), GuildID: "g1", ReplyToMessage: fmt.Sprintf("m%05d", i+1)} + } + rows, err := s.hydrateMessageThreadContext(ctx, replies, count) + require.NoError(t, err) + require.Len(t, rows, 2*count) + require.Equal(t, replies, rows[:count]) + require.Equal(t, "m00001", rows[count].MessageID) + require.Equal(t, "m32767", rows[2*count-1].MessageID) + require.Equal(t, "ping @Readers", rows[2*count-1].DisplayContent) + }) + + for _, kind := range []string{"user", "channel"} { + t.Run(kind+" names", func(t *testing.T) { + rows := make([]MessageRow, count) + for i := range rows { + content := fmt.Sprintf("<@u%d>", i+1) + if kind == "channel" { + content = fmt.Sprintf("<#c%d>", i+1) + } + rows[i] = MessageRow{GuildID: "g1", DisplayContent: content} + } + require.NoError(t, s.resolveInlineDiscordMentions(ctx, rows)) + if kind == "user" { + require.Equal(t, "<@u1>", rows[0].DisplayContent) + require.Equal(t, "@last-user", rows[count-1].DisplayContent) + } else { + require.Equal(t, "<#c1>", rows[0].DisplayContent) + require.Equal(t, "#last-channel", rows[count-1].DisplayContent) + } + }) + } +}