diff --git a/src/filesystem/__tests__/lib.test.ts b/src/filesystem/__tests__/lib.test.ts index e0ae61224f..5ce3f286d0 100644 --- a/src/filesystem/__tests__/lib.test.ts +++ b/src/filesystem/__tests__/lib.test.ts @@ -26,6 +26,23 @@ import { vi.mock('fs/promises'); const mockFs = fs as any; +function createMockFileHandle(content: Buffer) { + return { + read: vi.fn( + async (buffer: Buffer, offset: number, length: number, position: number) => { + const bytesRead = content.copy( + buffer, + offset, + position, + Math.min(position + length, content.length), + ); + return { bytesRead, buffer }; + }, + ), + close: vi.fn().mockResolvedValue(undefined), + }; +} + describe('Lib Functions', () => { beforeEach(() => { vi.clearAllMocks(); @@ -643,6 +660,23 @@ describe('Lib Functions', () => { expect(mockFileHandle.close).toHaveBeenCalled(); }); + it('preserves UTF-8 characters split across chunk boundaries', async () => { + const content = Buffer.concat([ + Buffer.from('discard\n'), + Buffer.from('界'), + Buffer.alloc(1017, 'a'), + Buffer.from('\nlast'), + ]); + const mockFileHandle = createMockFileHandle(content); + + mockFs.stat.mockResolvedValue({ size: content.length } as any); + mockFs.open.mockResolvedValue(mockFileHandle); + + const result = await tailFile('/test/file.txt', 2); + + expect(result).toBe(`界${'a'.repeat(1017)}\nlast`); + }); + it('handles read errors gracefully', async () => { mockFs.stat.mockResolvedValue({ size: 100 } as any); @@ -699,6 +733,20 @@ describe('Lib Functions', () => { expect(mockFileHandle.close).toHaveBeenCalled(); }); + it('preserves UTF-8 characters split across chunk boundaries', async () => { + const content = Buffer.concat([ + Buffer.alloc(1023, 'a'), + Buffer.from('界\nsecond'), + ]); + const mockFileHandle = createMockFileHandle(content); + + mockFs.open.mockResolvedValue(mockFileHandle); + + const result = await headFile('/test/file.txt', 1); + + expect(result).toBe(`${'a'.repeat(1023)}界`); + }); + it('handles files with leftover content', async () => { const mockFileHandle = { read: vi.fn(), diff --git a/src/filesystem/lib.ts b/src/filesystem/lib.ts index ce4af9f38a..2738302fde 100644 --- a/src/filesystem/lib.ts +++ b/src/filesystem/lib.ts @@ -2,6 +2,7 @@ import fs from "fs/promises"; import path from "path"; import os from 'os'; import { randomBytes } from 'crypto'; +import { StringDecoder } from 'string_decoder'; import { diffLines, createTwoFilesPatch } from 'diff'; import { minimatch } from 'minimatch'; import { normalizePath, expandHome } from './path-utils.js'; @@ -292,42 +293,28 @@ export async function tailFile(filePath: string, numLines: number): Promise 0 && linesFound < numLines) { + while (position > 0 && newlinesFound < numLines) { const size = Math.min(CHUNK_SIZE, position); position -= size; const { bytesRead } = await fileHandle.read(chunk, 0, size, position); if (!bytesRead) break; - - // Get the chunk as a string and prepend any remaining text from previous iteration - const readData = chunk.slice(0, bytesRead).toString('utf-8'); - const chunkText = readData + remainingText; - - // Split by newlines and count - const chunkLines = normalizeLineEndings(chunkText).split('\n'); - - // If this isn't the end of the file, the first line is likely incomplete - // Save it to prepend to the next chunk - if (position > 0) { - remainingText = chunkLines[0]; - chunkLines.shift(); // Remove the first (incomplete) line - } - - // Add lines to our result (up to the number we need) - for (let i = chunkLines.length - 1; i >= 0 && linesFound < numLines; i--) { - lines.unshift(chunkLines[i]); - linesFound++; + + const readData = Buffer.from(chunk.subarray(0, bytesRead)); + chunks.unshift(readData); + for (const byte of readData) { + if (byte === 0x0a) newlinesFound++; } } - - return lines.join('\n'); + + const text = normalizeLineEndings(Buffer.concat(chunks).toString('utf-8')); + return text.split('\n').slice(-numLines).join('\n'); } finally { await fileHandle.close(); } @@ -341,13 +328,14 @@ export async function headFile(filePath: string, numLines: number): Promise 0 && lines.length < numLines) {