From fe05c00411b77eb8ac7251a2c6ba31e6c659b889 Mon Sep 17 00:00:00 2001 From: "mykola.gervasyuk" Date: Thu, 20 Aug 2026 13:48:43 +0300 Subject: [PATCH 1/9] perf(db): index test runs by build, branch and name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestRun had no indexes at all, so every by-build query — including the previous-run lookup on every screenshot upload — was a sequential scan over the whole table. --- .../20260818134932_add_test_run_build_index/migration.sql | 2 ++ prisma/schema.prisma | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 prisma/migrations/20260818134932_add_test_run_build_index/migration.sql diff --git a/prisma/migrations/20260818134932_add_test_run_build_index/migration.sql b/prisma/migrations/20260818134932_add_test_run_build_index/migration.sql new file mode 100644 index 00000000..9e46a71d --- /dev/null +++ b/prisma/migrations/20260818134932_add_test_run_build_index/migration.sql @@ -0,0 +1,2 @@ +-- CreateIndex +CREATE INDEX "TestRun_buildId_branchName_name_idx" ON "TestRun"("buildId", "branchName", "name"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 97bf2d4f..4835aeae 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -76,6 +76,8 @@ model TestRun { build Build @relation(fields: [buildId], references: [id]) project Project? @relation(fields: [projectId], references: [id]) testVariation TestVariation? @relation(fields: [testVariationId], references: [id]) + + @@index([buildId, branchName, name]) } model TestVariation { From 221f225a09a68b6905b6de043d8fbee33f4b8a02 Mon Sep 17 00:00:00 2001 From: "mykola.gervasyuk" Date: Thu, 20 Aug 2026 13:48:43 +0300 Subject: [PATCH 2/9] perf(events): aggregate build statistics in the database BuildDto only needs per-status counts and a merge flag, but build_updated events and build listing loaded every test-run row of a build (thousands during ingestion, re-fetched on every debounced event burst). Compute the stats with two groupBy queries instead, and deduplicate the queued build ids before querying. --- src/builds/build-stats.ts | 55 +++++++++++++++++++++++++++++ src/builds/builds.service.spec.ts | 21 +++++++++-- src/builds/builds.service.ts | 10 +++--- src/builds/dto/build.dto.ts | 19 ++++++---- src/shared/events/events.gateway.ts | 33 ++++++++--------- 5 files changed, 104 insertions(+), 34 deletions(-) create mode 100644 src/builds/build-stats.ts diff --git a/src/builds/build-stats.ts b/src/builds/build-stats.ts new file mode 100644 index 00000000..b4355367 --- /dev/null +++ b/src/builds/build-stats.ts @@ -0,0 +1,55 @@ +import { TestStatus } from '@prisma/client'; +import { PrismaService } from '../prisma/prisma.service'; + +export interface BuildStats { + passedCount: number; + unresolvedCount: number; + failedCount: number; + merge: boolean; +} + +const PASSED_STATUSES: TestStatus[] = [TestStatus.ok, TestStatus.approved, TestStatus.autoApproved]; +const UNRESOLVED_STATUSES: TestStatus[] = [TestStatus.unresolved, TestStatus.new]; + +/** + * Aggregates test-run statistics per build inside the database instead of + * loading every test-run row into the API process: builds can hold thousands + * of runs and these stats are recomputed on every build_updated event burst + * during ingestion. + */ +export async function getBuildsStats(prisma: PrismaService, buildIds: string[]): Promise> { + const stats = new Map( + buildIds.map((id) => [id, { passedCount: 0, unresolvedCount: 0, failedCount: 0, merge: false }]) + ); + if (buildIds.length === 0) { + return stats; + } + + const [statusCounts, mergeBuilds] = await Promise.all([ + prisma.testRun.groupBy({ + by: ['buildId', 'status'], + where: { buildId: { in: buildIds } }, + _count: { _all: true }, + }), + prisma.testRun.groupBy({ + by: ['buildId'], + where: { buildId: { in: buildIds }, merge: true }, + }), + ]); + + for (const row of statusCounts) { + const buildStats = stats.get(row.buildId); + const count = row._count._all; + if (PASSED_STATUSES.includes(row.status)) { + buildStats.passedCount += count; + } else if (UNRESOLVED_STATUSES.includes(row.status)) { + buildStats.unresolvedCount += count; + } else if (row.status === TestStatus.failed) { + buildStats.failedCount += count; + } + } + for (const row of mergeBuilds) { + stats.get(row.buildId).merge = true; + } + return stats; +} diff --git a/src/builds/builds.service.spec.ts b/src/builds/builds.service.spec.ts index 542955b1..87afec47 100644 --- a/src/builds/builds.service.spec.ts +++ b/src/builds/builds.service.spec.ts @@ -23,6 +23,7 @@ const initService = async ({ testRunDeleteMock = jest.fn(), testRunApproveMock = jest.fn(), testRunFindManyMock = jest.fn(), + testRunGroupByMock = jest.fn().mockResolvedValue([]), eventsBuildUpdatedMock = jest.fn(), eventsBuildCreatedMock = jest.fn(), eventBuildDeletedMock = jest.fn(), @@ -47,6 +48,9 @@ const initService = async ({ upsert: buildUpsertMock, count: buildCountMock, }, + testRun: { + groupBy: testRunGroupByMock, + }, }, }, { @@ -115,13 +119,24 @@ describe('BuildsService', () => { it('findOne', async () => { const buildFindUniqueMock = jest.fn().mockResolvedValueOnce(build); - const testRunFindManyMock = jest.fn().mockResolvedValueOnce(build.testRuns); + const testRunGroupByMock = jest + .fn() + .mockResolvedValueOnce([ + { buildId: 'someId', status: TestStatus.ok, _count: { _all: 2 } }, + { buildId: 'someId', status: TestStatus.unresolved, _count: { _all: 1 } }, + ]) + .mockResolvedValueOnce([{ buildId: 'someId' }]); mocked(BuildDto).mockReturnValueOnce(buildDto as MockedObject); - service = await initService({ buildFindUniqueMock, testRunFindManyMock }); + service = await initService({ buildFindUniqueMock, testRunGroupByMock }); const result = await service.findOne('someId'); - expect(mocked(BuildDto)).toHaveBeenCalledWith({ ...build, testRuns: build.testRuns }); + expect(mocked(BuildDto)).toHaveBeenCalledWith(build, { + passedCount: 2, + unresolvedCount: 1, + failedCount: 0, + merge: true, + }); expect(result).toBe(buildDto); }); diff --git a/src/builds/builds.service.ts b/src/builds/builds.service.ts index bdc51604..1add9bc3 100644 --- a/src/builds/builds.service.ts +++ b/src/builds/builds.service.ts @@ -4,6 +4,7 @@ import { Build, Prisma, TestStatus } from '@prisma/client'; import { TestRunsService } from '../test-runs/test-runs.service'; import { EventsGateway } from '../shared/events/events.gateway'; import { BuildDto } from './dto/build.dto'; +import { getBuildsStats } from './build-stats'; import { PaginatedBuildDto } from './dto/build-paginated.dto'; import { ModifyBuildDto } from './dto/build-modify.dto'; @@ -20,16 +21,13 @@ export class BuildsService { ) {} async findOne(id: string): Promise { - const [build, testRuns] = await Promise.all([ + const [build, stats] = await Promise.all([ this.prismaService.build.findUnique({ where: { id }, }), - this.testRunsService.findMany(id), + getBuildsStats(this.prismaService, [id]), ]); - return new BuildDto({ - ...build, - testRuns, - }); + return new BuildDto(build, stats.get(id)); } async findMany(projectId: string, take: number, skip: number, ciBuildId?: string): Promise { diff --git a/src/builds/dto/build.dto.ts b/src/builds/dto/build.dto.ts index cc67073c..7529ae6a 100644 --- a/src/builds/dto/build.dto.ts +++ b/src/builds/dto/build.dto.ts @@ -1,6 +1,7 @@ import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; import { Build, TestStatus } from '@prisma/client'; import { TestRunDto } from 'src/test-runs/dto/testRun.dto'; +import { BuildStats } from '../build-stats'; export class BuildDto { @ApiProperty() @@ -41,7 +42,7 @@ export class BuildDto { @ApiProperty() merge: boolean; - constructor(build: Build & { testRuns?: TestRunDto[] }) { + constructor(build: Build & { testRuns?: TestRunDto[] }, stats?: BuildStats) { this.id = build.id; this.ciBuildId = build.ciBuildId; this.number = build.number; @@ -58,7 +59,12 @@ export class BuildDto { this.failedCount = 0; this.merge = false; - if (build.testRuns) { + if (stats) { + this.passedCount = stats.passedCount; + this.unresolvedCount = stats.unresolvedCount; + this.failedCount = stats.failedCount; + this.merge = stats.merge; + } else if (build.testRuns) { // determine if merge this.merge = build.testRuns.some((testRun) => testRun.merge); @@ -84,11 +90,10 @@ export class BuildDto { }); } - if (!build.testRuns || build.testRuns.length === 0) { - this.status = 'new'; - } else { - this.status = 'passed'; - } + const hasTestRuns = stats + ? this.passedCount + this.unresolvedCount + this.failedCount > 0 + : !!build.testRuns && build.testRuns.length > 0; + this.status = hasTestRuns ? 'passed' : 'new'; if (this.failedCount > 0) { this.status = 'failed'; } diff --git a/src/shared/events/events.gateway.ts b/src/shared/events/events.gateway.ts index 1503f566..ea8c7ec8 100644 --- a/src/shared/events/events.gateway.ts +++ b/src/shared/events/events.gateway.ts @@ -1,7 +1,8 @@ import { WebSocketGateway, WebSocketServer } from '@nestjs/websockets'; import { Server } from 'socket.io'; -import { Build, TestRun } from '@prisma/client'; +import { TestRun } from '@prisma/client'; import { BuildDto } from '../../builds/dto/build.dto'; +import { getBuildsStats } from '../../builds/build-stats'; import { debounce } from 'lodash'; import { PrismaService } from '../../prisma/prisma.service'; @@ -90,24 +91,20 @@ export class EventsGateway { private buildUpdatedDebounced = debounce( () => { - this.prismaService.build - .findMany({ - where: { - id: { - in: this.buildsUpdatedQueued, - }, - }, - include: { - testRuns: true, - }, - }) - .then((builds: Array) => { - this.server.emit( - 'build_updated', - builds.map((build: Build) => new BuildDto(build)) - ); - }); + // Deduplicate: during ingestion the queue holds one entry per test run. + // Stats are aggregated in the database — a build can hold thousands of + // test runs and this fires on every debounced event burst. + const buildIds = [...new Set(this.buildsUpdatedQueued)]; this.buildsUpdatedQueued = []; + Promise.all([ + this.prismaService.build.findMany({ where: { id: { in: buildIds } } }), + getBuildsStats(this.prismaService, buildIds), + ]).then(([builds, stats]) => { + this.server.emit( + 'build_updated', + builds.map((build) => new BuildDto(build, stats.get(build.id))) + ); + }); }, this.debounceTimeout, { From cd058a516fd5bd5d13317d1fd09cc64a3879ca0e Mon Sep 17 00:00:00 2001 From: "mykola.gervasyuk" Date: Thu, 20 Aug 2026 13:48:55 +0300 Subject: [PATCH 3/9] perf(static): stop decoding PNGs where raw bytes suffice - validate uploads by PNG signature instead of a full decode - expose getImageBuffer so callers that only need bytes skip decoding - copy baselines byte-for-byte on approve (copyImage) instead of decode + re-encode of a full screenshot per approved run - switch HDD storage to async fs calls All of this ran synchronously on the event loop for every upload and approve, stalling unrelated requests during build ingestion. --- src/static/aws/s3.service.ts | 35 ++++++++++++++- src/static/hdd/hdd.service.ts | 44 ++++++++++++------- src/static/static.interface.ts | 1 + src/static/static.service.ts | 4 ++ src/test-runs/test-runs.service.ts | 6 +-- .../test-variations.service.spec.ts | 17 ++++--- .../test-variations.service.ts | 7 ++- 7 files changed, 83 insertions(+), 31 deletions(-) diff --git a/src/static/aws/s3.service.ts b/src/static/aws/s3.service.ts index 9b527027..55820893 100644 --- a/src/static/aws/s3.service.ts +++ b/src/static/aws/s3.service.ts @@ -1,7 +1,13 @@ import { PNG, PNGWithMetadata } from 'pngjs'; import { Logger } from '@nestjs/common'; import { Static } from '../static.interface'; -import { DeleteObjectCommand, GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { + CopyObjectCommand, + DeleteObjectCommand, + GetObjectCommand, + PutObjectCommand, + S3Client, +} from '@aws-sdk/client-s3'; import { Readable } from 'stream'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import { generateNewImageName } from '../utils'; @@ -35,6 +41,17 @@ export class AWSS3Service implements Static { } async getImage(fileName: string): Promise { + if (!fileName) return null; + const imageBuffer = await this.getImageBuffer(fileName); + if (!imageBuffer) return undefined; + try { + return PNG.sync.read(imageBuffer); + } catch (ex) { + this.logger.error(`Error from read : Cannot decode image: ${fileName}. ${ex}`); + } + } + + async getImageBuffer(fileName: string): Promise { if (!fileName) return null; try { // the comparison pipeline treats an unreadable image as a missing @@ -65,6 +82,22 @@ export class AWSS3Service implements Static { } } + async copyImage(type: 'screenshot' | 'diff' | 'baseline', sourceImageName: string): Promise { + const imageName = generateNewImageName(type); + try { + await this.s3Client.send( + new CopyObjectCommand({ + Bucket: this.AWS_S3_BUCKET_NAME, + CopySource: `${this.AWS_S3_BUCKET_NAME}/${sourceImageName}`, + Key: imageName, + }) + ); + return imageName; + } catch (ex) { + throw new Error('Could not copy file at AWS S3 : ' + ex); + } + } + async getImageUrl(imageName: string): Promise { const command = new GetObjectCommand({ Bucket: `${this.AWS_S3_BUCKET_NAME}`, diff --git a/src/static/hdd/hdd.service.ts b/src/static/hdd/hdd.service.ts index 30a48deb..a08c6151 100644 --- a/src/static/hdd/hdd.service.ts +++ b/src/static/hdd/hdd.service.ts @@ -1,11 +1,13 @@ import { Logger } from '@nestjs/common'; import path from 'path'; -import { writeFileSync, readFileSync, unlink, mkdirSync, existsSync } from 'fs'; +import { promises as fs, mkdirSync, existsSync } from 'fs'; import { PNG, PNGWithMetadata } from 'pngjs'; import { Static } from '../static.interface'; import { HDD_IMAGE_PATH } from './constants'; import { generateNewImageName } from '../utils'; +const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + export class HddService implements Static { private readonly logger: Logger = new Logger(HddService.name); @@ -39,30 +41,34 @@ export class HddService implements Static { } async saveImage(type: 'screenshot' | 'diff' | 'baseline', imageBuffer: Buffer): Promise { - try { - new PNG().parse(imageBuffer); - } catch { + // Signature check instead of a full decode: parsing megapixel PNGs just to + // validate them blocks the event loop on every upload. + if ( + imageBuffer.length < PNG_SIGNATURE.length || + !imageBuffer.subarray(0, PNG_SIGNATURE.length).equals(PNG_SIGNATURE) + ) { throw new Error('Cannot parse image as PNG file'); } const { imageName, imagePath } = this.generateNewImage(type); - writeFileSync(imagePath, new Uint8Array(imageBuffer.buffer, imageBuffer.byteOffset, imageBuffer.byteLength)); + await fs.writeFile(imagePath, new Uint8Array(imageBuffer.buffer, imageBuffer.byteOffset, imageBuffer.byteLength)); return imageName; } async getImage(imageName: string): Promise { - if (!imageName) return; + const imageBuffer = await this.getImageBuffer(imageName); + if (!imageBuffer) return; try { - return PNG.sync.read(readFileSync(this.getImagePath(imageName))); + return PNG.sync.read(imageBuffer); } catch (ex) { - this.logger.error(`Cannot get image: ${imageName}. ${ex}`); + this.logger.error(`Cannot decode image: ${imageName}. ${ex}`); } } async getImageBuffer(imageName: string): Promise { if (!imageName) return null; try { - return readFileSync(this.getImagePath(imageName)); + return await fs.readFile(this.getImagePath(imageName)); } catch (ex) { this.logger.error(`Cannot get image: ${imageName}. ${ex}`); // an absent file is the only case that means "no image"; a permission or @@ -74,16 +80,20 @@ export class HddService implements Static { } } + async copyImage(type: 'screenshot' | 'diff' | 'baseline', sourceImageName: string): Promise { + const { imageName, imagePath } = this.generateNewImage(type); + await fs.copyFile(this.getImagePath(sourceImageName), imagePath); + return imageName; + } + async deleteImage(imageName: string): Promise { if (!imageName) return; - return new Promise((resolvePromise) => { - unlink(this.getImagePath(imageName), (err) => { - if (err) { - this.logger.error(err); - } - resolvePromise(true); - }); - }); + try { + await fs.unlink(this.getImagePath(imageName)); + } catch (err) { + this.logger.error(err); + } + return true; } private ensureDirectoryExistence(dir: string) { diff --git a/src/static/static.interface.ts b/src/static/static.interface.ts index 9e0a0629..92c4b6b1 100644 --- a/src/static/static.interface.ts +++ b/src/static/static.interface.ts @@ -4,6 +4,7 @@ export interface Static { saveImage(type: 'screenshot' | 'diff' | 'baseline', imageBuffer: Buffer): Promise; getImage(fileName: string): Promise; getImageBuffer(fileName: string): Promise; + copyImage(type: 'screenshot' | 'diff' | 'baseline', sourceImageName: string): Promise; deleteImage(imageName: string): Promise; getImageUrl(imageName: string): Promise; } diff --git a/src/static/static.service.ts b/src/static/static.service.ts index ec3d6c11..345aebfc 100644 --- a/src/static/static.service.ts +++ b/src/static/static.service.ts @@ -24,6 +24,10 @@ export class StaticService { return this.staticService.getImageBuffer(imageName); } + async copyImage(type: 'screenshot' | 'diff' | 'baseline', sourceImageName: string): Promise { + return this.staticService.copyImage(type, sourceImageName); + } + async deleteImage(imageName: string): Promise { return this.staticService.deleteImage(imageName); } diff --git a/src/test-runs/test-runs.service.ts b/src/test-runs/test-runs.service.ts index 2a73961d..44ae2926 100644 --- a/src/test-runs/test-runs.service.ts +++ b/src/test-runs/test-runs.service.ts @@ -106,9 +106,9 @@ export class TestRunsService { throw new Error('No test variation found. Re-create test run'); } - // save new baseline - const baseline = await this.staticService.getImage(testRun.imageName); - const baselineName = await this.staticService.saveImage('baseline', PNG.sync.write(baseline)); + // save new baseline as a byte-for-byte copy — decoding and re-encoding the + // PNG here blocked the event loop for every approved run + const baselineName = await this.staticService.copyImage('baseline', testRun.imageName); if (testRun.baselineBranchName !== testRun.branchName && !merge && !autoApprove) { // replace main branch with feature branch test variation diff --git a/src/test-variations/test-variations.service.spec.ts b/src/test-variations/test-variations.service.spec.ts index 28d4a8e0..271f169b 100644 --- a/src/test-variations/test-variations.service.spec.ts +++ b/src/test-variations/test-variations.service.spec.ts @@ -12,7 +12,7 @@ import { TestVariationUpdateDto } from './dto/test-variation-update.dto'; const initModule = async ({ imageDeleteMock = jest.fn(), - getImageMock = jest.fn(), + getImageBufferMock = jest.fn(), variationfindUniqueMock = jest.fn, variationFindManyMock = jest.fn().mockReturnValue([]), variationCreateMock = jest.fn(), @@ -34,7 +34,7 @@ const initModule = async ({ { provide: StaticService, useValue: { - getImage: getImageMock, + getImageBuffer: getImageBufferMock, deleteImage: imageDeleteMock, }, }, @@ -462,7 +462,12 @@ describe('TestVariationsService', () => { width: 10, height: 10, }); - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(image).mockReturnValueOnce(null); + const imageBuffer = PNG.sync.write(image); + const getImageBufferMock = jest + .fn() + .mockReturnValueOnce(imageBuffer) + .mockReturnValueOnce(imageBuffer) + .mockReturnValueOnce(null); const testRunCreateMock = jest.fn(); const buildUpdateMock = jest.fn(); const testRuncalCulateDiffMock = jest.fn(); @@ -472,7 +477,7 @@ describe('TestVariationsService', () => { testRunCreateMock, testRuncalCulateDiffMock, variationFindManyMock, - getImageMock, + getImageBufferMock, }); service.find = jest .fn() @@ -519,7 +524,7 @@ describe('TestVariationsService', () => { merge: true, ignoreAreas: JSON.parse(testVariation.ignoreAreas), }, - imageBuffer: PNG.sync.write(image), + imageBuffer, }); expect(testRunCreateMock).toHaveBeenNthCalledWith(2, { testVariation: testVariationTargetBranch, @@ -531,7 +536,7 @@ describe('TestVariationsService', () => { merge: true, ignoreAreas: JSON.parse(testVariationSecond.ignoreAreas), }, - imageBuffer: PNG.sync.write(image), + imageBuffer, }); expect(testRunCreateMock).toHaveBeenCalledTimes(2); expect(buildUpdateMock).toHaveBeenCalledWith(build.id, { isRunning: false }); diff --git a/src/test-variations/test-variations.service.ts b/src/test-variations/test-variations.service.ts index 3767588d..c5915409 100644 --- a/src/test-variations/test-variations.service.ts +++ b/src/test-variations/test-variations.service.ts @@ -4,7 +4,6 @@ import { TestVariation, Baseline, Build, TestRun, User } from '@prisma/client'; import { StaticService } from '../static/static.service'; import { BuildsService } from '../builds/builds.service'; import { TestRunsService } from '../test-runs/test-runs.service'; -import { PNG } from 'pngjs'; import { CreateTestRequestDto } from 'src/test-runs/dto/create-test-request.dto'; import { BuildDto } from 'src/builds/dto/build.dto'; import { getTestVariationUniqueData } from '../utils'; @@ -183,8 +182,8 @@ export class TestVariationsService { // compare source to destination branch variations for (const sourceBranchTestVariation of testVariations) { - const baseline = await this.staticService.getImage(sourceBranchTestVariation.baselineName); - if (baseline) { + const baselineBuffer = await this.staticService.getImageBuffer(sourceBranchTestVariation.baselineName); + if (baselineBuffer) { // get destination branch request const createTestRequestDto: CreateTestRequestDto = { ...sourceBranchTestVariation, @@ -209,7 +208,7 @@ export class TestVariationsService { const testRun = await this.testRunsService.create({ testVariation: destintionBranchTestVariation, createTestRequestDto, - imageBuffer: PNG.sync.write(baseline), + imageBuffer: baselineBuffer, }); await this.testRunsService.calculateDiff(projectId, testRun); From eb950ebc8b83beef507e24fb98e41770daa10d2a Mon Sep 17 00:00:00 2001 From: "mykola.gervasyuk" Date: Thu, 20 Aug 2026 13:48:55 +0300 Subject: [PATCH 4/9] perf(compare): run pixelmatch diffs in worker threads Decode, pixelmatch and diff encode of full-size screenshots block the event loop for around a second each, which makes the whole API unresponsive while a build is ingesting. Run the CPU-bound part in a fixed worker_threads pool (cores - 1, capped at 8, DIFF_WORKERS_COUNT to override) and keep only async I/O on the main thread. Falls back to inline execution where the compiled worker file does not exist (ts-jest / ts-node). --- src/compare/compare.module.ts | 2 + src/compare/compare.service.spec.ts | 2 + src/compare/diff-worker-pool.ts | 114 ++++++++++++++++++ .../libs/pixelmatch/pixelmatch.core.ts | 77 ++++++++++++ .../pixelmatch/pixelmatch.service.spec.ts | 36 +++--- .../libs/pixelmatch/pixelmatch.service.ts | 74 ++++++------ .../libs/pixelmatch/pixelmatch.worker.ts | 10 ++ 7 files changed, 261 insertions(+), 54 deletions(-) create mode 100644 src/compare/diff-worker-pool.ts create mode 100644 src/compare/libs/pixelmatch/pixelmatch.core.ts create mode 100644 src/compare/libs/pixelmatch/pixelmatch.worker.ts diff --git a/src/compare/compare.module.ts b/src/compare/compare.module.ts index 7444f496..87d5aa14 100644 --- a/src/compare/compare.module.ts +++ b/src/compare/compare.module.ts @@ -8,11 +8,13 @@ import { OllamaController } from './libs/vlm/providers/ollama/ollama.controller' import { OllamaService } from './libs/vlm/providers/ollama/ollama.service'; import { GeminiService } from './libs/vlm/providers/gemini/gemini.service'; import { StaticModule } from '../static/static.module'; +import { DiffWorkerPool } from './diff-worker-pool'; @Module({ controllers: [OllamaController], providers: [ CompareService, + DiffWorkerPool, PixelmatchService, LookSameService, OdiffService, diff --git a/src/compare/compare.service.spec.ts b/src/compare/compare.service.spec.ts index 1ade4925..a688875b 100644 --- a/src/compare/compare.service.spec.ts +++ b/src/compare/compare.service.spec.ts @@ -2,6 +2,7 @@ import { Test, TestingModule } from '@nestjs/testing'; import { ConfigService } from '@nestjs/config'; import { PrismaService } from '../prisma/prisma.service'; import { CompareService } from './compare.service'; +import { DiffWorkerPool } from './diff-worker-pool'; import { LookSameService } from './libs/looks-same/looks-same.service'; import { OdiffService } from './libs/odiff/odiff.service'; import { PixelmatchService } from './libs/pixelmatch/pixelmatch.service'; @@ -27,6 +28,7 @@ describe('CompareService', () => { const module: TestingModule = await Test.createTestingModule({ providers: [ CompareService, + DiffWorkerPool, OdiffService, PixelmatchService, LookSameService, diff --git a/src/compare/diff-worker-pool.ts b/src/compare/diff-worker-pool.ts new file mode 100644 index 00000000..e6ae9b2b --- /dev/null +++ b/src/compare/diff-worker-pool.ts @@ -0,0 +1,114 @@ +import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; +import { Worker } from 'worker_threads'; +import { availableParallelism } from 'os'; +import { existsSync } from 'fs'; +import { join } from 'path'; +import { computePixelmatchDiff, PixelmatchJobInput, PixelmatchJobOutput } from './libs/pixelmatch/pixelmatch.core'; + +const WORKER_FILE = join(__dirname, 'libs', 'pixelmatch', 'pixelmatch.worker.js'); + +interface Job { + input: PixelmatchJobInput; + resolve: (output: PixelmatchJobOutput) => void; + reject: (error: Error) => void; +} + +/** + * Fixed pool of worker threads for CPU-bound image diffing. Keeps the event + * loop free during build ingestion so the API stays responsive while + * screenshots are compared. Pool size: DIFF_WORKERS_COUNT env var, defaulting + * to cores - 1 (capped) so the main thread always has a core left. + */ +@Injectable() +export class DiffWorkerPool implements OnModuleDestroy { + private readonly logger: Logger = new Logger(DiffWorkerPool.name); + private readonly size = Math.max( + 1, + Math.min(Number(process.env.DIFF_WORKERS_COUNT) || availableParallelism() - 1, 8) + ); + // The compiled worker file only exists in the built app (dist). Under + // ts-jest / ts-node run the job inline instead. + private readonly inline = !existsSync(WORKER_FILE); + private workers: Worker[] = []; + private idle: Worker[] = []; + private queue: Job[] = []; + private inFlight = new Map(); + private started = false; + private destroyed = false; + + async run(input: PixelmatchJobInput): Promise { + if (this.inline) { + return computePixelmatchDiff(input); + } + this.start(); + return new Promise((resolve, reject) => { + this.queue.push({ input, resolve, reject }); + this.dispatch(); + }); + } + + private start(): void { + if (this.started) return; + this.started = true; + for (let i = 0; i < this.size; i++) { + this.spawn(); + } + this.logger.log(`Started ${this.size} image diff workers`); + } + + private spawn(): void { + const worker = new Worker(WORKER_FILE); + worker.on('message', (output: PixelmatchJobOutput & { error?: string }) => { + const job = this.inFlight.get(worker); + this.inFlight.delete(worker); + this.idle.push(worker); + if (job) { + output.error ? job.reject(new Error(output.error)) : job.resolve(output); + } + this.dispatch(); + }); + worker.on('error', (error) => { + this.logger.error(`Image diff worker crashed: ${error}`); + this.replace(worker, error); + }); + worker.on('exit', (code) => { + if (code !== 0) { + this.replace(worker, new Error(`Image diff worker exited with code ${code}`)); + } + }); + this.workers.push(worker); + this.idle.push(worker); + } + + private replace(worker: Worker, error: Error): void { + if (!this.workers.includes(worker)) return; + this.workers = this.workers.filter((w) => w !== worker); + this.idle = this.idle.filter((w) => w !== worker); + const job = this.inFlight.get(worker); + this.inFlight.delete(worker); + if (job) { + job.reject(error); + } + if (!this.destroyed) { + this.spawn(); + this.dispatch(); + } + } + + private dispatch(): void { + while (this.idle.length > 0 && this.queue.length > 0) { + const worker = this.idle.pop(); + const job = this.queue.shift(); + this.inFlight.set(worker, job); + worker.postMessage(job.input); + } + } + + async onModuleDestroy(): Promise { + this.destroyed = true; + const workers = this.workers; + this.workers = []; + this.idle = []; + await Promise.all(workers.map((worker) => worker.terminate())); + } +} diff --git a/src/compare/libs/pixelmatch/pixelmatch.core.ts b/src/compare/libs/pixelmatch/pixelmatch.core.ts new file mode 100644 index 00000000..ef765dcd --- /dev/null +++ b/src/compare/libs/pixelmatch/pixelmatch.core.ts @@ -0,0 +1,77 @@ +import { PNG } from 'pngjs'; +import Pixelmatch from 'pixelmatch'; +import { IgnoreAreaDto } from '../../../test-runs/dto/ignore-area.dto'; +import { applyIgnoreAreas, scaleImageToSize } from '../../utils'; + +/** + * CPU-bound part of the pixelmatch comparison, extracted so it can run inside + * a worker thread (see pixelmatch.worker.ts / DiffWorkerPool): PNG decode, + * scaling, pixelmatch and diff encode of full-size screenshots block the event + * loop for ~a second each, which makes the whole API unresponsive during + * build ingestion. All input/output must stay structured-clone serializable. + */ +export interface PixelmatchJobInput { + baseline: Buffer | Uint8Array; + image: Buffer | Uint8Array; + ignoreAreas: IgnoreAreaDto[]; + threshold: number; + includeAA: boolean; + allowDiffDimensions: boolean; + diffTolerancePercent: number; + saveDiff: boolean; +} + +export interface PixelmatchJobOutput { + equal?: boolean; + isSameDimension?: boolean; + pixelMisMatchCount?: number; + diffPercent?: number; + diffBuffer?: Buffer | Uint8Array; +} + +// postMessage turns Buffers into Uint8Array views over their own ArrayBuffer. +function toBuffer(data: Buffer | Uint8Array): Buffer { + return Buffer.isBuffer(data) ? data : Buffer.from(data.buffer, data.byteOffset, data.byteLength); +} + +export function computePixelmatchDiff(input: PixelmatchJobInput): PixelmatchJobOutput { + const baseline = PNG.sync.read(toBuffer(input.baseline)); + const image = PNG.sync.read(toBuffer(input.image)); + + if (baseline.data.equals(new Uint8Array(image.data.buffer, image.data.byteOffset, image.data.byteLength))) { + return { equal: true }; + } + + const isSameDimension = baseline.width === image.width && baseline.height === image.height; + if (!isSameDimension && !input.allowDiffDimensions) { + return { equal: false, isSameDimension }; + } + + // scale image to max size + const maxWidth = Math.max(baseline.width, image.width); + const maxHeight = Math.max(baseline.height, image.height); + const scaledBaseline = scaleImageToSize(baseline, maxWidth, maxHeight); + const scaledImage = scaleImageToSize(image, maxWidth, maxHeight); + + // apply ignore areas + const baselineIgnored = applyIgnoreAreas(scaledBaseline, input.ignoreAreas); + const imageIgnored = applyIgnoreAreas(scaledImage, input.ignoreAreas); + + // compare + const diff = new PNG({ + width: maxWidth, + height: maxHeight, + }); + const pixelMisMatchCount = Pixelmatch(baselineIgnored.data, imageIgnored.data, diff.data, maxWidth, maxHeight, { + includeAA: input.includeAA, + threshold: input.threshold, + }); + const diffPercent = (pixelMisMatchCount * 100) / (scaledImage.width * scaledImage.height); + + let diffBuffer: Buffer; + if (diffPercent > input.diffTolerancePercent && input.saveDiff) { + diffBuffer = PNG.sync.write(diff); + } + + return { equal: false, isSameDimension, pixelMisMatchCount, diffPercent, diffBuffer }; +} diff --git a/src/compare/libs/pixelmatch/pixelmatch.service.spec.ts b/src/compare/libs/pixelmatch/pixelmatch.service.spec.ts index c458f82e..1696b97d 100644 --- a/src/compare/libs/pixelmatch/pixelmatch.service.spec.ts +++ b/src/compare/libs/pixelmatch/pixelmatch.service.spec.ts @@ -4,20 +4,28 @@ import Pixelmatch from 'pixelmatch'; import { PNG } from 'pngjs'; import { mocked } from 'jest-mock'; import { StaticService } from '../../../static/static.service'; +import { DiffWorkerPool } from '../../diff-worker-pool'; import { DIFF_DIMENSION_RESULT, EQUAL_RESULT, NO_BASELINE_RESULT } from '../consts'; import { DEFAULT_CONFIG, PixelmatchService } from './pixelmatch.service'; import { PixelmatchConfig } from './pixelmatch.types'; jest.mock('pixelmatch'); -const initService = async ({ getImageMock = jest.fn(), saveImageMock = jest.fn(), deleteImageMock = jest.fn() }) => { +const toBuffer = (png: PNG | undefined): Buffer | undefined => (png ? PNG.sync.write(png) : undefined); + +const initService = async ({ + getImageBufferMock = jest.fn(), + saveImageMock = jest.fn(), + deleteImageMock = jest.fn(), +}) => { const module: TestingModule = await Test.createTestingModule({ providers: [ PixelmatchService, + DiffWorkerPool, { provide: StaticService, useValue: { - getImage: getImageMock, + getImageBuffer: getImageBufferMock, saveImage: saveImageMock, deleteImage: deleteImageMock, }, @@ -54,8 +62,8 @@ describe('getDiff', () => { }); it('no baseline', async () => { - const getImageMock = jest.fn().mockReturnValueOnce(undefined).mockReturnValueOnce(image); - service = await initService({ getImageMock }); + const getImageBufferMock = jest.fn().mockReturnValueOnce(undefined).mockReturnValueOnce(toBuffer(image)); + service = await initService({ getImageBufferMock }); const result = await service.getDiff( { @@ -72,8 +80,8 @@ describe('getDiff', () => { }); it('diff not found', async () => { - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(image); - service = await initService({ getImageMock }); + const getImageBufferMock = jest.fn().mockReturnValueOnce(toBuffer(image)).mockReturnValueOnce(toBuffer(image)); + service = await initService({ getImageBufferMock }); const result = await service.getDiff( { @@ -94,8 +102,8 @@ describe('getDiff', () => { width: 10, height: 10, }); - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(baseline); - service = await initService({ getImageMock }); + const getImageBufferMock = jest.fn().mockReturnValueOnce(toBuffer(baseline)).mockReturnValueOnce(toBuffer(image)); + service = await initService({ getImageBufferMock }); const result = await service.getDiff( { @@ -120,11 +128,11 @@ describe('getDiff', () => { width: 2, height: 4, }); - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(baseline); + const getImageBufferMock = jest.fn().mockReturnValueOnce(toBuffer(baseline)).mockReturnValueOnce(toBuffer(image)); const diffName = 'diff name'; const saveImageMock = jest.fn().mockReturnValueOnce(diffName); mocked(Pixelmatch).mockReturnValueOnce(5); - service = await initService({ saveImageMock, getImageMock }); + service = await initService({ saveImageMock, getImageBufferMock }); const result = await service.getDiff( { @@ -181,9 +189,9 @@ describe('getDiff', () => { width: 100, height: 100, }); - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(baseline); + const getImageBufferMock = jest.fn().mockReturnValueOnce(toBuffer(baseline)).mockReturnValueOnce(toBuffer(image)); const saveImageMock = jest.fn(); - service = await initService({ saveImageMock, getImageMock }); + service = await initService({ saveImageMock, getImageBufferMock }); const pixelMisMatchCount = 150; mocked(Pixelmatch).mockReturnValueOnce(pixelMisMatchCount); @@ -218,14 +226,14 @@ describe('getDiff', () => { width: 100, height: 100, }); - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(baseline); + const getImageBufferMock = jest.fn().mockReturnValueOnce(toBuffer(baseline)).mockReturnValueOnce(toBuffer(image)); const pixelMisMatchCount = 200; mocked(Pixelmatch).mockReturnValueOnce(pixelMisMatchCount); const diffName = 'diff name'; const saveImageMock = jest.fn().mockReturnValueOnce(diffName); service = await initService({ saveImageMock, - getImageMock, + getImageBufferMock, }); const result = await service.getDiff( diff --git a/src/compare/libs/pixelmatch/pixelmatch.service.ts b/src/compare/libs/pixelmatch/pixelmatch.service.ts index 2689882b..8a47df89 100644 --- a/src/compare/libs/pixelmatch/pixelmatch.service.ts +++ b/src/compare/libs/pixelmatch/pixelmatch.service.ts @@ -1,14 +1,13 @@ import { Injectable, Logger } from '@nestjs/common'; import { TestStatus } from '@prisma/client'; -import Pixelmatch from 'pixelmatch'; -import { PNG } from 'pngjs'; import { StaticService } from '../../../static/static.service'; import { DiffResult } from '../../../test-runs/diffResult'; -import { scaleImageToSize, applyIgnoreAreas, parseConfig } from '../../utils'; +import { parseConfig } from '../../utils'; import { DIFF_DIMENSION_RESULT, EQUAL_RESULT, NO_BASELINE_RESULT } from '../consts'; import { ImageComparator } from '../image-comparator.interface'; import { ImageCompareInput } from '../ImageCompareInput'; import { PixelmatchConfig } from './pixelmatch.types'; +import { DiffWorkerPool } from '../../diff-worker-pool'; export const DEFAULT_CONFIG: PixelmatchConfig = { threshold: 0.1, ignoreAntialiasing: true }; @@ -16,59 +15,54 @@ export const DEFAULT_CONFIG: PixelmatchConfig = { threshold: 0.1, ignoreAntialia export class PixelmatchService implements ImageComparator { private readonly logger: Logger = new Logger(PixelmatchService.name); - constructor(private readonly staticService: StaticService) {} + constructor( + private readonly staticService: StaticService, + private readonly diffWorkerPool: DiffWorkerPool + ) {} parseConfig(configJson: string): PixelmatchConfig { return parseConfig(configJson, DEFAULT_CONFIG, this.logger); } async getDiff(data: ImageCompareInput, config: PixelmatchConfig): Promise { - const result: DiffResult = { - ...NO_BASELINE_RESULT, - }; - - const baseline = await this.staticService.getImage(data.baseline); - const image = await this.staticService.getImage(data.image); - - if (!baseline) { + const baselineBuffer = await this.staticService.getImageBuffer(data.baseline); + if (!baselineBuffer) { return NO_BASELINE_RESULT; } + const imageBuffer = await this.staticService.getImageBuffer(data.image); + if (!imageBuffer) { + throw new Error(`Cannot get image: ${data.image}`); + } + + // decode + pixelmatch + diff encode run off the event loop + const output = await this.diffWorkerPool.run({ + baseline: baselineBuffer, + image: imageBuffer, + ignoreAreas: data.ignoreAreas, + threshold: config.threshold, + includeAA: config.ignoreAntialiasing, + allowDiffDimensions: config.allowDiffDimensions, + diffTolerancePercent: data.diffTollerancePercent, + saveDiff: data.saveDiffAsFile, + }); - if (baseline.data.equals(new Uint8Array(image.data.buffer, image.data.byteOffset, image.data.byteLength))) { + if (output.equal) { return EQUAL_RESULT; } - - result.isSameDimension = baseline.width === image.width && baseline.height === image.height; - if (!result.isSameDimension && !config.allowDiffDimensions) { + if (!output.isSameDimension && !config.allowDiffDimensions) { return DIFF_DIMENSION_RESULT; } - // scale image to max size - const maxWidth = Math.max(baseline.width, image.width); - const maxHeight = Math.max(baseline.height, image.height); - const scaledBaseline = scaleImageToSize(baseline, maxWidth, maxHeight); - const scaledImage = scaleImageToSize(image, maxWidth, maxHeight); - - // apply ignore areas - const baselineIgnored = applyIgnoreAreas(scaledBaseline, data.ignoreAreas); - const imageIgnored = applyIgnoreAreas(scaledImage, data.ignoreAreas); - - // compare - const diff = new PNG({ - width: maxWidth, - height: maxHeight, - }); - result.pixelMisMatchCount = Pixelmatch(baselineIgnored.data, imageIgnored.data, diff.data, maxWidth, maxHeight, { - includeAA: config.ignoreAntialiasing, - threshold: config.threshold, - }); - result.diffPercent = (result.pixelMisMatchCount * 100) / (scaledImage.width * scaledImage.height); + const result: DiffResult = { + ...NO_BASELINE_RESULT, + isSameDimension: output.isSameDimension, + pixelMisMatchCount: output.pixelMisMatchCount, + diffPercent: output.diffPercent, + }; - // process result if (result.diffPercent > data.diffTollerancePercent) { - // save diff - if (data.saveDiffAsFile) { - result.diffName = await this.staticService.saveImage('diff', PNG.sync.write(diff)); + if (output.diffBuffer) { + result.diffName = await this.staticService.saveImage('diff', Buffer.from(output.diffBuffer)); } result.status = TestStatus.unresolved; } else { diff --git a/src/compare/libs/pixelmatch/pixelmatch.worker.ts b/src/compare/libs/pixelmatch/pixelmatch.worker.ts new file mode 100644 index 00000000..2434d105 --- /dev/null +++ b/src/compare/libs/pixelmatch/pixelmatch.worker.ts @@ -0,0 +1,10 @@ +import { parentPort } from 'worker_threads'; +import { computePixelmatchDiff, PixelmatchJobInput } from './pixelmatch.core'; + +parentPort.on('message', (input: PixelmatchJobInput) => { + try { + parentPort.postMessage(computePixelmatchDiff(input)); + } catch (error) { + parentPort.postMessage({ error: error instanceof Error ? error.message : String(error) }); + } +}); From 01a22f3727413e9482c2179c994b6dd9bd128d7f Mon Sep 17 00:00:00 2001 From: "mykola.gervasyuk" Date: Thu, 20 Aug 2026 13:49:02 +0300 Subject: [PATCH 5/9] perf(api): gzip JSON responses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Large builds return multi-megabyte test-run lists; gzip shrinks them about tenfold. Registered before swagger so every route is covered; PNG responses are excluded — they are already compressed. --- package-lock.json | 86 +++++++++++++++++++++++++++++++++++++++++------ package.json | 2 ++ src/main.ts | 9 +++++ 3 files changed, 86 insertions(+), 11 deletions(-) diff --git a/package-lock.json b/package-lock.json index ffe3ff9f..7d6f44fe 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,11 +27,13 @@ "@nestjs/websockets": "^11.1.28", "@prisma/client": "^6.19.3", "@socket.io/redis-adapter": "^8.3.0", + "@types/compression": "^1.8.1", "ajv": "^8.17.1", "bcryptjs": "^2.4.3", "cache-manager": "^7.2.4", "class-transformer": "^0.5.1", "class-validator": "^0.14.0", + "compression": "^1.8.1", "fs-extra": "^11.3.2", "ldapts": "^7.1.0", "looks-same": "^9.0.0", @@ -4122,7 +4124,6 @@ "version": "1.19.6", "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz", "integrity": "sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==", - "dev": true, "license": "MIT", "dependencies": { "@types/connect": "*", @@ -4136,11 +4137,20 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/compression": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/@types/compression/-/compression-1.8.1.tgz", + "integrity": "sha512-kCFuWS0ebDbmxs0AXYn6e2r2nrGAb5KwQhknjSPSPgJcGd8+HVSILlUyFhGqML2gk39HcG7D1ydW9/qpYkN00Q==", + "license": "MIT", + "dependencies": { + "@types/express": "*", + "@types/node": "*" + } + }, "node_modules/@types/connect": { "version": "3.4.38", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", - "dev": true, "license": "MIT", "dependencies": { "@types/node": "*" @@ -4195,7 +4205,6 @@ "version": "4.17.25", "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.25.tgz", "integrity": "sha512-dVd04UKsfpINUnK0yBoYHDF3xu7xVH4BuDotC/xGuycx4CgbP48X/KF/586bcObxT0HENHXEU8Nqtu6NR+eKhw==", - "dev": true, "license": "MIT", "dependencies": { "@types/body-parser": "*", @@ -4208,7 +4217,6 @@ "version": "4.19.9", "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.9.tgz", "integrity": "sha512-QP2ESEe/ImWY0HDwNAnK9PvEffUyhLTnWkk7KXzHfyeWAnlrDe1fN77bXl6ia8KT3wPlmA7t9/VPRpnf4Ex9sg==", - "dev": true, "license": "MIT", "dependencies": { "@types/node": "*", @@ -4231,7 +4239,6 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.5.tgz", "integrity": "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==", - "dev": true, "license": "MIT" }, "node_modules/@types/istanbul-lib-coverage": { @@ -4472,7 +4479,6 @@ "version": "1.3.5", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", - "dev": true, "license": "MIT" }, "node_modules/@types/ms": { @@ -4569,14 +4575,12 @@ "version": "6.15.1", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.15.1.tgz", "integrity": "sha512-GZHUBZR9hckSUhrxmp1nG6NwdpM9fCunJwyThLW1X3AyHgd9IlHb6VANpQQqDr2o/qQp6McZ3y/IA2rVzKzSbw==", - "dev": true, "license": "MIT" }, "node_modules/@types/range-parser": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", - "dev": true, "license": "MIT" }, "node_modules/@types/retry": { @@ -4589,7 +4593,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/@types/send/-/send-1.2.1.tgz", "integrity": "sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ==", - "dev": true, "license": "MIT", "dependencies": { "@types/node": "*" @@ -4599,7 +4602,6 @@ "version": "1.15.10", "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.10.tgz", "integrity": "sha512-tRs1dB+g8Itk72rlSI2ZrW6vZg0YrLI81iQSTkMmOqnqCaNr/8Ek4VwWcN5vZgCYWbg/JJSGBlUaYGAOP73qBw==", - "dev": true, "license": "MIT", "dependencies": { "@types/http-errors": "*", @@ -4611,7 +4613,6 @@ "version": "0.17.6", "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.6.tgz", "integrity": "sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og==", - "dev": true, "license": "MIT", "dependencies": { "@types/mime": "^1", @@ -6688,6 +6689,60 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/compressible": { + "version": "2.0.18", + "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", + "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", + "license": "MIT", + "dependencies": { + "mime-db": ">= 1.43.0 < 2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/compression": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/compression/-/compression-1.8.1.tgz", + "integrity": "sha512-9mAqGPHLakhCLeNyxPkK4xVo746zQ/czLH1Ky+vkitMnWfWZps8r0qXuwhwizagCRttsL4lfG4pIOvaWLpAP0w==", + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "compressible": "~2.0.18", + "debug": "2.6.9", + "negotiator": "~0.6.4", + "on-headers": "~1.1.0", + "safe-buffer": "5.2.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/compression/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/compression/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/compression/node_modules/negotiator": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.4.tgz", + "integrity": "sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -14147,6 +14202,15 @@ "node": ">= 0.8" } }, + "node_modules/on-headers": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.1.0.tgz", + "integrity": "sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", diff --git a/package.json b/package.json index 5b3a650e..93b9d397 100644 --- a/package.json +++ b/package.json @@ -45,11 +45,13 @@ "@nestjs/websockets": "^11.1.28", "@prisma/client": "^6.19.3", "@socket.io/redis-adapter": "^8.3.0", + "@types/compression": "^1.8.1", "ajv": "^8.17.1", "bcryptjs": "^2.4.3", "cache-manager": "^7.2.4", "class-transformer": "^0.5.1", "class-validator": "^0.14.0", + "compression": "^1.8.1", "fs-extra": "^11.3.2", "ldapts": "^7.1.0", "looks-same": "^9.0.0", diff --git a/src/main.ts b/src/main.ts index 31a1b729..7781323b 100644 --- a/src/main.ts +++ b/src/main.ts @@ -4,6 +4,7 @@ import { setupSwagger } from './swagger'; import { Logger, ValidationPipe } from '@nestjs/common'; import { join } from 'path'; import * as bodyParser from 'body-parser'; +import compression from 'compression'; import { readFileSync, existsSync } from 'fs'; import { HttpsOptions } from '@nestjs/common/interfaces/external/https-options.interface'; import { NestExpressApplication } from '@nestjs/platform-express'; @@ -29,6 +30,14 @@ async function bootstrap() { httpsOptions: getHttpsOptions(), }); app.useGlobalPipes(new ValidationPipe()); + + // Large builds return multi-megabyte test-run lists; gzip shrinks them ~10x. + // Images are already-compressed PNGs — recompressing them wastes CPU. + // Must be registered before any routes (incl. swagger) to cover them. + app.use( + compression({ filter: (req, res) => compression.filter(req, res) && res.getHeader('Content-Type') !== 'image/png' }) + ); + setupSwagger(app); // Fan out socket.io events across API instances when running multiple From a266d6bc5aa30d481f999e3d1970b2800a299fae Mon Sep 17 00:00:00 2001 From: "mykola.gervasyuk" Date: Thu, 20 Aug 2026 14:03:40 +0300 Subject: [PATCH 6/9] perf(db): build the test-run index concurrently A plain CREATE INDEX write-locks TestRun for the duration of the build on large production tables. Kept as the single statement in the migration on purpose: Prisma runs one-statement migrations outside a transaction, which CREATE INDEX CONCURRENTLY requires. --- .../20260818134932_add_test_run_build_index/migration.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/prisma/migrations/20260818134932_add_test_run_build_index/migration.sql b/prisma/migrations/20260818134932_add_test_run_build_index/migration.sql index 9e46a71d..63e4e779 100644 --- a/prisma/migrations/20260818134932_add_test_run_build_index/migration.sql +++ b/prisma/migrations/20260818134932_add_test_run_build_index/migration.sql @@ -1,2 +1,5 @@ -- CreateIndex -CREATE INDEX "TestRun_buildId_branchName_name_idx" ON "TestRun"("buildId", "branchName", "name"); +-- Single statement on purpose: Prisma runs one-statement migrations outside a +-- transaction, which CREATE INDEX CONCURRENTLY requires. Concurrent build +-- avoids write-locking TestRun on large production tables. +CREATE INDEX CONCURRENTLY IF NOT EXISTS "TestRun_buildId_branchName_name_idx" ON "TestRun"("buildId", "branchName", "name"); From 465749af6adbda9c1435675baa6592b336654bdb Mon Sep 17 00:00:00 2001 From: "mykola.gervasyuk" Date: Thu, 20 Aug 2026 14:03:40 +0300 Subject: [PATCH 7/9] fix(compare): bound the diff queue and reject pending jobs on shutdown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Queued jobs hold both full image buffers, so an unbounded queue could exhaust memory under a flood of concurrent uploads — cap it (DIFF_QUEUE_LIMIT, default 256). On module destroy, reject queued and in-flight jobs instead of leaving their callers hanging. --- src/compare/diff-worker-pool.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/compare/diff-worker-pool.ts b/src/compare/diff-worker-pool.ts index e6ae9b2b..8f236e16 100644 --- a/src/compare/diff-worker-pool.ts +++ b/src/compare/diff-worker-pool.ts @@ -26,6 +26,9 @@ export class DiffWorkerPool implements OnModuleDestroy { 1, Math.min(Number(process.env.DIFF_WORKERS_COUNT) || availableParallelism() - 1, 8) ); + // Queued jobs hold both full image buffers, so an unbounded queue could + // exhaust memory under a flood of concurrent uploads. + private readonly queueLimit = Number(process.env.DIFF_QUEUE_LIMIT) || 256; // The compiled worker file only exists in the built app (dist). Under // ts-jest / ts-node run the job inline instead. private readonly inline = !existsSync(WORKER_FILE); @@ -40,6 +43,12 @@ export class DiffWorkerPool implements OnModuleDestroy { if (this.inline) { return computePixelmatchDiff(input); } + if (this.destroyed) { + throw new Error('Image diff worker pool is shut down'); + } + if (this.queue.length >= this.queueLimit) { + throw new Error(`Image diff queue is full (${this.queueLimit} jobs)`); + } this.start(); return new Promise((resolve, reject) => { this.queue.push({ input, resolve, reject }); @@ -106,6 +115,15 @@ export class DiffWorkerPool implements OnModuleDestroy { async onModuleDestroy(): Promise { this.destroyed = true; + const shutdownError = new Error('Image diff worker pool is shutting down'); + for (const job of this.queue) { + job.reject(shutdownError); + } + this.queue = []; + for (const job of this.inFlight.values()) { + job.reject(shutdownError); + } + this.inFlight.clear(); const workers = this.workers; this.workers = []; this.idle = []; From 075867e1e0af4f29709cd9c911113267b588252d Mon Sep 17 00:00:00 2001 From: "mykola.gervasyuk" Date: Thu, 20 Aug 2026 14:03:40 +0300 Subject: [PATCH 8/9] fix(static): validate the PNG signature for every storage backend The signature check lived in HddService only, letting non-PNG bytes reach S3-backed storage. Move it to the StaticService facade so both backends reject invalid uploads. --- src/static/aws/s3.service.ts | 11 ----------- src/static/hdd/hdd.service.ts | 11 ----------- src/static/static.service.ts | 5 +++++ src/static/utils.ts | 10 ++++++++++ 4 files changed, 15 insertions(+), 22 deletions(-) diff --git a/src/static/aws/s3.service.ts b/src/static/aws/s3.service.ts index 55820893..965bc05a 100644 --- a/src/static/aws/s3.service.ts +++ b/src/static/aws/s3.service.ts @@ -41,17 +41,6 @@ export class AWSS3Service implements Static { } async getImage(fileName: string): Promise { - if (!fileName) return null; - const imageBuffer = await this.getImageBuffer(fileName); - if (!imageBuffer) return undefined; - try { - return PNG.sync.read(imageBuffer); - } catch (ex) { - this.logger.error(`Error from read : Cannot decode image: ${fileName}. ${ex}`); - } - } - - async getImageBuffer(fileName: string): Promise { if (!fileName) return null; try { // the comparison pipeline treats an unreadable image as a missing diff --git a/src/static/hdd/hdd.service.ts b/src/static/hdd/hdd.service.ts index a08c6151..9c507699 100644 --- a/src/static/hdd/hdd.service.ts +++ b/src/static/hdd/hdd.service.ts @@ -6,8 +6,6 @@ import { Static } from '../static.interface'; import { HDD_IMAGE_PATH } from './constants'; import { generateNewImageName } from '../utils'; -const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); - export class HddService implements Static { private readonly logger: Logger = new Logger(HddService.name); @@ -41,15 +39,6 @@ export class HddService implements Static { } async saveImage(type: 'screenshot' | 'diff' | 'baseline', imageBuffer: Buffer): Promise { - // Signature check instead of a full decode: parsing megapixel PNGs just to - // validate them blocks the event loop on every upload. - if ( - imageBuffer.length < PNG_SIGNATURE.length || - !imageBuffer.subarray(0, PNG_SIGNATURE.length).equals(PNG_SIGNATURE) - ) { - throw new Error('Cannot parse image as PNG file'); - } - const { imageName, imagePath } = this.generateNewImage(type); await fs.writeFile(imagePath, new Uint8Array(imageBuffer.buffer, imageBuffer.byteOffset, imageBuffer.byteLength)); return imageName; diff --git a/src/static/static.service.ts b/src/static/static.service.ts index 345aebfc..26294efc 100644 --- a/src/static/static.service.ts +++ b/src/static/static.service.ts @@ -1,6 +1,7 @@ import { Injectable, Logger } from '@nestjs/common'; import { PNGWithMetadata } from 'pngjs'; import { StaticFactoryService } from './static.factory'; +import { isPngBuffer } from './utils'; import { Static } from './static.interface'; @Injectable() @@ -13,6 +14,10 @@ export class StaticService { } async saveImage(type: 'screenshot' | 'diff' | 'baseline', imageBuffer: Buffer): Promise { + // validate here so every storage backend rejects non-PNG uploads + if (!isPngBuffer(imageBuffer)) { + throw new Error('Cannot parse image as PNG file'); + } return this.staticService.saveImage(type, imageBuffer); } diff --git a/src/static/utils.ts b/src/static/utils.ts index 310061d3..20e65b3e 100644 --- a/src/static/utils.ts +++ b/src/static/utils.ts @@ -11,3 +11,13 @@ export function isS3ServiceConfigured() { export function generateNewImageName(type: 'screenshot' | 'diff' | 'baseline'): string { return `${uuidAPIKey.create({ noDashes: true }).apiKey}.${type}.png`; } + +const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + +// Signature check instead of a full decode: parsing megapixel PNGs just to +// validate them blocks the event loop on every upload. +export function isPngBuffer(imageBuffer: Buffer): boolean { + return ( + imageBuffer.length >= PNG_SIGNATURE.length && imageBuffer.subarray(0, PNG_SIGNATURE.length).equals(PNG_SIGNATURE) + ); +} From 1eed20a51b8740bde2ecc59d531258f89a5b79f0 Mon Sep 17 00:00:00 2001 From: "mykola.gervasyuk" Date: Sun, 23 Aug 2026 09:26:39 +0300 Subject: [PATCH 9/9] fix(compare): keep the diff worker pool sane when a worker dies Three things could go wrong around a failing worker: - a message arriving after the worker was dropped or the pool shut down put it back in the idle list, so it was handed a job it would never answer - a worker script that cannot load fails the same way on every respawn, which turned the error handler into a spawn loop; the pool now gives up after three failures and compares on the main thread instead, so uploads still go through - the job in flight when a worker died was failed outright; it is now handed to another worker once before being given up on The migration also drops IF NOT EXISTS: a concurrent index build that fails leaves an invalid index behind, and skipping it on a retry would let the migration report success while the index stays unusable. --- .../migration.sql | 6 +- src/compare/diff-worker-pool.ts | 61 ++++++++++++++++--- 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/prisma/migrations/20260818134932_add_test_run_build_index/migration.sql b/prisma/migrations/20260818134932_add_test_run_build_index/migration.sql index 63e4e779..dd20ce2e 100644 --- a/prisma/migrations/20260818134932_add_test_run_build_index/migration.sql +++ b/prisma/migrations/20260818134932_add_test_run_build_index/migration.sql @@ -2,4 +2,8 @@ -- Single statement on purpose: Prisma runs one-statement migrations outside a -- transaction, which CREATE INDEX CONCURRENTLY requires. Concurrent build -- avoids write-locking TestRun on large production tables. -CREATE INDEX CONCURRENTLY IF NOT EXISTS "TestRun_buildId_branchName_name_idx" ON "TestRun"("buildId", "branchName", "name"); +-- +-- Deliberately without IF NOT EXISTS: a concurrent build that fails leaves an +-- invalid index behind, and skipping it on a retry would let the migration +-- succeed while the index stays unusable. Drop the invalid index, then retry. +CREATE INDEX CONCURRENTLY "TestRun_buildId_branchName_name_idx" ON "TestRun"("buildId", "branchName", "name"); diff --git a/src/compare/diff-worker-pool.ts b/src/compare/diff-worker-pool.ts index 8f236e16..20324d45 100644 --- a/src/compare/diff-worker-pool.ts +++ b/src/compare/diff-worker-pool.ts @@ -7,10 +7,16 @@ import { computePixelmatchDiff, PixelmatchJobInput, PixelmatchJobOutput } from ' const WORKER_FILE = join(__dirname, 'libs', 'pixelmatch', 'pixelmatch.worker.js'); +// consecutive worker failures after which the pool stops respawning +const MAX_SPAWN_FAILURES = 3; +// how many times a job may be handed to a worker before it is given up on +const MAX_JOB_ATTEMPTS = 2; + interface Job { input: PixelmatchJobInput; resolve: (output: PixelmatchJobOutput) => void; reject: (error: Error) => void; + attempts: number; } /** @@ -31,13 +37,14 @@ export class DiffWorkerPool implements OnModuleDestroy { private readonly queueLimit = Number(process.env.DIFF_QUEUE_LIMIT) || 256; // The compiled worker file only exists in the built app (dist). Under // ts-jest / ts-node run the job inline instead. - private readonly inline = !existsSync(WORKER_FILE); + private inline = !existsSync(WORKER_FILE); private workers: Worker[] = []; private idle: Worker[] = []; private queue: Job[] = []; private inFlight = new Map(); private started = false; private destroyed = false; + private spawnFailures = 0; async run(input: PixelmatchJobInput): Promise { if (this.inline) { @@ -51,7 +58,7 @@ export class DiffWorkerPool implements OnModuleDestroy { } this.start(); return new Promise((resolve, reject) => { - this.queue.push({ input, resolve, reject }); + this.queue.push({ input, resolve, reject, attempts: 0 }); this.dispatch(); }); } @@ -70,7 +77,12 @@ export class DiffWorkerPool implements OnModuleDestroy { worker.on('message', (output: PixelmatchJobOutput & { error?: string }) => { const job = this.inFlight.get(worker); this.inFlight.delete(worker); - this.idle.push(worker); + // a message can arrive after the worker was dropped or the pool shut + // down; taking it back would hand it a job it will never answer + if (this.workers.includes(worker)) { + this.idle.push(worker); + this.spawnFailures = 0; + } if (job) { output.error ? job.reject(new Error(output.error)) : job.resolve(output); } @@ -96,11 +108,46 @@ export class DiffWorkerPool implements OnModuleDestroy { const job = this.inFlight.get(worker); this.inFlight.delete(worker); if (job) { - job.reject(error); + // a worker dying takes its job down with it; give the job one more run + // rather than failing the upload waiting behind it + if (!this.destroyed && job.attempts < MAX_JOB_ATTEMPTS) { + job.attempts += 1; + this.queue.unshift(job); + } else { + job.reject(error); + } } - if (!this.destroyed) { - this.spawn(); - this.dispatch(); + if (this.destroyed) { + return; + } + + // a worker script that fails to load fails the same way every time, so + // respawning on each error would spin. Fall back to comparing on this + // thread instead: slower, but the uploads still go through. + this.spawnFailures += 1; + if (this.spawnFailures >= MAX_SPAWN_FAILURES) { + this.inline = true; + this.logger.error( + `Image diff workers failed to run ${this.spawnFailures} times, comparing on the main thread from now on` + ); + this.drainQueueInline(); + return; + } + + this.spawn(); + this.dispatch(); + } + + // the pool gave up on workers: answer what is already queued here + private drainQueueInline(): void { + const queued = this.queue; + this.queue = []; + for (const job of queued) { + try { + job.resolve(computePixelmatchDiff(job.input)); + } catch (error) { + job.reject(error instanceof Error ? error : new Error(String(error))); + } } }