From feed4bd8b58ddde1c40f5428a42d8d440e4b97e4 Mon Sep 17 00:00:00 2001 From: Aleksandr Minkin Date: Wed, 8 Apr 2026 10:42:32 +0300 Subject: [PATCH 1/2] Harden EIS parsing and run tests in CI --- .github/workflows/ci.yml | 10 ++++-- src/sources/eis/eis-mapper.ts | 18 +++++++--- src/sources/eis/eis-parser.spec.ts | 57 ++++++++++++++++++++++++++++++ src/sources/eis/eis-parser.ts | 50 ++++++++++++++++++-------- src/sources/eis/eis-source.ts | 9 ++++- 5 files changed, 123 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31dbd1b..ed13605 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,8 +5,11 @@ on: branches: ["main", "master"] pull_request: +permissions: + contents: read + jobs: - build: + verify: runs-on: ubuntu-latest steps: - name: Checkout @@ -19,10 +22,13 @@ jobs: cache: npm - name: Install dependencies - run: npm install + run: npm ci - name: Type check run: npm run check + - name: Run tests + run: npm test + - name: Build run: npm run build diff --git a/src/sources/eis/eis-mapper.ts b/src/sources/eis/eis-mapper.ts index 4cc2c78..cd3151b 100644 --- a/src/sources/eis/eis-mapper.ts +++ b/src/sources/eis/eis-mapper.ts @@ -20,7 +20,7 @@ export function mapEisNoticeToCollectedRecord(input: { sourceName, sourceType }; - const targetStationName = extractTargetStationName(notice, matchedQuery); + const targetStationName = extractTargetStationName(notice, sourceType, matchedQuery); return { url: notice.externalUrl, @@ -73,13 +73,23 @@ export function mapEisNoticeToCollectedRecord(input: { function extractTargetStationName( notice: Pick, + sourceType: "procurement" | "contract", matchedQuery?: string ): string | undefined { - return resolveNppStationNameFromText([ + const directMatch = resolveNppStationNameFromText([ notice.title, notice.description, notice.customerName, - notice.supplierName, - matchedQuery + notice.supplierName ]); + + if (directMatch) { + return directMatch; + } + + if (sourceType === "contract") { + return resolveNppStationNameFromText([matchedQuery]); + } + + return undefined; } diff --git a/src/sources/eis/eis-parser.spec.ts b/src/sources/eis/eis-parser.spec.ts index 658c5b6..a7e667a 100644 --- a/src/sources/eis/eis-parser.spec.ts +++ b/src/sources/eis/eis-parser.spec.ts @@ -404,4 +404,61 @@ describe("eis-parser", () => { supplierName: undefined }); }); + + it("drops boilerplate title when parser lands on EIS documents page or popup-like shell", () => { + const html = ` + + + + Поделитесь мнением о качестве работы единой информационной системы + Перейти к опросу Система торгов Сбербанк-АСТ SBERBANK-AST.RU + Единая электронная торговая площадка ROSELTORG.RU + Техническая поддержка Ваши идеи по улучшению сайта + + + +

+ Поделитесь мнением о качестве работы единой информационной системы + Перейти к опросу Система торгов Сбербанк-АСТ SBERBANK-AST.RU + Единая электронная торговая площадка ROSELTORG.RU + Техническая поддержка Ваши идеи по улучшению сайта +

+ + + `; + + const notice = parseEisNoticePage( + html, + "https://zakupki.gov.ru/epz/order/notice/notice223/documents.html?regNumber=32615886957" + ); + + expect(notice.externalId).toBe("32615886957"); + expect(notice.title).toBeUndefined(); + expect(notice.description).toBeUndefined(); + expect(notice.customerName).toBeUndefined(); + }); + + it("prefers common-info pages over documents pages for the same notice", () => { + const html = ` + + + Документы + Карточка + + + `; + + const results = parseEisSearchResults(html, { + baseUrl: "https://zakupki.gov.ru", + maxItems: 10 + }); + + expect(results).toEqual([ + { + externalId: "32615886957", + detailUrl: "https://zakupki.gov.ru/epz/order/notice/notice223/common-info.html?regNumber=32615886957", + title: "Карточка" + } + ]); + }); }); diff --git a/src/sources/eis/eis-parser.ts b/src/sources/eis/eis-parser.ts index 519d1ca..63c7f63 100644 --- a/src/sources/eis/eis-parser.ts +++ b/src/sources/eis/eis-parser.ts @@ -146,16 +146,18 @@ export function parseEisNoticePage( readPageTitle($) ].map(normalizeExternalIdCandidate).find(Boolean) ?? "unknown"; - const title = + const title = sanitizeNoticeTitle( findFirstValue($, TITLE_LABELS, structuredValues) ?? - readPrimaryHeading($) ?? - findMetaContent($, "og:title") ?? - findMetaContent($, "twitter:title"); + readPrimaryHeading($) ?? + findMetaContent($, "og:title") ?? + findMetaContent($, "twitter:title") + ); - const description = + const description = sanitizeNoticeDescription( findFirstValue($, DESCRIPTION_LABELS, structuredValues) ?? - findMetaContent($, "description") ?? - undefined; + findMetaContent($, "description") ?? + undefined + ); const customerName = sanitizePartyName(findFirstValue($, CUSTOMER_LABELS, structuredValues)); const supplierName = sanitizePartyName(findFirstValue($, SUPPLIER_LABELS, structuredValues)); @@ -315,12 +317,12 @@ function readPrimaryHeading($: CheerioAPI): string | undefined { .first() .text(); const heading = preferredHeading || $("h1, h2").first().text(); - return cleanText(heading) || undefined; + return sanitizeNoticeTitle(heading); } function readPageTitle($: CheerioAPI): string | undefined { const title = $("title").first().text(); - return cleanText(title) || undefined; + return sanitizeNoticeTitle(title); } function findMetaContent($: CheerioAPI, name: string): string | undefined { @@ -398,15 +400,19 @@ function isLikelyEisNoticeUrl(url: string, patterns?: string[]): boolean { function getDetailUrlPriority(url: string): number { if (url.includes("/view/common-info.html")) { - return 3; + return 5; + } + + if (url.includes("common-info.html") || url.includes("contract-info.html")) { + return 4; } - if (url.includes("common-info.html")) { - return 2; + if (url.includes("/documents.html")) { + return -2; } if (url.includes("/printForm/")) { - return 0; + return -3; } return 1; @@ -512,7 +518,22 @@ function sanitizeRegion(value: string | undefined): string | undefined { return cleaned; } +function sanitizeNoticeTitle(value: string | undefined): string | undefined { + return sanitizeBoilerplateText(value, { maxLength: 400 }); +} + +function sanitizeNoticeDescription(value: string | undefined): string | undefined { + return sanitizeBoilerplateText(value, { maxLength: 4_000 }); +} + function sanitizePartyName(value: string | undefined): string | undefined { + return sanitizeBoilerplateText(value, { maxLength: 220 }); +} + +function sanitizeBoilerplateText( + value: string | undefined, + options?: { maxLength?: number } +): string | undefined { const cleaned = cleanText(value); if (!cleaned) { @@ -524,9 +545,10 @@ function sanitizePartyName(value: string | undefined): string | undefined { const hasBoilerplateMarker = EIS_BOILERPLATE_MARKERS.some((marker) => normalized.includes(marker)); const hasPlatformNoise = EIS_PLATFORM_DOMAIN_MARKERS.filter((marker) => normalized.includes(marker)).length >= 2; + const maxLength = options?.maxLength ?? 220; if ( - cleaned.length > 220 || + cleaned.length > maxLength || urlMatches.length >= 3 || hasBoilerplateMarker || hasPlatformNoise || diff --git a/src/sources/eis/eis-source.ts b/src/sources/eis/eis-source.ts index b68e591..78e8410 100644 --- a/src/sources/eis/eis-source.ts +++ b/src/sources/eis/eis-source.ts @@ -189,7 +189,14 @@ function isRelevantNppItem( return true; } - if (resolveNppStationNameFromText([notice.title, notice.description, options?.matchedQuery])) { + if ( + resolveNppStationNameFromText([ + notice.title, + notice.description, + notice.customerName, + notice.supplierName + ]) + ) { return true; } From f684cc3619d5bb8976e13588fc0f877c3a19f3f5 Mon Sep 17 00:00:00 2001 From: Aleksandr Minkin Date: Wed, 8 Apr 2026 21:55:04 +0300 Subject: [PATCH 2/2] chore: add production deploy workflow --- .github/workflows/production-deploy.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/production-deploy.yml diff --git a/.github/workflows/production-deploy.yml b/.github/workflows/production-deploy.yml new file mode 100644 index 0000000..c6d702c --- /dev/null +++ b/.github/workflows/production-deploy.yml @@ -0,0 +1,19 @@ +name: Production Deploy + +on: + push: + branches: ["main", "master"] + workflow_dispatch: + +permissions: + contents: read + +jobs: + deploy: + uses: nppweb/infra/.github/workflows/deploy.yml@main + with: + repo_name: scrape-helper + branch: ${{ github.ref_name }} + target_sha: ${{ github.sha }} + auto_rollback: true + secrets: inherit