diff --git a/README.md b/README.md index a9867da..7c14cf2 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,18 @@ CALENDAR_TEST_API_KEY=<локальный ключ Calendar MCP> task test:e2e ``` +Для отдельной проверки настоящей AI-модели сначала подними model-профиль в `deploy`, затем выполни: + +```powershell +task test:model +``` + +Этот opt-in сценарий отправляет фразу «Поставь завтра в 19:00 созвон по проекту на полчаса», проверяет +дату, время, длительность и часовой пояс созданного действия, а затем отменяет его. Он не входит в +обычный CI: скорость и доступность локальной либо внешней модели не должны делать детерминированный +pipeline нестабильным. Максимальное время ответа по умолчанию — 120 секунд; локально его можно +изменить параметром `-MaxSeconds` у `scripts/run-natural-calendar.ps1`. + Runner сам получает короткоживущий JWT у локального Keycloak. В CI вместо тестового логина и пароля можно передать готовый `ACTION_TOKEN`. Скрипт принимает только локальные HTTP-адреса. Проверочный API `fake-calendar` доступен только в тестовом режиме и требует отдельный `X-Test-Key`; секреты не diff --git a/SERVICE.md b/SERVICE.md index 711c89a..36ce943 100644 --- a/SERVICE.md +++ b/SERVICE.md @@ -8,5 +8,6 @@ | Chaos | отдельный ручной запуск, выключен по умолчанию | | Первый acceptance-путь | Channel Gateway → Conversation Service → Agent Runtime → Widget decision через Channel Gateway → Action Service → Temporal → Calendar MCP | | Локальный запуск | `task test:e2e` против уже поднятого окружения | +| Проверка AI-модели | `task test:model` против model-профиля; не входит в обычный CI | | Граница ответственности | Не поднимает сервисы и не содержит их component-тесты | diff --git a/Taskfile.yml b/Taskfile.yml index 358dc25..bdc4324 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -34,6 +34,11 @@ tasks: cmds: - '{{.POWERSHELL}} {{.POWERSHELL_ARGS}} -File ./scripts/run-calendar.ps1' + test:model: + desc: Проверить понимание естественной фразы локальной AI-моделью + cmds: + - '{{.POWERSHELL}} {{.POWERSHELL_ARGS}} -File ./scripts/run-natural-calendar.ps1' + test:load: desc: Выполнить настраиваемый нагрузочный тест cmds: diff --git a/scripts/check.ps1 b/scripts/check.ps1 index 40887aa..ccc05c1 100644 --- a/scripts/check.ps1 +++ b/scripts/check.ps1 @@ -1,5 +1,5 @@ $ErrorActionPreference = "Stop" -foreach ($file in @(".env.example", "compose.yaml", "Taskfile.yml", "tests/smoke.js", "tests/calendar-event.js", "scripts/run-calendar.ps1", "chaos/pod-delay.yaml", "README.md", "AGENTS.md", "SERVICE.md")) { +foreach ($file in @(".env.example", "compose.yaml", "Taskfile.yml", "tests/smoke.js", "tests/calendar-event.js", "tests/natural-calendar.js", "scripts/run-calendar.ps1", "scripts/run-natural-calendar.ps1", "chaos/pod-delay.yaml", "README.md", "AGENTS.md", "SERVICE.md")) { if (-not (Test-Path $file)) { throw "Required file is missing: $file" } } if (-not (Get-Command docker -ErrorAction SilentlyContinue)) { throw "Docker is not installed." } @@ -27,8 +27,18 @@ foreach ($required in @("ACTION_TOKEN", "KEYCLOAK_URL", "OIDC_REALM", "OIDC_CLIE if ($runner -notmatch [regex]::Escape($required)) { throw "Calendar runner does not support $required." } } +$naturalScript = Get-Content tests/natural-calendar.js -Raw +foreach ($required in @("завтра в 19:00", "на полчаса", "Europe/Moscow", "CANCEL", "CANCELLED", "ACTION_TOKEN")) { + if ($naturalScript -notmatch [regex]::Escape($required)) { throw "Natural-language test does not contain $required." } +} + +$naturalRunner = Get-Content scripts/run-natural-calendar.ps1 -Raw +foreach ($required in @("MODEL_TEST_MAX_SECONDS", "natural-calendar.js", "run-calendar.ps1")) { + if ($naturalRunner -notmatch [regex]::Escape($required)) { throw "Natural-language runner does not support $required." } +} + $taskfile = Get-Content Taskfile.yml -Raw -foreach ($required in @("verify:", "test:smoke:", "test:e2e:", "test:load:")) { +foreach ($required in @("verify:", "test:smoke:", "test:e2e:", "test:model:", "test:load:")) { if ($taskfile -notmatch [regex]::Escape($required)) { throw "Taskfile does not contain $required." } } diff --git a/scripts/run-calendar.ps1 b/scripts/run-calendar.ps1 index af46525..c64eff5 100644 --- a/scripts/run-calendar.ps1 +++ b/scripts/run-calendar.ps1 @@ -3,7 +3,9 @@ param( [string]$ActionUrl = "", [string]$CalendarTestUrl = "", [string]$KeycloakUrl = "", - [string]$DockerNetwork = "" + [string]$DockerNetwork = "", + [ValidateSet("calendar-event.js", "natural-calendar.js")] + [string]$TestFile = "calendar-event.js" ) $ErrorActionPreference = "Stop" @@ -70,7 +72,7 @@ $dockerArgs += @( "--env", "CALENDAR_TEST_URL=$CalendarTestUrl", "--env", "CALENDAR_TEST_API_KEY=$calendarTestKey", "--env", "ACTION_TOKEN=$token", - $k6Image, "run", "/tests/calendar-event.js" + $k6Image, "run", "/tests/$TestFile" ) & docker @dockerArgs diff --git a/scripts/run-natural-calendar.ps1 b/scripts/run-natural-calendar.ps1 new file mode 100644 index 0000000..bb355dc --- /dev/null +++ b/scripts/run-natural-calendar.ps1 @@ -0,0 +1,16 @@ +param( + [ValidateRange(30, 600)] + [int]$MaxSeconds = 120 +) + +$ErrorActionPreference = "Stop" +$oldMaxSeconds = [Environment]::GetEnvironmentVariable("MODEL_TEST_MAX_SECONDS", "Process") + +try { + [Environment]::SetEnvironmentVariable("MODEL_TEST_MAX_SECONDS", $MaxSeconds, "Process") + & "$PSScriptRoot/run-calendar.ps1" -TestFile "natural-calendar.js" + if ($LASTEXITCODE -ne 0) { throw "Natural-language calendar scenario failed." } +} +finally { + [Environment]::SetEnvironmentVariable("MODEL_TEST_MAX_SECONDS", $oldMaxSeconds, "Process") +} diff --git a/tests/natural-calendar.js b/tests/natural-calendar.js new file mode 100644 index 0000000..e528f3f --- /dev/null +++ b/tests/natural-calendar.js @@ -0,0 +1,131 @@ +import http from 'k6/http'; +import { check, fail, sleep } from 'k6'; + +const channelUrl = requiredUrl('CHANNEL_URL'); +const actionUrl = requiredUrl('ACTION_URL'); +const userToken = required('ACTION_TOKEN'); +const maxSeconds = Number(__ENV.MODEL_TEST_MAX_SECONDS || 120); +const timeZone = 'Europe/Moscow'; + +export const options = { + scenarios: { + natural_calendar: { + executor: 'shared-iterations', + vus: 1, + iterations: 1, + maxDuration: `${maxSeconds}s`, + }, + }, + thresholds: { + checks: ['rate==1'], + http_req_failed: ['rate==0'], + http_req_duration: [`max<${maxSeconds * 1000}`], + }, +}; + +export default function () { + const requestKey = `natural-calendar-${Date.now()}-${__VU}-${__ITER}`; + const card = askAgent(requestKey); + const action = getAction(card.actionId); + const startAt = Date.parse(action.payload?.startAt); + const endAt = Date.parse(action.payload?.endAt); + + check(action, { + 'action waits for approval': (item) => item.status === 'AWAITING_APPROVAL', + 'agent understands tomorrow at 19:00 Moscow time': () => startAt === expectedStart(), + 'agent understands half an hour': () => endAt - startAt === 30 * 60 * 1000, + 'agent keeps the trusted time zone': (item) => item.payload?.timeZone === timeZone, + 'agent keeps a useful title': (item) => /проект/i.test(item.payload?.title || ''), + }); + + cancelAction(card); + check(waitForCancelled(card.actionId), { + 'test action is cancelled': (item) => item.status === 'CANCELLED', + }); +} + +function askAgent(requestKey) { + const response = http.post( + `${channelUrl}/api/v1/conversations/messages`, + JSON.stringify({ + requestKey, + text: 'Поставь завтра в 19:00 созвон по проекту на полчаса', + context: { + locale: 'ru-RU', + timeZone, + }, + }), + authHeaders(), + ); + expectStatus(response, 200, 'agent did not return a confirmation'); + const card = response.json().reply?.card; + if (card?.widget !== 'action_confirmation') { + fail('agent returned no action confirmation'); + } + return card; +} + +function getAction(actionId) { + const response = http.get(`${actionUrl}/api/v1/actions/${actionId}`, authHeaders()); + expectStatus(response, 200, 'action-service did not return the action'); + return response.json(); +} + +function cancelAction(card) { + const response = http.post( + `${channelUrl}/api/v1/actions/${card.actionId}/decisions`, + JSON.stringify({ decision: 'CANCEL', payloadHash: card.payloadHash }), + authHeaders(), + ); + expectStatus(response, 202, 'action-service did not accept cancellation'); +} + +function waitForCancelled(actionId) { + for (let attempt = 0; attempt < 10; attempt += 1) { + const action = getAction(actionId); + if (action.status === 'CANCELLED') { + return action; + } + sleep(0.2); + } + fail('action was not cancelled'); +} + +function expectedStart() { + const moscowNow = new Date(Date.now() + 3 * 60 * 60 * 1000); + return Date.UTC( + moscowNow.getUTCFullYear(), + moscowNow.getUTCMonth(), + moscowNow.getUTCDate() + 1, + 16, + 0, + 0, + ); +} + +function authHeaders() { + return { + headers: { + Authorization: `Bearer ${userToken}`, + 'Content-Type': 'application/json', + }, + }; +} + +function expectStatus(response, expected, message) { + if (response.status !== expected) { + fail(`${message}: expected ${expected}, got ${response.status}`); + } +} + +function required(name) { + const value = __ENV[name]; + if (!value) { + throw new Error(`${name} is required`); + } + return value; +} + +function requiredUrl(name) { + return required(name).replace(/\/$/, ''); +}