diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index f789d87..6bd3ae2 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -20,9 +20,9 @@ jobs: # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@a37ce9120846195fa4ece8f58b268e6043cb2f26 # v3.7.0 - name: CI/CD ${{ matrix.node-version }} - uses: actions/setup-node@v3 + uses: actions/setup-node@3235b876344d2a9aa001b8d1453c930bba69e610 # v3.9.1 with: node-version: ${{ matrix.node-version }} cache: 'npm' @@ -31,7 +31,7 @@ jobs: - run: npm i - run: npm run compile --if-present - run: npm run coverage-all --if-present - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: code-coverage-${{ matrix.node-version }} path: coverage/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..63ed593 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,106 @@ +# leo-sdk-nodejs (package `leo-sdk`) + +## Overview + +Node/TypeScript SDK that writes events to and reads events from an RStreams (LEO) bus. +Entry point is `index.js` (`RStreamsSdk`, typed by `index.d.ts`). It is the reference +implementation other-language SDKs are measured against, so observable behavior (DynamoDB +params, call order, option names) is load-bearing even where it looks accidental. +Version lives in `package.json`; GitHub repo `LeoPlatform/Nodejs`. + +## Tech Stack + +- Node 16/18/20/22 (CI matrix in `.github/workflows/node.js.yml`), TypeScript 5.x, CommonJS output. +- AWS SDK v3 clients (`@aws-sdk/client-kinesis`, `client-firehose`, `client-s3`, + `client-dynamodb` + `lib-dynamodb`, `client-lambda`, `client-secrets-manager`, `lib-storage`). +- Streams: `through2`, `pump`, `pumpify`, `event-stream`, `split2`, `flush-write-stream`. +- Tests: mocha + chai + sinon + sinon-chai, coverage by nyc. + +## Source layout + +- `index.js` / `index.d.ts` — SDK facade (`read`, `load`, `enrich`, `offload`, `putEvent`, `bot`, `streams`, `aws`). +- `lib/stream/leo-stream.js` — the core: `toLeo`, `load`, `fromLeo`, `process`, `enrich`, `offload`, + `toCheckpoint`, `toDynamoDB`, `toS3`, `fromS3`, `magic`, `autoSwitch`, `createCorrelation`. +- `lib/stream/helper/` — `leos3.js`, `chunkEventStream.js` (hand-written JS); `leo-stream-helper.ts`, + `parser-util.ts`, `worker-pool.ts` and friends (TypeScript). +- `lib/streams.js` — stream primitives (`through`, `buffer`, `batch`, `pipe`, `eventIdFromTimestamp`, + `eventIdToTimestamp`). +- `lib/cron.js` — bot/cron table access: `checkpoint`, `checkLock`, `reportComplete`, `start`, `end`, `get`, + `shouldRun`, `createBot`, `subscribe`. +- `lib/dynamodb.js` — DynamoDB document-client helpers (`get`, `put`, `merge`, `update`, `query`, + `batchGetTable`, `createTableWriteStream`, `getSetting`, `saveSetting`). +- `lib/rstreams-config-provider-chain.ts` — config resolution chain (`ConfigProviderChain`, `EnvironmentConfiguration`, + `FileTreeConfiguration`, `LeoConfiguration`, `ObjectConfiguration`, `AWSSecretsConfiguration`). +- `lib/event-trace.ts` — upstream/downstream event tracing via the cron and stats tables. +- `lib/reference.js` — `queue:`/`bot:`/`system:` reference parsing (`ref`, `refId`, `botRef`). +- `wrappers/` — Lambda wrappers: `cron.js`, `fanout.js` (multi-instance fan-out), `wrapAsAsync.js`. +- `lib/mock-*.ts`, `lib/mock.js` — in-memory SDK doubles used by consumers' tests. +- `connectors/`, `lib/apis/`, `lib/template/` — legacy, excluded from coverage. + +## Build & Test + +```bash +npm ci +npx tsc --watch false # always pass --watch false; tsconfig.json has "watch": true +npm run coverage-all # nyc --all + mocha over every compiled *.utest.js +npx mocha --exit test/lib.dynamodb.utest.js # run one compiled test file +``` + +- Tests are TypeScript in `test/*.utest.ts`; mocha runs the compiled `test/*.utest.js`. Compile first, every time. +- `.js` compiled from `.ts` is gitignored (`.gitignore` has `*.js`, `!*.ts`). Hand-written JS sources such as + `lib/cron.js`, `lib/dynamodb.js`, `lib/streams.js`, `lib/stream/leo-stream.js`, `wrappers/fanout.js` are tracked; + a brand-new `.js` source needs `git add -f`. +- `.nycrc.json` is the repo default (TypeScript sources are remapped to `.ts`). To report against compiled JS lines, pass `--no-source-map --no-cache` to nyc on the command line. + `coverage/` and `.nyc_output/` are gitignored; never commit them. +- `npm run compile` also builds the worker-thread bundle via webpack; not needed for tests. +- CI (`node.js.yml`) installs v2 `aws-sdk` and the v3 clients with `--no-save`, then runs `compile` and `coverage-all`. + +## Test conventions + +- Style reference: `test/lib.cron.utest.ts` — `sinon.createSandbox()`, `afterEach(() => sandbox.restore())`, chai + `assert`/`expect`. +- Stub DynamoDB by stubbing `DynamoDBDocument.from` (from `@aws-sdk/lib-dynamodb`) to return + `{ get, put, update, query, batchGet, batchWrite }` stubs that take `(params, callback)`. +- After a module has been built, its clients can be stubbed on the instance: `ls.kinesis.putRecords`, + `ls.firehose.putRecordBatch`, `ls.dynamodb.docClient.query`, `ls.s3.send`, `ls.cron.checkpoint`, `ls.toS3`, `ls.fromS3`. +- Call `delete process.__config` in `beforeEach`; `process.__config.registry` is shared by every + SDK/cron/stream instance in the process. +- Any test touching the `wrappers/fanout.js` factory must restore `leo.streams.fromLeo`, `leo.read`, + `leo.bot.checkpoint`, `leo.bot.checkLock`, `leo.bot.reportComplete` afterwards (see `test/wrappers.fanout.utest.ts`). +- Use a short fibonacci backoff in tests (`backoff: { initialDelay: 1, maxDelay: 2 }`); `maxDelay` must be greater + than `initialDelay`. +- `wrappers/fanout.utest.js` is a tracked hand-written test; the TypeScript fanout tests are in `test/wrappers.fanout.utest.ts`. + +## Coding conventions + +- Tabs for indentation; double quotes in TypeScript, mixed in legacy JS — match the file. +- Callback-first APIs (`(err, data)`) with promise wrappers added later; streams are object-mode. +- Errors are frequently strings (`"Stale Checkpoint"`, `"already running"`), not `Error` objects. Keep them. +- Options objects are merged with `Object.assign({defaults}, opts)`; option names are public API. + +## Key patterns + +- Write path: `load` -> `through` (wrap payload) -> size guard (> 614400 bytes goes to S3) -> `toLeo` + (chunk, gzip, Kinesis/Firehose, emits `__cmd: "checkpoint"`) -> `toCheckpoint` (cron table). +- Read path: `fromLeo` does one `batchGet` (cron row + event row), then pages the stream table with + `query`, prefetching the next page; items are inline `gzip` or `s3` pointers read via `fromS3`. +- Event ids look like `z/YYYY/MM/DD/HH/mm/-<7 digit record>`; compare as strings. +- Config resolution order is defined in `ConfigProviderChain.defaultProviders` (leo-config first). + +## Domain context + +- Bus tables: `LeoStream` (events), `LeoEvent` (per-queue state, `max_eid`), `LeoCron` (bots, checkpoints, locks), + `LeoSystem`, `LeoSettings`; plus `LeoS3` bucket, `LeoKinesisStream`, `LeoFirehoseStream`. +- A checkpoint is per bot, per queue, `read` or `write`; `cron.checkpoint` uses a conditional update when a read + checkpoint is expected and reports `"Stale Checkpoint"` on conflict. + +## Gotchas + +- `lib/cron.js` `start()` with no callback returns a promise whose overridden `then` returns `undefined`; + `cron.run()` depends on it. +- `wrappers/fanout.js`: when the second argument is not a function it replaces the whole options object, so a third + argument is ignored. Each factory call wraps the previous override on the shared `leo` singleton. +- `lib/stream/leo-stream.js` `toCheckpoint` builds its async tasks inside a `var` loop, so every task checkpoints the + last bot/queue. +- `lib/event-trace.ts` formats eids from local time (no `.utc()`), so results depend on the process time zone. +- Do not run `npx tsc` without `--watch false`; it never exits. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..43c994c --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/test/lib.cron.lifecycle.utest.ts b/test/lib.cron.lifecycle.utest.ts new file mode 100644 index 0000000..7d85d34 --- /dev/null +++ b/test/lib.cron.lifecycle.utest.ts @@ -0,0 +1,654 @@ +import sinon from "sinon"; +import { expect, assert } from "chai"; +import zlib from "zlib"; +import cronLib from "../lib/cron"; +import { DynamoDBDocument } from "@aws-sdk/lib-dynamodb"; + +describe("lib/cron.js lifecycle", function () { + const sandbox = sinon.createSandbox(); + + function build(doc: Record, registry?: any) { + const wrapped = DynamoDBDocument.from as any; + if (wrapped.restore) { + wrapped.restore(); + } + sandbox.stub(DynamoDBDocument, "from").returns(doc as unknown as DynamoDBDocument); + const configure: any = { + onUpdate: () => { }, + resources: { LeoCron: "MockLeoCron", LeoSettings: "MockLeoSettings", LeoSystem: "MockLeoSystem" }, + aws: {}, + registry + }; + return { cron: (cronLib as any)(configure), configure }; + } + + beforeEach(() => { + delete global.cron_run_again; + delete global.preventRunAgain; + delete process["__config"]; + }); + + afterEach(() => { + sandbox.restore(); + }); + + describe("update", function () { + it("update merges the bot onto the stored cron row and resolves Success", async function () { + const get = sandbox.stub().callsFake((p, cb) => cb(null, { Item: { id: "b1", name: "old", paused: false } })); + const put = sandbox.stub().callsFake((p, cb) => cb(null, {})); + const { cron } = build({ get, put }); + const result = await cron.update({ id: "b1", name: "new" }); + assert.equal(result, "Success"); + assert.equal(get.getCall(0).args[0].TableName, "MockLeoCron"); + assert.deepEqual(put.getCall(0).args[0].Item, { id: "b1", name: "new", paused: false }); + }); + + it("update also invokes the callback", async function () { + const get = sandbox.stub().callsFake((p, cb) => cb(null, {})); + const put = sandbox.stub().callsFake((p, cb) => cb(null, {})); + const { cron } = build({ get, put }); + const cbArgs = await new Promise((resolve) => { + cron.update({ id: "b1" }, (...args) => resolve(args)); + }); + assert.deepEqual(cbArgs, [null, "Success"]); + }); + + it("update rejects and calls back with the failure", async function () { + const boom = new Error("get failed"); + const get = sandbox.stub().callsFake((p, cb) => cb(boom)); + const { cron } = build({ get }); + const cb = sandbox.stub(); + let err; + try { await cron.update({ id: "b1" }, cb); } catch (e) { err = e; } + assert.equal(err, boom); + assert.equal(cb.getCall(0).args[0], boom); + }); + }); + + describe("subscribe/createBot", function () { + it("subscribe requires id and queue", async function () { + const { cron } = build({}); + let err; + try { await cron.subscribe("", "q"); } catch (e) { err = e; } + assert.equal(err.message, "'id' and 'queue' are required"); + err = undefined; + try { await cron.subscribe("bot1"); } catch (e) { err = e; } + assert.equal(err.message, "'id' and 'queue' are required"); + }); + + it("subscribe creates a bot that triggers on the queue and resolves the bot ref", async function () { + const update = sandbox.stub().callsFake((p, cb) => cb(null, {})); + const { cron } = build({ update }); + const result = await cron.subscribe("bot1", "q1", { name: "My Bot" }); + assert.deepEqual(result, { refId: "bot:bot1" }); + const params = update.getCall(0).args[0]; + assert.equal(params.TableName, "MockLeoCron"); + assert.deepEqual(params.Key, { id: "bot1" }); + assert.equal(params.ExpressionAttributeValues[":name"], "My Bot"); + assert.equal(params.ExpressionAttributeValues[":lambdaName"], "bot1"); + assert.equal(params.ExpressionAttributeValues[":executionType"], "lambda"); + assert.deepEqual(params.ExpressionAttributeValues[":triggers"], ["q1", "queue:q1"]); + assert.deepEqual(params.ExpressionAttributeValues[":checkpoints"], { + read: { "queue:q1": {} }, + write: {} + }); + assert.deepEqual(params.ExpressionAttributeValues[":lambda"], { settings: [{ source: "q1" }] }); + assert.include(params.UpdateExpression, "#instances = if_not_exists(#instances, :instances)"); + assert.include(params.UpdateExpression, "#checkpoints = if_not_exists(#checkpoints, :checkpoints)"); + assert.include(params.UpdateExpression, "#requested_kinesis = if_not_exists(#requested_kinesis, :requested_kinesis)"); + assert.include(params.UpdateExpression, "#name = :name"); + }); + + it("subscribe accepts an array of queues as triggers", async function () { + const update = sandbox.stub().callsFake((p, cb) => cb(null, {})); + const { cron } = build({ update }); + await cron.subscribe("bot1", ["q1", "q2"]); + const values = update.getCall(0).args[0].ExpressionAttributeValues; + assert.deepEqual(values[":triggers"], ["q1", "q2", "queue:q1,q2"]); + }); + + it("createBot builds write checkpoints from the destination and skips source trigger when settings.time is set", async function () { + const update = sandbox.stub().callsFake((p, cb) => cb(null, {})); + const { cron } = build({ update }); + await cron.createBot("bot1", { + triggers: ["t1"], + settings: { source: "src", destination: "dest", time: "0 * * * *" } + }); + const values = update.getCall(0).args[0].ExpressionAttributeValues; + assert.deepEqual(values[":triggers"], ["t1"]); + assert.deepEqual(values[":checkpoints"], { + read: { "queue:t1": {}, "queue:src": {} }, + write: { "queue:dest": {} } + }); + }); + + it("createBot rejects when the dynamodb update fails", async function () { + const boom = new Error("update failed"); + const update = sandbox.stub().callsFake((p, cb) => cb(boom)); + const { cron } = build({ update }); + let err; + try { await cron.createBot("bot1", { settings: {} }); } catch (e) { err = e; } + assert.equal(err, boom); + }); + + it("createBot merges caller opts into the update options", async function () { + const update = sandbox.stub().callsFake((p, cb) => cb(null, {})); + const { cron } = build({ update }); + await cron.createBot("bot1", { settings: {} }, { ReturnValues: "ALL_NEW" }); + assert.equal(update.getCall(0).args[0].ReturnValues, "ALL_NEW"); + }); + }); + + describe("runAgain/setMessage/getLastResult", function () { + it("runAgain sets the global run-again flag", function () { + const { cron } = build({}); + assert.isUndefined(global.cron_run_again); + cron.runAgain(); + assert.equal(global.cron_run_again, true); + }); + + it("setMessage wraps a string as a msg entry on the registry", function () { + const { cron, configure } = build({}); + cron.setMessage("hello"); + assert.deepEqual(configure.registry.cron_message, [{ msg: "hello" }]); + }); + + it("setMessage stores non-string messages as given", function () { + const { cron, configure } = build({}); + const msg = [{ msg: "a", level: "warn" }]; + cron.setMessage(msg); + assert.equal(configure.registry.cron_message, msg); + }); + + it("getLastResult decodes the gzipped base64 result", function () { + const encoded = zlib.gzipSync(JSON.stringify({ ok: true })).toString("base64"); + const { cron } = build({}, { __cron: { lastResult: encoded } }); + assert.deepEqual(cron.getLastResult(), { ok: true }); + }); + + it("getLastResult returns null when there is no result", function () { + const { cron } = build({}, { __cron: {} }); + assert.isNull(cron.getLastResult()); + }); + }); + + describe("getAttachedSystem/getAttachedSystemByAlias", function () { + const systems = { + s1: { id: "s1", aliases: ["source", "default"] }, + s2: { id: "s2", aliases: ["destination"] } + }; + + it("getAttachedSystem looks up by system id", function () { + const { cron } = build({}, { __cron: { systems } }); + assert.equal(cron.getAttachedSystem("s2"), systems.s2); + }); + + it("getAttachedSystem strips a system. prefix", function () { + const { cron } = build({}, { __cron: { systems } }); + assert.equal(cron.getAttachedSystem("system.s1"), systems.s1); + }); + + it("getAttachedSystem accepts a system: reference", function () { + const { cron } = build({}, { __cron: { systems } }); + assert.equal(cron.getAttachedSystem("system:s1"), systems.s1); + }); + + it("getAttachedSystem returns undefined for an unknown system or no registry", function () { + const { cron } = build({}, { __cron: { systems } }); + assert.isUndefined(cron.getAttachedSystem("nope")); + delete process["__config"]; + const other = build({}); + assert.isUndefined(other.cron.getAttachedSystem("s1")); + }); + + it("getAttachedSystemByAlias returns the first system carrying the alias", function () { + const { cron } = build({}, { __cron: { systems } }); + assert.equal(cron.getAttachedSystemByAlias("destination"), systems.s2); + assert.equal(cron.getAttachedSystemByAlias("default"), systems.s1); + }); + + it("getAttachedSystemByAlias returns undefined when no alias matches", function () { + const { cron } = build({}, { __cron: { systems } }); + assert.isUndefined(cron.getAttachedSystemByAlias("missing")); + delete process["__config"]; + const other = build({}); + assert.isUndefined(other.cron.getAttachedSystemByAlias("source")); + }); + }); + + describe("get/buildPayloads", function () { + it("get builds a payload for a stored bot and registers checkpoints", async function () { + const row = { + id: "b1", + name: "Bot One", + lambdaName: "lam", + trigger: 111, + instances: { "0": {} }, + checkpoints: { read: { "queue:q1": { checkpoint: "z/1" } }, write: {} }, + lambda: { settings: [{ source: "queue:q1", batch: 5 }] } + }; + const get = sandbox.stub().callsFake((p, cb) => cb(null, { Item: row })); + const { cron, configure } = build({ get }); + sandbox.stub(Date, "now").returns(5000); + const payload: any = await new Promise((resolve, reject) => cron.get("b1", {}, (err, d) => err ? reject(err) : resolve(d))); + assert.deepEqual(get.getCall(0).args[0].Key, { id: "b1" }); + assert.deepEqual(payload.__cron, { + id: "b1", + name: "lam", + ts: 111, + time: 5000, + checkpoints: row.checkpoints, + botName: "Bot One", + instances: row.instances, + iid: "0" + }); + assert.equal(payload.botId, "b1"); + assert.equal(payload.source, "queue:q1"); + assert.equal(payload.batch, 5); + assert.deepEqual(configure.registry.__cron.checkpoints.read["queue:q1"], { checkpoint: "z/1" }); + }); + + it("get applies overrides and the requested instance", async function () { + const get = sandbox.stub().callsFake((p, cb) => cb(null, { Item: { id: "b1", lambda: { settings: [{ a: 1 }] } } })); + const { cron } = build({ get }); + const payload: any = await new Promise((resolve, reject) => cron.get("b1", { instance: 3, overrides: { a: 2, extra: true } }, (err, d) => err ? reject(err) : resolve(d))); + assert.equal(payload.a, 2); + assert.isTrue(payload.extra); + assert.equal(payload.__cron.iid, "3"); + }); + + it("get returns one payload per instance when instances is given", async function () { + const get = sandbox.stub().callsFake((p, cb) => cb(null, { Item: { id: "b1", namedSettings: { b: { flag: "B" } } } })); + const { cron } = build({ get }); + const payloads: any[] = await new Promise((resolve, reject) => cron.get("b1", { instances: ["a", "b"] }, (err, d) => err ? reject(err) : resolve(d))); + assert.equal(payloads.length, 2); + assert.deepEqual(payloads.map((p) => p.__cron.iid), ["a", "b"]); + assert.isUndefined(payloads[0].flag); + assert.equal(payloads[1].flag, "B"); + }); + + it("get falls back to a stub row when the bot does not exist", async function () { + const get = sandbox.stub().callsFake((p, cb) => cb(null, {})); + const { cron } = build({ get }); + const payload: any = await new Promise((resolve, reject) => cron.get("bot:b9", {}, (err, d) => err ? reject(err) : resolve(d))); + assert.equal(payload.__cron.id, "b9"); + assert.equal(payload.botId, "b9"); + }); + + it("get with register false leaves the registry alone", async function () { + const get = sandbox.stub().callsFake((p, cb) => cb(null, { Item: { id: "b1", checkpoints: { read: { q: { checkpoint: "x" } } } } })); + const { cron, configure } = build({ get }); + await new Promise((resolve, reject) => cron.get("b1", { register: false }, (err, d) => err ? reject(err) : resolve(d))); + assert.isUndefined(configure.registry.__cron); + }); + + it("get passes the dynamodb error to the callback", async function () { + const boom = new Error("get failed"); + const get = sandbox.stub().callsFake((p, cb) => cb(boom)); + const { cron } = build({ get }); + const err = await new Promise((resolve) => cron.get("b1", {}, (e) => resolve(e))); + assert.equal(err, boom); + }); + + it("buildPayloads sends the requested checkpoint for the source", async function () { + const { cron } = build({}); + const [payload]: any[] = await cron.buildPayloads({ + id: "b1", + checkpoint: "z/9", + lambda: { settings: [{ source: "queue:q1" }] } + }, {}, {}); + assert.deepEqual(payload.__cron.checkpoints, { read: { "queue:q1": { checkpoint: "z/9" } } }); + }); + + it("buildPayloads defaults to instance 0 and merges namedSettings", async function () { + const { cron } = build({}); + const [payload]: any[] = await cron.buildPayloads({ + id: "b1", + lambda: { settings: { x: 1 } }, + namedSettings: { "0": { y: 2 } } + }, {}, undefined); + assert.equal(payload.x, 1); + assert.equal(payload.y, 2); + assert.equal(payload.__cron.iid, "0"); + }); + + it("buildPayloads loads attached systems from the system table", async function () { + const batchGet = sandbox.stub().callsFake((p, cb) => cb(null, { Responses: { MockLeoSystem: [] } })); + const { cron } = build({ batchGet }); + const [payload]: any[] = await cron.buildPayloads({ + id: "b1", + system: { id: "sys-default" }, + lambda: { settings: [{ source: "system:sys-src", destination: "system:sys-dest" }] } + }, {}, {}); + assert.deepEqual(batchGet.getCall(0).args[0], { + RequestItems: { MockLeoSystem: { Keys: [{ id: "sys-dest" }, { id: "sys-src" }, { id: "sys-default" }] } } + }); + assert.deepEqual(Object.keys(payload.__cron.systems).sort(), ["sys-default", "sys-dest", "sys-src"]); + assert.deepEqual(payload.__cron.systems["sys-src"].aliases, ["source"]); + assert.deepEqual(payload.__cron.systems["sys-dest"].aliases, ["destination"]); + assert.deepEqual(payload.__cron.systems["sys-default"].aliases, ["default"]); + }); + + it("buildPayloads rejects when the system lookup fails", async function () { + const boom = new Error("batchGet failed"); + const batchGet = sandbox.stub().callsFake((p, cb) => cb(boom)); + const { cron } = build({ batchGet }); + let err; + try { + await cron.buildPayloads({ id: "b1", system: { id: "s" } }, {}, {}); + } catch (e) { err = e; } + assert.equal(err, boom); + }); + }); + + describe("hasMoreToDo", function () { + it("hasMoreToDo is true when a trigger queue has no checkpoint", function () { + const { cron } = build({}); + assert.isTrue(cron.hasMoreToDo({}, { + triggers: ["q1"], requested_kinesis: { q1: "z/2" }, checkpoints: { read: {} } + })); + }); + + it("hasMoreToDo is true when the latest event is past the checkpoint", function () { + const { cron } = build({}); + assert.isTrue(cron.hasMoreToDo({}, { + triggers: ["q1"], requested_kinesis: { q1: "z/2" }, checkpoints: { read: { q1: { checkpoint: "z/1" } } } + })); + }); + + it("hasMoreToDo is false when the checkpoint has caught up", function () { + const { cron } = build({}); + assert.isFalse(cron.hasMoreToDo({}, { + triggers: ["q1"], requested_kinesis: { q1: "z/2" }, checkpoints: { read: { q1: { checkpoint: "z/2" } } } + })); + }); + + it("hasMoreToDo ignores queues that are not triggers", function () { + const { cron } = build({}); + assert.isFalse(cron.hasMoreToDo({}, { + triggers: ["q1"], requested_kinesis: { other: "z/9" }, checkpoints: { read: {} } + })); + }); + + it("hasMoreToDo ignores triggers when ignoreHasMore is set", function () { + const { cron } = build({}); + assert.isFalse(cron.hasMoreToDo({}, { + triggers: ["q1"], ignoreHasMore: true, requested_kinesis: { q1: "z/9" }, checkpoints: { read: {} } + })); + }); + + it("hasMoreToDo restricts to the given key", function () { + const { cron } = build({}); + const image = { triggers: ["q1", "q2"], requested_kinesis: { q1: "z/9", q2: "z/9" }, checkpoints: { read: { q1: { checkpoint: "z/9" } } } }; + assert.isFalse(cron.hasMoreToDo({}, image, "q1")); + assert.isTrue(cron.hasMoreToDo({}, image, "q2")); + }); + + it("hasMoreToDo is false without requested_kinesis", function () { + const { cron } = build({}); + assert.isFalse(cron.hasMoreToDo({}, { triggers: ["q1"] })); + }); + }); + + describe("shouldRun", function () { + const NOW = 1_700_000_000_000; + + function run(cron, oldImage, newImage, cache) { + return new Promise((resolve) => cron.shouldRun(oldImage, newImage, cache, resolve)); + } + + beforeEach(() => { + sandbox.stub(Date, "now").returns(NOW); + }); + + it("shouldRun is false when the bot has no code", async function () { + const { cron } = build({}); + const r = await run(cron, {}, { id: "b1", trigger: NOW }, { _instances: { b1: {} } }); + assert.isFalse(r.value); + }); + + it("shouldRun is false when paused unless ignorePaused", async function () { + const { cron } = build({}); + const cache = { _instances: { b1: {} } }; + const base = { id: "b1", lambdaName: "l", trigger: NOW }; + assert.isFalse((await run(cron, {}, { ...base, paused: true }, cache)).value); + assert.isTrue((await run(cron, {}, { ...base, paused: true, ignorePaused: true }, cache)).value); + }); + + it("shouldRun is false after more than 10 errors", async function () { + const { cron } = build({}); + const r = await run(cron, {}, { id: "b1", lambdaName: "l", trigger: NOW, errorCount: 11 }, { _instances: { b1: {} } }); + assert.isFalse(r.value); + }); + + it("shouldRun treats cache._remoteCode as having code", async function () { + const { cron } = build({}); + const r = await run(cron, {}, { id: "b1", trigger: NOW }, { _instances: { b1: {} }, _remoteCode: true }); + assert.isTrue(r.value); + }); + + it("shouldRun is false for step-function execution types", async function () { + const { cron } = build({}); + const r = await run(cron, {}, { id: "b1", lambdaName: "l", executionType: "stepfunction", trigger: NOW }, { _instances: { b1: {} } }); + assert.deepEqual(r, { msg: "Step functions not supported yet", value: false }); + }); + + it("shouldRun is true when triggered after the last invoke", async function () { + const { cron } = build({}); + const r = await run(cron, {}, { id: "b1", lambdaName: "l", trigger: NOW }, { _instances: { b1: { "0": { completedTime: NOW - 10, invokeTime: NOW - 100, token: NOW - 50 } } } }); + assert.isTrue(r.value); + assert.isFalse(r.timedout); + }); + + it("shouldRun is false while an instance is still running within its timeout", async function () { + const { cron } = build({}); + const r = await run(cron, {}, { id: "b1", lambdaName: "l", trigger: NOW }, { _instances: { b1: { "0": { invokeTime: NOW - 1000 } } } }); + assert.isFalse(r.value); + }); + + it("shouldRun is true for a running instance that has timed out", async function () { + const { cron } = build({}); + const r = await run(cron, {}, { id: "b1", lambdaName: "l", trigger: NOW, timeout: 1000 }, { _instances: { b1: { "0": { invokeTime: NOW - 5000, token: 1 } } } }); + assert.isTrue(r.value); + assert.isTrue(r.timedout); + }); + + it("shouldRun is false when the token matches and nothing more is pending", async function () { + const { cron } = build({}); + const r = await run(cron, {}, { id: "b1", lambdaName: "l", trigger: NOW }, { _instances: { b1: { "0": { completedTime: NOW - 10, invokeTime: NOW - 100, token: NOW } } } }); + assert.isFalse(r.value); + }); + + it("shouldRun is true with a matching token when more work is pending", async function () { + const { cron } = build({}); + const r = await run(cron, {}, { + id: "b1", lambdaName: "l", trigger: NOW, triggers: ["q1"], requested_kinesis: { q1: "z/2" }, checkpoints: { read: { q1: { checkpoint: "z/1" } } } + }, { _instances: { b1: { "0": { completedTime: NOW - 10, invokeTime: NOW - 100, token: NOW } } } }); + assert.isTrue(r.value); + }); + + it("shouldRun honors the delay after the last invoke", async function () { + const { cron } = build({}); + const image = { id: "b1", lambdaName: "l", trigger: NOW, delay: "PT1M" }; + const cache = { _instances: { b1: { "0": { completedTime: NOW - 10, invokeTime: NOW - 1000, token: 1 } } } }; + assert.isFalse((await run(cron, {}, image, cache)).value); + assert.isTrue((await run(cron, {}, { ...image, trigger: NOW + 120000 }, cache)).value); + }); + + it("shouldRun evaluates each named trigger and lets only one sync instance run", async function () { + const { cron } = build({}); + const image = { + id: "b1", lambdaName: "l", trigger: 0, namedTrigger: { a: NOW, b: NOW }, + lambda: { settings: [{}] } + }; + const results = await run(cron, {}, image, { _instances: { b1: { a: { completedTime: 1, invokeTime: 1 }, b: { completedTime: 1, invokeTime: 2 } } } }); + assert.isArray(results); + assert.deepEqual(results.map((r) => r.iid).sort(), ["0", "a", "b"]); + assert.equal(results.filter((r) => r.value).length, 1); + }); + }); + + describe("start/end/run", function () { + function lockableRow(extra = {}) { + return { + id: "b1", + name: "Bot One", + lambdaName: "lam", + trigger: 1, + instances: {}, + checkpoints: { read: {}, write: {} }, + lambda: { settings: [{}] }, + ...extra + }; + } + + it("start without a callback marks the run forceComplete and stamps ts", async function () { + const { cron, configure } = build({}); + sandbox.stub(Date, "now").returns(4242); + const event: any = { id: "b1" }; + await cron.start(event); + assert.equal(event.__cron.ts, 4242); + assert.isTrue(event.__cron.forceComplete); + assert.equal(configure.registry.__cron, event.__cron); + }); + + it("start with a callback loads the bot, writes the instance token and then checks the lock", async function () { + const get = sandbox.stub().callsFake((p, cb) => cb(null, { Item: lockableRow() })); + const update = sandbox.stub().callsFake((p, cb) => cb(null, {})); + const { cron, configure } = build({ get, update }); + const event: any = { id: "b1" }; + await new Promise((resolve, reject) => { cron.start(event, (err) => err ? reject(err) : resolve()); }); + const command = update.getCall(0).args[0]; + assert.deepEqual(command.Key, { id: "b1" }); + assert.equal(command.UpdateExpression, "set #instances.#index = :value, #invokeTime = :invokeTime remove #checkpoint, #ignorePaused"); + assert.equal(command.ExpressionAttributeNames["#index"], "0"); + assert.equal(command.ExpressionAttributeValues[":value"].status, null); + assert.equal(update.callCount, 2); + assert.match(update.getCall(1).args[0].UpdateExpression, /^set #instances\.#index\.#startTime = :startTime/); + assert.equal(configure.registry.__cron.id, "b1"); + assert.equal(event.__cron.id, "b1"); + }); + + it("start creates the cron row when the bot has no checkpoints", async function () { + const get = sandbox.stub().callsFake((p, cb) => cb(null, {})); + const put = sandbox.stub().callsFake((p, cb) => cb(null, {})); + const update = sandbox.stub().callsFake((p, cb) => cb(null, {})); + const { cron } = build({ get, put, update }); + await new Promise((resolve, reject) => { cron.start({ id: "b9" }, (err) => err ? reject(err) : resolve()); }); + assert.equal(update.callCount, 1); + const item = put.getCall(0).args[0].Item; + assert.equal(item.id, "b9"); + assert.equal(item.name, "b9"); + assert.deepEqual(item.checkpoints, { read: {}, write: {} }); + assert.deepEqual(Object.keys(item.instances), ["0"]); + }); + + it("start reports a get error", async function () { + const boom = new Error("get failed"); + const get = sandbox.stub().callsFake((p, cb) => cb(boom)); + const { cron } = build({ get }); + let err; + try { + await cron.start({ id: "b1" }, () => { }); + } catch (e) { err = e; } + assert.equal(err, boom); + }); + + it("start with lock acquires the lock and rejects when already running", async function () { + const get = sandbox.stub().callsFake((p, cb) => cb(null, { Item: lockableRow() })); + const update = sandbox.stub().callsFake((p, cb) => cb(null, {})); + const { cron } = build({ get, update }); + const checkLock = sandbox.stub(cron, "checkLock"); + checkLock.onFirstCall().callsFake((c, runid, remaining, cb) => cb(null, {})); + checkLock.onSecondCall().callsFake((c, runid, remaining, cb) => cb(new Error("locked"))); + + const event: any = { id: "b1" }; + await cron.start(event, { lock: true, timeout: 1234, requestId: "req-1" }, () => { }); + assert.equal(checkLock.getCall(0).args[2], 1234); + assert.equal(checkLock.getCall(0).args[1], "req-1"); + assert.isFalse(event.__cron.forceComplete); + + let err; + try { + await cron.start({ id: "b1" }, { lock: true, timeout: 1234 }, () => { }); + } catch (e) { err = e; } + assert.equal(err, "already running"); + }); + + it("end with nothing registered calls back and resolves", async function () { + const { cron } = build({}); + const cb = sandbox.stub(); + await cron.end(cb); + assert.equal(cb.callCount, 1); + }); + + it("end reports complete with forceComplete from the registry", async function () { + const { cron, configure } = build({}); + configure.registry.__cron = { id: "b1", __requestId: "r1", forceComplete: true }; + const reportComplete = sandbox.stub(cron, "reportComplete").callsFake((c, runid, status, log, opts, cb: any) => cb()); + await cron.end(); + assert.deepEqual(reportComplete.getCall(0).args.slice(1, 5), ["r1", "complete", "", { forceComplete: true }]); + }); + + it("end with a non-complete status reports an error with that status as the log", async function () { + const { cron, configure } = build({}); + configure.registry.__cron = { id: "b1", __requestId: "r1", forceComplete: false }; + const reportComplete = sandbox.stub(cron, "reportComplete").callsFake((c, runid, status, log, opts, cb: any) => cb()); + await cron.end("it broke", {}); + const args = reportComplete.getCall(0).args; + assert.equal(args[2], "error"); + assert.equal(args[3], "it broke"); + assert.deepEqual(args[4], { forceComplete: false }); + }); + + it("end accepts status then callback", async function () { + const { cron, configure } = build({}); + configure.registry.__cron = { id: "b1" }; + sandbox.stub(cron, "reportComplete").callsFake((c, runid, status, log, opts, cb: any) => cb()); + const cb = sandbox.stub(); + await cron.end("boom", cb); + assert.equal(cb.callCount, 1); + }); + + it("start without a callback returns a promise whose then returns undefined and never calls the callback", async function () { + const get = sandbox.stub().callsFake((p, cb) => cb(null, { Item: lockableRow() })); + const update = sandbox.stub().callsFake((p, cb) => cb(null, {})); + const { cron } = build({ get, update }); + const result = cron.start({ id: "b1" }); + assert.instanceOf(result, Promise); + const callback = sandbox.stub(); + assert.isUndefined(result.then(callback)); + await new Promise((r) => setTimeout(r, 20)); + assert.equal(callback.callCount, 0); + }); + + it("run throws a TypeError when start has no callback because its then returns undefined", function () { + const get = sandbox.stub().callsFake((p, cb) => cb(null, { Item: lockableRow() })); + const update = sandbox.stub().callsFake((p, cb) => cb(null, {})); + const { cron } = build({ get, update }); + const handler = sandbox.stub(); + const callback = sandbox.stub(); + assert.throws(() => cron.run({ id: "b1" }, handler, callback), TypeError); + assert.equal(handler.callCount, 0); + assert.equal(callback.callCount, 0); + }); + + it("run calls start, the handler and end then the callback", async function () { + const { cron } = build({}); + const calls: string[] = []; + sandbox.stub(cron, "start").callsFake(() => { calls.push("start"); return Promise.resolve(); }); + sandbox.stub(cron, "end").callsFake((err) => { calls.push(`end:${err}`); return Promise.resolve(); }); + await cron.run({ id: "b1" }, () => { calls.push("handler"); }, () => { calls.push("callback"); }); + assert.deepEqual(calls, ["start", "handler", "end:undefined", "callback"]); + }); + + it("run passes a handler failure to end", async function () { + const { cron } = build({}); + const end = sandbox.stub(cron, "end").resolves(); + sandbox.stub(cron, "start").resolves(); + const boom = new Error("handler failed"); + await cron.run({ id: "b1" }, () => { throw boom; }, () => { }); + assert.equal(end.getCall(0).args[0], boom); + }); + }); +}); diff --git a/test/lib.dynamodb.utest.ts b/test/lib.dynamodb.utest.ts new file mode 100644 index 0000000..d5ebe47 --- /dev/null +++ b/test/lib.dynamodb.utest.ts @@ -0,0 +1,523 @@ +import sinon from "sinon"; +import { expect, assert } from "chai"; +import { DynamoDBDocument } from "@aws-sdk/lib-dynamodb"; +import dynamodbLib from "../lib/dynamodb"; + +describe("lib/dynamodb.js", function () { + const sandbox = sinon.createSandbox(); + + function build(doc: Record) { + sandbox.stub(DynamoDBDocument, "from").returns(doc as unknown as DynamoDBDocument); + return (dynamodbLib as any)({ + onUpdate: () => { }, + resources: { LeoSettings: "MockLeoSettings" }, + aws: { region: "us-west-2" } + }); + } + + beforeEach(() => { + delete process["__config"]; + }); + + afterEach(() => { + sandbox.restore(); + }); + + describe("get/put/merge/update/scan", function () { + it("get reads with ConsistentRead and returns Item", function (done) { + const get = sandbox.stub().callsFake((p, cb) => cb(null, { Item: { id: "a", v: 1 } })); + const ddb = build({ get }); + ddb.get("T", "a", (err, item) => { + assert.isNull(err); + assert.deepEqual(item, { id: "a", v: 1 }); + assert.deepEqual(get.getCall(0).args[0], { + TableName: "T", + Key: { id: "a" }, + ConsistentRead: true, + ReturnConsumedCapacity: "TOTAL" + }); + done(); + }); + }); + + it("get uses opts.id as the key name", function (done) { + const get = sandbox.stub().callsFake((p, cb) => cb(null, {})); + const ddb = build({ get }); + ddb.get("T", "a", { id: "pk" }, (err, item) => { + assert.isUndefined(item); + assert.deepEqual(get.getCall(0).args[0].Key, { pk: "a" }); + done(); + }); + }); + + it("get passes the error to the callback", function (done) { + const boom = new Error("boom"); + const get = sandbox.stub().callsFake((p, cb) => cb(boom)); + const ddb = build({ get }); + ddb.get("T", "a", (err) => { + assert.equal(err, boom); + done(); + }); + }); + + it("put writes the item with the id merged in and returns Success", function (done) { + const put = sandbox.stub().callsFake((p, cb) => cb(null, {})); + const ddb = build({ put }); + ddb.put("T", "a", { x: 1 }, (err, result) => { + assert.isNull(err); + assert.equal(result, "Success"); + assert.deepEqual(put.getCall(0).args[0], { + TableName: "T", + Key: { id: "a" }, + Item: { x: 1, id: "a" }, + ReturnConsumedCapacity: "TOTAL" + }); + done(); + }); + }); + + it("put uses opts.id for the key name", function (done) { + const put = sandbox.stub().callsFake((p, cb) => cb(null, {})); + const ddb = build({ put }); + ddb.put("T", "a", { x: 1 }, { id: "pk" }, () => { + assert.deepEqual(put.getCall(0).args[0].Key, { pk: "a" }); + assert.deepEqual(put.getCall(0).args[0].Item, { x: 1, pk: "a" }); + done(); + }); + }); + + it("put passes the error to the callback", function (done) { + const boom = new Error("put failed"); + const put = sandbox.stub().callsFake((p, cb) => cb(boom)); + const ddb = build({ put }); + ddb.put("T", "a", {}, (err, result) => { + assert.equal(err, boom); + assert.isUndefined(result); + done(); + }); + }); + + it("merge deep-merges onto the existing item and puts it back", function (done) { + const get = sandbox.stub().callsFake((p, cb) => cb(null, { Item: { id: "a", nested: { a: 1, b: 2 } } })); + const put = sandbox.stub().callsFake((p, cb) => cb(null, {})); + const ddb = build({ get, put }); + ddb.merge("T", "a", { nested: { b: 3, c: 4 } }, (err, result) => { + assert.isNull(err); + assert.equal(result, "Success"); + assert.deepEqual(put.getCall(0).args[0].Item, { id: "a", nested: { a: 1, b: 3, c: 4 } }); + done(); + }); + }); + + it("merge returns the get error without putting", function (done) { + const boom = new Error("get failed"); + const get = sandbox.stub().callsFake((p, cb) => cb(boom)); + const put = sandbox.stub(); + const ddb = build({ get, put }); + ddb.merge("T", "a", { x: 1 }, {}, (err) => { + assert.equal(err, boom); + expect(put.called).to.equal(false); + done(); + }); + }); + + it("update builds a set expression and skips undefined values", function (done) { + const update = sandbox.stub().callsFake((p, cb) => cb(null, {})); + const ddb = build({ update }); + ddb.update("T", { id: "a" }, { one: 1, two: undefined, "three-x": "3" }, () => { + assert.deepEqual(update.getCall(0).args[0], { + TableName: "T", + Key: { id: "a" }, + UpdateExpression: "set #one = :one, #three_x = :three_x", + ExpressionAttributeNames: { "#one": "one", "#three_x": "three-x" }, + ExpressionAttributeValues: { ":one": 1, ":three_x": "3" }, + ReturnConsumedCapacity: "TOTAL" + }); + done(); + }); + }); + + it("update uses if_not_exists for fields marked once", function (done) { + const update = sandbox.stub().callsFake((p, cb) => cb(null, {})); + const ddb = build({ update }); + ddb.update("T", { id: "a" }, { one: 1, two: 2 }, { fields: { one: { once: true } } }, () => { + assert.equal(update.getCall(0).args[0].UpdateExpression, "set #one = if_not_exists(#one, :one), #two = :two"); + done(); + }); + }); + + it("update passes ReturnValues through", function (done) { + const update = sandbox.stub().callsFake((p, cb) => cb(null, {})); + const ddb = build({ update }); + ddb.update("T", { id: "a" }, { one: 1 }, { ReturnValues: "ALL_NEW" }, () => { + assert.equal(update.getCall(0).args[0].ReturnValues, "ALL_NEW"); + done(); + }); + }); + + it("update with no settable fields sends undefined expression parts", function (done) { + const update = sandbox.stub().callsFake((p, cb) => cb(null, {})); + const ddb = build({ update }); + ddb.update("T", { id: "a" }, { one: undefined }, () => { + const params = update.getCall(0).args[0]; + assert.isUndefined(params.UpdateExpression); + assert.isUndefined(params.ExpressionAttributeNames); + assert.isUndefined(params.ExpressionAttributeValues); + done(); + }); + }); + + it("updateMulti runs one update per item and collects results in order", function (done) { + const update = sandbox.stub().callsFake((p, cb) => cb(null, { table: p.TableName })); + const ddb = build({ update }); + ddb.updateMulti([ + { table: "T1", key: { id: "a" }, set: { x: 1 } }, + { table: "T2", key: { id: "b" }, set: { y: 2 } } + ], (err, results) => { + assert.isNull(err); + assert.deepEqual(results, [{ table: "T1" }, { table: "T2" }]); + assert.equal(update.callCount, 2); + assert.deepEqual(update.getCall(1).args[0].Key, { id: "b" }); + done(); + }); + }); + + it("updateMulti reports the first error", function (done) { + const boom = new Error("nope"); + const update = sandbox.stub().callsFake((p, cb) => cb(boom)); + const ddb = build({ update }); + ddb.updateMulti([{ table: "T1", key: { id: "a" }, set: { x: 1 } }], { limit: 1 }, (err) => { + assert.equal(err, boom); + done(); + }); + }); + + it("scan returns Items", function (done) { + const scan = sandbox.stub().callsFake((p, cb) => cb(null, { Items: [{ id: 1 }, { id: 2 }] })); + const ddb = build({ scan }); + ddb.scan("T", null, (err, items) => { + assert.isNull(err); + assert.deepEqual(items, [{ id: 1 }, { id: 2 }]); + assert.deepEqual(scan.getCall(0).args[0], { TableName: "T", ReturnConsumedCapacity: "TOTAL" }); + done(); + }); + }); + + it("scan passes the error to the callback", function (done) { + const boom = new Error("scan failed"); + const scan = sandbox.stub().callsFake((p, cb) => cb(boom)); + const ddb = build({ scan }); + ddb.scan("T", null, (err) => { + assert.equal(err, boom); + done(); + }); + }); + }); + + describe("getSetting/saveSetting", function () { + it("saveSetting puts a value row into LeoSettings", function (done) { + const put = sandbox.stub().callsFake((p, cb) => cb(null, {})); + const ddb = build({ put }); + ddb.saveSetting("s1", { a: 1 }, (err, result) => { + assert.isNull(err); + assert.equal(result, "Success"); + assert.deepEqual(put.getCall(0).args[0].Item, { id: "s1", value: { a: 1 } }); + assert.equal(put.getCall(0).args[0].TableName, "MockLeoSettings"); + done(); + }); + }); + + it("getSetting reads the row from LeoSettings", function (done) { + const get = sandbox.stub().callsFake((p, cb) => cb(null, { Item: { id: "s1", value: 5 } })); + const ddb = build({ get }); + ddb.getSetting("s1", (err, item) => { + assert.deepEqual(item, { id: "s1", value: 5 }); + assert.equal(get.getCall(0).args[0].TableName, "MockLeoSettings"); + done(); + }); + }); + + it("getSettingPromise resolves the stored row", async function () { + const get = sandbox.stub().callsFake((p, cb) => cb(null, { Item: { id: "s1", value: 5 } })); + const ddb = build({ get }); + assert.deepEqual(await ddb.getSettingPromise("s1"), { id: "s1", value: 5 }); + }); + + it("setSettingPromise resolves Success", async function () { + const put = sandbox.stub().callsFake((p, cb) => cb(null, {})); + const ddb = build({ put }); + assert.equal(await ddb.setSettingPromise("s1", 9), "Success"); + assert.deepEqual(put.getCall(0).args[0].Item, { id: "s1", value: 9 }); + }); + + it("getSettingPromise rejects with the get error", async function () { + const boom = new Error("get failed"); + const get = sandbox.stub().callsFake((p, cb) => cb(boom)); + const ddb = build({ get }); + let err; + try { await ddb.getSettingPromise("s1"); } catch (e) { err = e; } + assert.equal(err, boom); + }); + }); + + describe("query", function () { + it("query returns data with _stats when there is no LastEvaluatedKey", async function () { + const query = sandbox.stub().callsFake((p, cb) => cb(null, { Items: [{ id: 1 }], Count: 1 })); + const ddb = build({ query }); + const data = await ddb.query({ TableName: "T" }); + assert.deepEqual(data.Items, [{ id: 1 }]); + assert.deepEqual(data._stats, { mb: 1, count: 1 }); + }); + + it("query pages with ExclusiveStartKey and merges Items/Count/ScannedCount/capacity", async function () { + const query = sandbox.stub(); + query.onFirstCall().callsFake((p, cb) => cb(null, { + Items: [{ id: 1 }], Count: 1, ScannedCount: 2, LastEvaluatedKey: { id: 1 }, ConsumedCapacity: { CapacityUnits: 1 } + })); + query.onSecondCall().callsFake((p, cb) => cb(null, { + Items: [{ id: 2 }], Count: 1, ScannedCount: 3, ConsumedCapacity: { CapacityUnits: 4 } + })); + const ddb = build({ query }); + const data = await ddb.query({ TableName: "T" }); + assert.deepEqual(data.Items, [{ id: 1 }, { id: 2 }]); + assert.equal(data.Count, 2); + assert.equal(data.ScannedCount, 5); + assert.equal(data.ConsumedCapacity.CapacityUnits, 5); + assert.isUndefined(data.LastEvaluatedKey); + assert.deepEqual(data._stats, { mb: 2, count: 2 }); + assert.deepEqual(query.getCall(1).args[0].ExclusiveStartKey, { id: 1 }); + }); + + it("query stops paging once the mb limit is reached", async function () { + const query = sandbox.stub().callsFake((p, cb) => cb(null, { Items: [{ id: 1 }], Count: 1, LastEvaluatedKey: { id: 1 } })); + const ddb = build({ query }); + const data = await ddb.query({ TableName: "T" }, { mb: 1 }); + assert.equal(query.callCount, 1); + assert.deepEqual(data.LastEvaluatedKey, { id: 1 }); + }); + + it("query stops paging once the count limit is reached", async function () { + const query = sandbox.stub().callsFake((p, cb) => cb(null, { Items: [{ id: 1 }], Count: 1, LastEvaluatedKey: { id: 1 } })); + const ddb = build({ query }); + await ddb.query({ TableName: "T" }, { count: 1 }); + assert.equal(query.callCount, 1); + }); + + it("query stops paging when progress calls back false", async function () { + const query = sandbox.stub().callsFake((p, cb) => cb(null, { Items: [{ id: 1 }], Count: 1, LastEvaluatedKey: { id: 1 } })); + const ddb = build({ query }); + await ddb.query({ TableName: "T" }, { progress: (d, s, cb) => cb(false) }); + assert.equal(query.callCount, 1); + }); + + it("query with method scan calls docClient.scan", async function () { + const scan = sandbox.stub().callsFake((p, cb) => cb(null, { Items: [], Count: 0 })); + const query = sandbox.stub(); + const ddb = build({ query, scan }); + await ddb.query({ TableName: "T" }, { method: "scan" }); + assert.equal(scan.callCount, 1); + assert.equal(query.callCount, 0); + }); + + it("query rejects with the client error", async function () { + const boom = new Error("query failed"); + const query = sandbox.stub().callsFake((p, cb) => cb(boom)); + const ddb = build({ query }); + let err; + try { await ddb.query({ TableName: "T" }); } catch (e) { err = e; } + assert.equal(err, boom); + }); + + it("query rejects when a subsequent page fails", async function () { + const boom = new Error("page 2 failed"); + const query = sandbox.stub(); + query.onFirstCall().callsFake((p, cb) => cb(null, { Items: [{ id: 1 }], Count: 1, LastEvaluatedKey: { id: 1 } })); + query.onSecondCall().callsFake((p, cb) => cb(boom)); + const ddb = build({ query }); + let err; + try { await ddb.query({ TableName: "T" }); } catch (e) { err = e; } + assert.equal(err, boom); + }); + }); + + describe("batchGetTable/batchGetHashkey", function () { + it("batchGetTable dedupes keys and sends a batchGet per chunk", function (done) { + const batchGet = sandbox.stub().callsFake((p, cb) => cb(null, { + Responses: { T: p.RequestItems.T.Keys.map((k) => ({ ...k, found: true })) } + })); + const ddb = build({ batchGet }); + ddb.batchGetTable("T", [{ id: "a" }, { id: "b" }, { id: "a" }], (err, results) => { + assert.isNull(err); + assert.deepEqual(results, [{ id: "a", found: true }, { id: "b", found: true }]); + assert.equal(batchGet.callCount, 1); + assert.deepEqual(batchGet.getCall(0).args[0], { + RequestItems: { T: { Keys: [{ id: "a" }, { id: "b" }] } }, + ReturnConsumedCapacity: "TOTAL" + }); + done(); + }); + }); + + it("batchGetTable splits keys by chunk_size", function (done) { + const batchGet = sandbox.stub().callsFake((p, cb) => cb(null, { + Responses: { T: p.RequestItems.T.Keys.map((k) => ({ ...k })) } + })); + const ddb = build({ batchGet }); + ddb.batchGetTable("T", [{ id: "a" }, { id: "b" }, { id: "c" }], { chunk_size: 2, concurrency: 1 }, (err, results) => { + assert.isNull(err); + assert.equal(results.length, 3); + assert.deepEqual(batchGet.getCall(0).args[0].RequestItems.T.Keys, [{ id: "a" }, { id: "b" }]); + assert.deepEqual(batchGet.getCall(1).args[0].RequestItems.T.Keys, [{ id: "c" }]); + done(); + }); + }); + + it("batchGetTable with no keys calls back with an empty result", function (done) { + const batchGet = sandbox.stub(); + const ddb = build({ batchGet }); + ddb.batchGetTable("T", [], (err, results) => { + assert.isNull(err); + assert.deepEqual(results, []); + assert.equal(batchGet.callCount, 0); + done(); + }); + }); + + it("batchGetHashkey maps rows by their hash key", function (done) { + const batchGet = sandbox.stub().callsFake((p, cb) => cb(null, { + Responses: { T: [{ pk: "x", v: 1 }, { pk: "y", v: 2 }] } + })); + const ddb = build({ batchGet }); + ddb.batchGetHashkey("T", "pk", ["x", "y"], (err, byKey) => { + assert.isNull(err); + assert.deepEqual(byKey, { x: { pk: "x", v: 1 }, y: { pk: "y", v: 2 } }); + assert.deepEqual(batchGet.getCall(0).args[0].RequestItems.T.Keys, [{ pk: "x" }, { pk: "y" }]); + done(); + }); + }); + }); + + describe("batchTableWrite/createTableWriteStream", function () { + it("batchTableWrite sends the records as RequestItems and returns empty on success", function (done) { + const batchWrite = sandbox.stub().callsFake((p, cb) => cb(null, { UnprocessedItems: {} })); + const ddb = build({ batchWrite }); + const records = [{ PutRequest: { Item: { id: 1 } } }]; + ddb.batchTableWrite("T", records, (err, unprocessed) => { + assert.isNull(err); + assert.deepEqual(unprocessed, []); + assert.deepEqual(batchWrite.getCall(0).args[0], { + RequestItems: { T: records }, + ReturnConsumedCapacity: "TOTAL" + }); + done(); + }); + }); + + it("batchTableWrite returns the records with the error on failure", function (done) { + const boom = new Error("write failed"); + const batchWrite = sandbox.stub().callsFake((p, cb) => cb(boom)); + const ddb = build({ batchWrite }); + const records = [{ PutRequest: { Item: { id: 1 } } }]; + ddb.batchTableWrite("T", records, (err, unprocessed) => { + assert.equal(err, boom); + assert.equal(unprocessed, records); + done(); + }); + }); + + it("batchTableWrite reports unprocessed records", function (done) { + const left = [{ PutRequest: { Item: { id: 2 } } }]; + const batchWrite = sandbox.stub().callsFake((p, cb) => cb(null, { UnprocessedItems: { T: left } })); + const ddb = build({ batchWrite }); + ddb.batchTableWrite("T", [], (err, unprocessed) => { + assert.equal(err, "unprocessed records"); + assert.equal(unprocessed, left); + done(); + }); + }); + + it("createTableWriteStream batches puts into PutRequests and calls end back", function (done) { + const batchWrite = sandbox.stub().callsFake((p, cb) => cb(null, { UnprocessedItems: {} })); + const ddb = build({ batchWrite }); + const stream = ddb.createTableWriteStream("T", { chunk_size: 2, concurrency: 1, concurrency_delay: 1 }); + stream.put({ id: 1 }); + stream.put({ id: 2 }); + stream.put({ id: 3 }); + stream.end((err) => { + assert.isNull(err); + const sent = batchWrite.getCalls().map((c) => c.args[0].RequestItems.T.map((r) => r.PutRequest.Item.id)); + assert.deepEqual(sent, [[1, 2], [3]]); + done(); + }); + }); + + it("createTableWriteStream keeps only the newest record per key when keys are given", function (done) { + const batchWrite = sandbox.stub().callsFake((p, cb) => cb(null, { UnprocessedItems: {} })); + const ddb = build({ batchWrite }); + const stream = ddb.createTableWriteStream("T", { keys: ["id", "sort"], chunk_size: 10, concurrency: 1, concurrency_delay: 1 }); + stream.put({ id: "a", sort: 1, v: "old" }); + stream.put({ id: "b", sort: 1, v: "other" }); + stream.put({ id: "a", sort: 1, v: "new" }); + stream.end((err) => { + assert.isNull(err); + assert.equal(batchWrite.callCount, 1); + const items = batchWrite.getCall(0).args[0].RequestItems.T.map((r) => r.PutRequest.Item); + assert.deepEqual(items, [{ id: "b", sort: 1, v: "other" }, { id: "a", sort: 1, v: "new" }]); + done(); + }); + }); + + it("createTableWriteStream with nothing queued ends without writing", function (done) { + const batchWrite = sandbox.stub(); + const ddb = build({ batchWrite }); + const stream = ddb.createTableWriteStream("T"); + stream.end((err, remaining) => { + assert.isNull(err); + assert.deepEqual(remaining, []); + assert.equal(batchWrite.callCount, 0); + done(); + }); + }); + + it("createTableWriteStream retries unprocessed records and gives up with an error", function (done) { + this.timeout(5000); + const batchWrite = sandbox.stub().callsFake((p, cb) => cb(new Error("always fails"))); + const ddb = build({ batchWrite }); + const stream = ddb.createTableWriteStream("T", { chunk_size: 5, concurrency: 1, concurrency_delay: 1, retry: 1, retryDelay: 1 }); + stream.put({ id: 1 }); + stream.end((err, count) => { + assert.equal(err, "Cannot process all the entries"); + assert.equal(count, 1); + assert.isAtLeast(batchWrite.callCount, 2); + done(); + }); + }); + }); + + describe("config updates", function () { + it("the onUpdate listener throws a TypeError because the v3 document client has no service", function () { + const callbacks: any[] = []; + sandbox.stub(DynamoDBDocument, "from").returns({} as unknown as DynamoDBDocument); + (dynamodbLib as any)({ onUpdate: (cb) => callbacks.push(cb), aws: { region: "us-west-2" } }); + assert.equal(callbacks.length, 1); + assert.throws(() => callbacks[0]({ aws: { region: "eu-west-1" }, credentials: {} }), TypeError); + }); + }); + + describe("construction", function () { + it("maps legacy maxRetries/httpOptions into client config", function () { + const from = sandbox.stub(DynamoDBDocument, "from").returns({} as unknown as DynamoDBDocument); + (dynamodbLib as any)({ onUpdate: () => { }, aws: { region: "us-west-2" } }, { + maxRetries: 4, + httpOptions: { connectTimeout: 111, timeout: 222 } + }); + const client = from.getCall(0).args[0] as any; + return Promise.resolve(client.config.maxAttempts()).then((attempts) => { + assert.equal(attempts, 5); + assert.deepEqual(from.getCall(0).args[1], { + marshallOptions: { convertEmptyValues: true, removeUndefinedValues: true } + }); + }); + }); + }); +}); diff --git a/test/lib.event-trace.children.utest.ts b/test/lib.event-trace.children.utest.ts new file mode 100644 index 0000000..67d6aa3 --- /dev/null +++ b/test/lib.event-trace.children.utest.ts @@ -0,0 +1,323 @@ +import RStreamsSdk from "../index"; +import sinon from "sinon"; +import { assert } from "chai"; +import moment from "moment"; +import { trace as eventTrace } from "../lib/event-trace"; + +const mockSdkConfig = { + Region: "mock-Region", + LeoStream: "mock-LeoStream", + LeoCron: "mock-LeoCron", + LeoEvent: "mock-LeoEvent", + LeoS3: "mock-leos3", + LeoKinesisStream: "mock-LeoKinesisStream", + LeoFirehoseStream: "mock-LeoFirehoseStream", + LeoSettings: "mock-LeoSettings", +}; + +const STATS_TABLE = "MOCK_STATS_TABLE"; +const START_EID = "z/2022/03/14/22/38/1647297513618-0000035"; +const START_TS = 1647297513413; + +describe("lib/event-trace.ts children", function () { + let sandbox: sinon.SinonSandbox; + let sdk: any; + let query: sinon.SinonStub; + let fromLeo: sinon.SinonStub; + + function startEvent() { + return { id: "bot0", event: "queue:q0", eid: START_EID, timestamp: START_TS, event_source_timestamp: START_TS - 100, payload: {}, correlation_id: { source: "src", start: "z/1" } }; + } + + function stat(bucket: string, time: number, queue: string, checkpoint: string) { + return { bucket, time, current: { read: { [queue]: { checkpoint } } } }; + } + + function statsFor(botRefId: string, hourItems: any[], minuteItems: any[] = hourItems) { + query.callsFake((params: any, cb: any) => { + if (params.ExpressionAttributeValues[":id"] !== botRefId) { + return; + } + cb(null, { Items: params.Limit === 60 ? minuteItems : hourItems }); + }); + } + + beforeEach(() => { + sandbox = sinon.createSandbox(); + sdk = RStreamsSdk(mockSdkConfig); + query = sandbox.stub(sdk.aws.dynamodb.docClient, "query"); + fromLeo = sandbox.stub(sdk.streams, "fromLeo"); + }); + + afterEach(() => { + sandbox.restore(); + }); + + function queueReads(map: Record) { + fromLeo.callsFake((id, queue) => { + const events = map[String(queue)]; + return sdk.streams.eventstream.readArray(events || []); + }); + } + + it("children trace reads the starting event with start just before the eid and limit 1", async function () { + queueReads({ "queue:q0": [startEvent()] }); + statsFor("bot:bot1", []); + await eventTrace(sdk, STATS_TABLE, { queue: "q0", eid: START_EID, children: ["bot1", "queue1"] }); + assert.deepEqual(fromLeo.getCall(0).args.slice(0, 3), ["test", "queue:q0", { start: "z/2022/03/14/22/38/1647297513618-0000034", limit: 1 }]); + }); + + it("children trace queries hourly then minute stats for the bot and reads the queue window", async function () { + const childEvent = { + eid: "z/2022/03/14/22/40/1647297650000-0000001", event: "queue:queue1", id: "bot1", payload: { x: 1 }, + timestamp: 1647297650500, event_source_timestamp: 1647297650000, correlation_id: { start: START_EID, units: 1 } + }; + const statTime = 1647297600000; + queueReads({ "queue:q0": [startEvent()], "queue:queue1": [childEvent] }); + statsFor("bot:bot1", [stat("hour_2022-03-14 22", statTime, "queue:q0", START_EID)]); + + const result: any = await eventTrace(sdk, STATS_TABLE, { queue: "q0", eid: START_EID, children: ["bot1", "queue1"] }); + + assert.deepEqual(query.getCall(0).args[0], { + TableName: STATS_TABLE, + KeyConditionExpression: "#id = :id and #bucket >= :bucket", + ExpressionAttributeNames: { "#bucket": "bucket", "#id": "id" }, + Limit: 336, + ExpressionAttributeValues: { ":bucket": "hour_2022-03-14 22", ":id": "bot:bot1" }, + ReturnConsumedCapacity: "TOTAL" + }); + assert.deepEqual(query.getCall(1).args[0], { + TableName: STATS_TABLE, + KeyConditionExpression: "#id = :id and #bucket >= :bucket", + ExpressionAttributeNames: { "#bucket": "bucket", "#id": "id" }, + Limit: 60, + ExpressionAttributeValues: { ":bucket": "minute_2022-03-14 22:40", ":id": "bot:bot1" }, + ReturnConsumedCapacity: "TOTAL" + }); + const readArgs = fromLeo.getCall(1).args; + assert.equal(String(readArgs[1]), "queue:queue1"); + assert.deepEqual(readArgs[2], { + start: moment(statTime).format("[z/]YYYY/MM/DD/HH/mm/ss"), + maxOverride: moment(statTime).add(1, "m").format("[z/]YYYY/MM/DD/HH/mm/ss"), + fast_s3_read: true + }); + assert.deepEqual(result, { + "bot:bot1": { checkpoint: childEvent.eid, lag: 500 }, + "queue:queue1": { payload: { x: 1 }, lag: 500, checkpoint: childEvent.eid } + }); + }); + + it("children trace follows the correlation chain across two hops", async function () { + const hop1 = { + eid: "z/2022/03/14/22/40/1647297650000-0000001", event: "queue:queue1", id: "bot1", payload: { hop: 1 }, + timestamp: 1647297650500, event_source_timestamp: 1647297650000, correlation_id: { start: START_EID } + }; + const hop2 = { + eid: "z/2022/03/14/22/41/1647297710000-0000002", event: "queue:queue2", id: "bot2", payload: { hop: 2 }, + timestamp: 1647297710900, event_source_timestamp: 1647297650000, correlation_id: { start: hop1.eid } + }; + queueReads({ "queue:q0": [startEvent()], "queue:queue1": [hop1], "queue:queue2": [hop2] }); + query.callsFake((params: any, cb: any) => { + const id = params.ExpressionAttributeValues[":id"]; + if (id === "bot:bot1") { + return cb(null, { Items: [stat("hour_2022-03-14 22", 1647297600000, "queue:q0", START_EID)] }); + } + cb(null, { Items: [stat("hour_2022-03-14 22", 1647297660000, "queue:queue1", hop1.eid)] }); + }); + + const result: any = await eventTrace(sdk, STATS_TABLE, { queue: "q0", eid: START_EID, children: ["bot1", "queue1", "bot2", "queue2"] }); + + assert.deepEqual(result, { + "bot:bot1": { checkpoint: hop1.eid, lag: 500 }, + "queue:queue1": { payload: { hop: 1 }, lag: 500, checkpoint: hop1.eid }, + "bot:bot2": { checkpoint: hop2.eid, lag: 60900 }, + "queue:queue2": { payload: { hop: 2 }, lag: 60900, checkpoint: hop2.eid } + }); + assert.equal(query.getCall(2).args[0].ExpressionAttributeValues[":id"], "bot:bot2"); + }); + + it("children trace skips stats whose checkpoint is behind the correlation start", async function () { + const childEvent = { + eid: "z/2022/03/14/22/40/1647297650000-0000001", event: "queue:queue1", id: "bot1", payload: {}, + timestamp: 1647297650500, event_source_timestamp: 1647297650000, correlation_id: { start: START_EID } + }; + queueReads({ "queue:q0": [startEvent()], "queue:queue1": [childEvent] }); + statsFor("bot:bot1", [ + stat("hour_2022-03-14 22", 1647297000000, "queue:q0", "z/2022/03/14/22/00/1647295200000-0000000"), + stat("hour_2022-03-14 23", 1647297600000, "queue:q0", START_EID) + ]); + await eventTrace(sdk, STATS_TABLE, { queue: "q0", eid: START_EID, children: ["bot1", "queue1"] }); + assert.equal(query.getCall(1).args[0].ExpressionAttributeValues[":bucket"], "minute_2022-03-14 22:40"); + }); + + it("children trace matches an event whose correlation range covers the start", async function () { + const rangeEvent = { + eid: "z/2022/03/14/22/40/1647297650000-0000009", event: "queue:queue1", id: "bot1", payload: { range: true }, + timestamp: 1647297650500, event_source_timestamp: 1647297650000, + correlation_id: { start: "z/2022/03/14/22/38/1647297500000-0000001", end: "z/2022/03/14/22/38/1647297600000-0000099" } + }; + queueReads({ "queue:q0": [startEvent()], "queue:queue1": [rangeEvent] }); + statsFor("bot:bot1", [stat("hour_2022-03-14 22", 1647297600000, "queue:q0", START_EID)]); + const result: any = await eventTrace(sdk, STATS_TABLE, { queue: "q0", eid: START_EID, children: ["bot1", "queue1"] }); + assert.deepEqual(result["queue:queue1"].payload, { range: true }); + }); + + it("children trace keeps the first match when later events also match", async function () { + const first = { + eid: "z/2022/03/14/22/40/1647297650000-0000001", event: "queue:queue1", id: "bot1", payload: { n: 1 }, + timestamp: 1647297650500, event_source_timestamp: 1647297650000, correlation_id: { start: START_EID } + }; + const second = { ...first, eid: "z/2022/03/14/22/40/1647297651000-0000002", payload: { n: 2 } }; + queueReads({ "queue:q0": [startEvent()], "queue:queue1": [first, second] }); + statsFor("bot:bot1", [stat("hour_2022-03-14 22", 1647297600000, "queue:q0", START_EID)]); + const result: any = await eventTrace(sdk, STATS_TABLE, { queue: "q0", eid: START_EID, children: ["bot1", "queue1"] }); + assert.deepEqual(result["queue:queue1"].payload, { n: 1 }); + }); + + it("children trace reports empty entries when no stats match", async function () { + queueReads({ "queue:q0": [startEvent()] }); + statsFor("bot:bot1", [stat("hour_2022-03-14 22", 1647297600000, "queue:other", START_EID)]); + const result: any = await eventTrace(sdk, STATS_TABLE, { queue: "q0", eid: START_EID, children: ["bot1", "queue1"] }); + assert.deepEqual(result, { "bot:bot1": {}, "queue:queue1": {} }); + assert.equal(query.callCount, 1); + }); + + it("children trace rejects when the stats query fails", async function () { + queueReads({ "queue:q0": [startEvent()] }); + const boom = new Error("stats failed"); + query.callsFake((params: any, cb: any) => cb(boom)); + let err; + try { + await eventTrace(sdk, STATS_TABLE, { queue: "q0", eid: START_EID, children: ["bot1", "queue1"] }); + } catch (e) { err = e; } + assert.equal(err, boom); + }); + + it("children trace rejects when the minute stats query fails", async function () { + queueReads({ "queue:q0": [startEvent()] }); + const boom = new Error("minute stats failed"); + query.callsFake((params: any, cb: any) => { + if (params.Limit === 60) { + return cb(boom); + } + cb(null, { Items: [stat("hour_2022-03-14 22", 1647297600000, "queue:q0", START_EID)] }); + }); + let err; + try { + await eventTrace(sdk, STATS_TABLE, { queue: "q0", eid: START_EID, children: ["bot1", "queue1"] }); + } catch (e) { err = e; } + assert.equal(err, boom); + }); + + it("children trace queue ids with a 0 suffix step back to a space", async function () { + queueReads({ "queue:q0": [startEvent()] }); + statsFor("bot:bot1", []); + await eventTrace(sdk, STATS_TABLE, { queue: "q0", eid: "z/2022/03/14/22/38/1647297513618-0000030", children: ["bot1", "queue1"] }); + assert.equal(fromLeo.getCall(0).args[2].start, "z/2022/03/14/22/38/1647297513618-000003 "); + }); + + it("children trace uses the correlation timestamp when stats are older than the event", async function () { + const childEvent = { + eid: "z/2022/03/14/22/40/1647297650000-0000001", event: "queue:queue1", id: "bot1", payload: {}, + timestamp: 1647297650500, event_source_timestamp: 1647297650000, correlation_id: { start: START_EID } + }; + queueReads({ "queue:q0": [startEvent()], "queue:queue1": [childEvent] }); + statsFor("bot:bot1", [stat("hour_2022-03-14 22", 1647297000000, "queue:q0", START_EID)]); + await eventTrace(sdk, STATS_TABLE, { queue: "q0", eid: START_EID, children: ["bot1", "queue1"] }); + assert.equal(fromLeo.getCall(1).args[2].start, moment(START_TS).format("[z/]YYYY/MM/DD/HH/mm/ss")); + }); + + it("children trace ignores queue events that do not correlate to the start", async function () { + const other = { + eid: "z/2022/03/14/22/40/1647297640000-0000001", event: "queue:queue1", id: "bot1", payload: { other: true }, + timestamp: 1647297640500, event_source_timestamp: 1647297640000, correlation_id: { start: "z/2022/03/14/22/39/1-0000000" } + }; + const match = { ...other, eid: "z/2022/03/14/22/40/1647297650000-0000002", payload: { match: true }, correlation_id: { start: START_EID } }; + queueReads({ "queue:q0": [startEvent()], "queue:queue1": [other, match] }); + statsFor("bot:bot1", [stat("hour_2022-03-14 22", 1647297600000, "queue:q0", START_EID)]); + const result: any = await eventTrace(sdk, STATS_TABLE, { queue: "q0", eid: START_EID, children: ["bot1", "queue1"] }); + assert.deepEqual(result["queue:queue1"].payload, { match: true }); + }); + + describe("parents and children tree", function () { + const cronRows = [ + { + id: "botA", name: "Bot A", + checkpoints: { + read: { "queue:qSrc": { checkpoint: "z/2022/03/14/23/00/1", source_timestamp: START_TS + 1000 } }, + write: { "queue:qMid": {} } + } + }, + { + id: "botB", name: "Bot B", + checkpoints: { + read: { "queue:qMid": { checkpoint: "z/2022/03/14/23/00/1" } }, + write: { "queue:qSrc": {} } + } + }, + { + id: "botC", + checkpoints: { + read: { "queue:qSrc": { checkpoint: "z/2022/03/14/22/00/1", source_timestamp: START_TS + 2000 } }, + write: {} + } + }, + { id: "botNoCheckpoints" } + ]; + + function srcEvent(extra: any = {}) { + return { id: "botX", event: "qSrc", eid: START_EID, timestamp: START_TS, event_source_timestamp: START_TS - 100, payload: {}, ...extra }; + } + + beforeEach(() => { + sandbox.stub(sdk.aws.dynamodb, "query").callsFake(() => Promise.resolve({ Items: cronRows })); + }); + + it("trace stops at an event without a correlation and walks processed bots downstream", async function () { + queueReads({ "queue:qSrc": [srcEvent()] }); + const result: any = await eventTrace(sdk, STATS_TABLE, { queue: "qSrc", eid: START_EID }); + assert.deepEqual(result.parents, [{ id: "bot:botX", server_id: "bot:botX", type: "bot", label: "bot:botX" }]); + assert.equal(result.event.eid, START_EID); + assert.equal(result.event.type, "queue"); + assert.equal(result.event.lag, 100); + assert.deepEqual(Object.keys(result.children), ["bot:botA", "bot:botC"]); + + const botA = result.children["bot:botA"]; + assert.equal(botA.label, "Bot A"); + assert.isTrue(botA.has_processed); + assert.equal(botA.lag, 0); + const mid = botA.children["queue:qMid"]; + assert.deepEqual([mid.type, mid.lag, mid.event], ["queue", null, null]); + const botB = mid.children["bot:botB"]; + assert.isTrue(botB.has_processed); + assert.deepEqual(botB.children["queue:qSrc"].children, {}); + + const botC = result.children["bot:botC"]; + assert.equal(botC.label, "bot:botC"); + assert.isFalse(botC.has_processed); + assert.equal(botC.lag, 2100); + assert.deepEqual(botC.children, {}); + }); + + it("trace follows correlation_id to parent queues and stops on a repeated bot", async function () { + queueReads({ + "queue:qSrc": [srcEvent({ correlation_id: { source: "qUp", start: "z/2022/03/14/22/30/1-0000000" } })], + "queue:qUp": [{ id: "botX", event: "qUp", eid: "z/2022/03/14/22/30/1-0000000", timestamp: START_TS - 5000, event_source_timestamp: START_TS - 6000, payload: {} }] + }); + const result: any = await eventTrace(sdk, STATS_TABLE, { queue: "qSrc", eid: START_EID }); + assert.equal(fromLeo.callCount, 2); + assert.equal(String(fromLeo.getCall(1).args[1]), "queue:qUp"); + assert.deepEqual(fromLeo.getCall(1).args[2], { start: "z/2022/03/14/22/30/1-000000 ", limit: 1 }); + assert.equal(result.parents.length, 1); + }); + + it("trace rejects when reading the queue fails", async function () { + const { Readable } = require("stream"); + const boom = new Error("read failed"); + fromLeo.callsFake(() => new Readable({ objectMode: true, read() { this.destroy(boom); } })); + let err; + try { await eventTrace(sdk, STATS_TABLE, { queue: "qSrc", eid: START_EID }); } catch (e) { err = e; } + assert.equal(err, boom); + }); + }); +}); diff --git a/test/lib.rstreams-config-provider-chain.order.utest.ts b/test/lib.rstreams-config-provider-chain.order.utest.ts new file mode 100644 index 0000000..f36907b --- /dev/null +++ b/test/lib.rstreams-config-provider-chain.order.utest.ts @@ -0,0 +1,364 @@ +import sinon from "sinon"; +import { assert } from "chai"; +import fs from "fs"; +import os from "os"; +import path from "path"; +import ConfigProviderChain, { AWSSecretsConfiguration, EnvironmentConfiguration, FileTreeConfiguration, ProvidersInputType } from "../lib/rstreams-config-provider-chain"; +import awsSdkSync from "../lib/aws-sdk-sync"; + +const ENV_NAMES = ["RSTREAMS_CONFIG", "leosdk", "leo-sdk", "LEOSDK", "LEO-SDK"]; +const SECRET_ENV = ["LEO_CONFIG_SECRET", "RSTREAMS_CONFIG_SECRET", "AWS_REGION"]; +const RESOURCE_KEYS = ["Region", "LeoStream", "LeoCron", "LeoEvent", "LeoS3", "LeoKinesisStream", "LeoFirehoseStream", "LeoSettings"]; + +const CONFIG_ENV_PREFIXES = ["leosdk_", "leo-sdk_", "LEOSDK_", "LEO-SDK_", "RSTREAMS_CONFIG_"]; + +function isConfigEnvKey(k: string) { + return CONFIG_ENV_PREFIXES.some((prefix) => k.startsWith(prefix)); +} + +function cfg(tag: string) { + const out: any = {}; + RESOURCE_KEYS.forEach((k) => { out[k] = `${tag}-${k}`; }); + return out; +} + +describe("lib/rstreams-config-provider-chain.ts resolution order", function () { + const sandbox = sinon.createSandbox(); + const leoConfig = require("leo-config"); + let savedEnv: Record; + let tmpRoot: string; + + function resolved() { + return new ConfigProviderChain().resolveSync().Region; + } + + beforeEach(() => { + savedEnv = {}; + [...ENV_NAMES, ...SECRET_ENV].forEach((k) => { savedEnv[k] = process.env[k]; delete process.env[k]; }); + Object.keys(process.env).filter(isConfigEnvKey).forEach((k) => { savedEnv[k] = process.env[k]; delete process.env[k]; }); + AWSSecretsConfiguration.clearCache(); + tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), "rstreams-cfg-")); + sandbox.stub(process, "cwd").returns(path.join(tmpRoot, "a", "b")); + fs.mkdirSync(path.join(tmpRoot, "a", "b"), { recursive: true }); + }); + + afterEach(() => { + sandbox.restore(); + Object.keys(process.env).filter(isConfigEnvKey).forEach((k) => { delete process.env[k]; }); + Object.keys(savedEnv).forEach((k) => { + if (savedEnv[k] === undefined) { delete process.env[k]; } else { process.env[k] = savedEnv[k]; } + }); + ["leosdk", "leo_sdk", "leo-sdk", "rstreamssdk", "rstreams_sdk", "rstreams-sdk"].forEach((k) => { delete leoConfig[k]; }); + ["leosdk", "leo-sdk", "rstreams_config"].forEach((k) => { delete process[k]; delete global[k]; }); + fs.rmSync(tmpRoot, { recursive: true, force: true }); + AWSSecretsConfiguration.clearCache(); + }); + + describe("environment variable names", function () { + ENV_NAMES.forEach((name) => { + it(`${name} holds the whole config as JSON`, function () { + process.env[name] = JSON.stringify(cfg(name)); + assert.equal(resolved(), `${name}-Region`); + }); + }); + + it("split variables accept upper and lower case key names", function () { + RESOURCE_KEYS.forEach((k, i) => { + const value = `split-${k}`; + if (k === "Region") { + process.env[`leosdk_${k.toUpperCase()}`] = value; + } else if (i % 2) { + process.env[`leosdk_${k.toLowerCase()}`] = value; + } else { + process.env[`leosdk_${k}`] = value; + } + }); + try { + const config = new EnvironmentConfiguration("leosdk").resolveSync(); + assert.equal(config.Region, "split-Region"); + assert.equal(config.LeoStream, "split-LeoStream"); + assert.equal(config.LeoCron, "split-LeoCron"); + assert.equal(config.LeoSettings, "split-LeoSettings"); + } finally { + RESOURCE_KEYS.forEach((k) => { + delete process.env[`leosdk_${k}`]; + delete process.env[`leosdk_${k.toUpperCase()}`]; + delete process.env[`leosdk_${k.toLowerCase()}`]; + }); + } + }); + + it("LeoSettings is the only optional split variable", function () { + const keys = RESOURCE_KEYS.filter((k) => k !== "LeoSettings"); + keys.forEach((k) => { process.env[`leosdk_${k}`] = `opt-${k}`; }); + try { + const config = new EnvironmentConfiguration("leosdk").resolveSync(); + assert.isUndefined(config.LeoSettings); + assert.equal(config.LeoS3, "opt-LeoS3"); + } finally { + keys.forEach((k) => { delete process.env[`leosdk_${k}`]; }); + } + }); + }); + + describe("environment variable precedence", function () { + it("RSTREAMS_CONFIG wins over leosdk, leo-sdk, LEOSDK and LEO-SDK", function () { + ENV_NAMES.forEach((n) => { process.env[n] = JSON.stringify(cfg(n)); }); + assert.equal(resolved(), "RSTREAMS_CONFIG-Region"); + }); + + it("leosdk wins once RSTREAMS_CONFIG is gone", function () { + ENV_NAMES.slice(1).forEach((n) => { process.env[n] = JSON.stringify(cfg(n)); }); + assert.equal(resolved(), "leosdk-Region"); + }); + + it("leo-sdk wins once leosdk is gone", function () { + ENV_NAMES.slice(2).forEach((n) => { process.env[n] = JSON.stringify(cfg(n)); }); + assert.equal(resolved(), "leo-sdk-Region"); + }); + + it("LEOSDK wins once leo-sdk is gone", function () { + ENV_NAMES.slice(3).forEach((n) => { process.env[n] = JSON.stringify(cfg(n)); }); + assert.equal(resolved(), "LEOSDK-Region"); + }); + + it("an unparsable earlier variable falls through to the next name", function () { + process.env.RSTREAMS_CONFIG = "{not json"; + process.env.leosdk = JSON.stringify(cfg("leosdk")); + assert.equal(resolved(), "leosdk-Region"); + }); + }); + + describe("location order", function () { + function writeFile(rel: string, region: string) { + const file = path.join(tmpRoot, rel); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, JSON.stringify(cfg(region))); + } + + it("leo-config is consulted before every environment variable", function () { + leoConfig.bootstrap({ _global: { leosdk: cfg("leo-config") } }); + process.env.RSTREAMS_CONFIG = JSON.stringify(cfg("env")); + assert.equal(resolved(), "leo-config-Region"); + }); + + it("leo-config accepts the leo_sdk, leo-sdk, rstreamssdk, rstreams_sdk and rstreams-sdk names", function () { + ["leo_sdk", "leo-sdk", "rstreamssdk", "rstreams_sdk", "rstreams-sdk"].forEach((name) => { + leoConfig.bootstrap({ _global: { [name]: cfg(name) } }); + assert.equal(resolved(), `${name}-Region`); + delete leoConfig[name]; + }); + }); + + it("environment variables win over process and global objects", function () { + process.env.LEOSDK = JSON.stringify(cfg("env")); + process["leosdk"] = cfg("process"); + global["leosdk"] = cfg("global"); + assert.equal(resolved(), "env-Region"); + }); + + it("process.leosdk wins over process['leo-sdk'] and process.rstreams_config", function () { + process["leosdk"] = cfg("process-leosdk"); + process["leo-sdk"] = cfg("process-leo-sdk"); + process["rstreams_config"] = cfg("process-rstreams"); + assert.equal(resolved(), "process-leosdk-Region"); + delete process["leosdk"]; + assert.equal(resolved(), "process-leo-sdk-Region"); + delete process["leo-sdk"]; + assert.equal(resolved(), "process-rstreams-Region"); + }); + + it("process objects win over global objects", function () { + process["rstreams_config"] = cfg("process"); + global["leosdk"] = cfg("global"); + assert.equal(resolved(), "process-Region"); + }); + + it("global.leosdk wins over global['leo-sdk'] and global.rstreams_config", function () { + global["leosdk"] = cfg("global-leosdk"); + global["leo-sdk"] = cfg("global-leo-sdk"); + global["rstreams_config"] = cfg("global-rstreams"); + assert.equal(resolved(), "global-leosdk-Region"); + delete global["leosdk"]; + assert.equal(resolved(), "global-leo-sdk-Region"); + delete global["leo-sdk"]; + assert.equal(resolved(), "global-rstreams-Region"); + }); + + it("global objects win over a config file", function () { + global["rstreams_config"] = cfg("global"); + writeFile("a/b/leo.config.json", "file"); + assert.equal(resolved(), "global-Region"); + }); + + it("a config file wins over secrets", function () { + process.env.LEO_CONFIG_SECRET = "leo-secret"; + sandbox.stub(awsSdkSync, "SecretsManager").returns({ getSecretValue: () => ({ SecretString: JSON.stringify(cfg("secret")) }) } as any); + writeFile("a/b/leo.config.json", "file"); + assert.equal(resolved(), "file-Region"); + }); + + it("later providers are not consulted once an earlier one resolves", function () { + process.env.LEOSDK = JSON.stringify(cfg("env")); + const secrets = sandbox.stub(awsSdkSync, "SecretsManager"); + assert.equal(resolved(), "env-Region"); + assert.equal(secrets.callCount, 0); + }); + }); + + describe("file tree", function () { + function writeFile(rel: string, content: string) { + const file = path.join(tmpRoot, rel); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, content); + } + const tree = () => new FileTreeConfiguration(path.join(tmpRoot, "a", "b"), ["leo.config.json", "rstreams.config.json", "leoconfig.js", "config/leo.config.json"]); + + it("file tree reads a JSON file from the starting directory", function () { + writeFile("a/b/leo.config.json", JSON.stringify(cfg("start"))); + assert.equal(tree().resolveSync().Region, "start-Region"); + }); + + it("file tree walks up to parent directories", function () { + writeFile("a/leo.config.json", JSON.stringify(cfg("parent"))); + assert.equal(tree().resolveSync().Region, "parent-Region"); + }); + + it("file tree prefers the nearest directory over an earlier filename", function () { + writeFile("a/leo.config.json", JSON.stringify(cfg("parent"))); + writeFile("a/b/rstreams.config.json", JSON.stringify(cfg("near"))); + assert.equal(tree().resolveSync().Region, "near-Region"); + }); + + it("file tree takes filenames in list order within a directory", function () { + writeFile("a/b/rstreams.config.json", JSON.stringify(cfg("second"))); + writeFile("a/b/leo.config.json", JSON.stringify(cfg("first"))); + assert.equal(tree().resolveSync().Region, "first-Region"); + }); + + it("file tree loads a .js module", function () { + writeFile("a/b/leoconfig.js", `module.exports = ${JSON.stringify(cfg("js"))};`); + assert.equal(tree().resolveSync().Region, "js-Region"); + }); + + it("file tree reads from a config subfolder", function () { + writeFile("a/b/config/leo.config.json", JSON.stringify(cfg("sub"))); + assert.equal(tree().resolveSync().Region, "sub-Region"); + }); + + it("file tree skips a file that fails to load and tries the next filename", function () { + writeFile("a/b/leo.config.json", "{broken"); + writeFile("a/b/rstreams.config.json", JSON.stringify(cfg("next"))); + assert.equal(tree().resolveSync().Region, "next-Region"); + }); + + it("file tree accepts a single filename string", function () { + writeFile("a/b/only.json", JSON.stringify(cfg("only"))); + const single = new FileTreeConfiguration(path.join(tmpRoot, "a", "b"), "only.json" as any); + assert.equal(single.resolveSync().Region, "only-Region"); + }); + + it("file tree reports the load errors when nothing loads", function () { + writeFile("a/b/leo.config.json", "{broken"); + let err: any; + try { tree().resolveSync(); } catch (e) { err = e; } + assert.equal(err.message, "Unable to find file config"); + assert.equal(err.code, "FileTreeConfigurationProviderFailure"); + assert.equal(err.errors.length, 1); + }); + }); + + describe("secrets", function () { + function stubSecrets(byId: Record) { + const getSecretValue = sandbox.stub().callsFake(({ SecretId }) => { + if (!(SecretId in byId)) { throw new Error(`no secret ${SecretId}`); } + return { SecretString: JSON.stringify(byId[SecretId]) }; + }); + const ctor = sandbox.stub(awsSdkSync, "SecretsManager").returns({ getSecretValue } as any); + return { getSecretValue, ctor }; + } + + it("LEO_CONFIG_SECRET is consulted before RSTREAMS_CONFIG_SECRET", function () { + process.env.LEO_CONFIG_SECRET = "leo-secret"; + process.env.RSTREAMS_CONFIG_SECRET = "rs-secret"; + stubSecrets({ "leo-secret": cfg("leo"), "rs-secret": cfg("rs") }); + assert.equal(resolved(), "leo-Region"); + }); + + it("RSTREAMS_CONFIG_SECRET is used when LEO_CONFIG_SECRET is unset", function () { + process.env.RSTREAMS_CONFIG_SECRET = "rs-secret"; + stubSecrets({ "rs-secret": cfg("rs") }); + assert.equal(resolved(), "rs-Region"); + }); + + it("RSTREAMS_CONFIG_SECRET is used when the LEO_CONFIG_SECRET secret cannot be read", function () { + process.env.LEO_CONFIG_SECRET = "missing"; + process.env.RSTREAMS_CONFIG_SECRET = "rs-secret"; + stubSecrets({ "rs-secret": cfg("rs") }); + assert.equal(resolved(), "rs-Region"); + }); + + it("the secret is fetched from AWS_REGION, defaulting to us-east-1", function () { + process.env.RSTREAMS_CONFIG_SECRET = "rs-secret"; + const { ctor, getSecretValue } = stubSecrets({ "rs-secret": cfg("rs") }); + resolved(); + assert.equal(ctor.getCall(0).args[0].region, "us-east-1"); + assert.deepEqual(getSecretValue.getCall(0).args[0], { SecretId: "rs-secret" }); + AWSSecretsConfiguration.clearCache(); + process.env.AWS_REGION = "eu-west-1"; + resolved(); + assert.equal(ctor.getCall(1).args[0].region, "eu-west-1"); + }); + + it("a fetched secret is cached per region and secret until it expires", function () { + process.env.RSTREAMS_CONFIG_SECRET = "rs-secret"; + const { getSecretValue } = stubSecrets({ "rs-secret": cfg("rs") }); + resolved(); + resolved(); + assert.equal(getSecretValue.callCount, 1); + process.env.AWS_REGION = "eu-west-1"; + resolved(); + assert.equal(getSecretValue.callCount, 2); + }); + + it("an expired cache entry is fetched again", function () { + process.env.RSTREAMS_CONFIG_SECRET = "rs-secret"; + const { getSecretValue } = stubSecrets({ "rs-secret": cfg("rs") }); + const clock = sandbox.useFakeTimers({ now: 1_000_000, toFake: ["Date"] }); + new AWSSecretsConfiguration("RSTREAMS_CONFIG_SECRET", 1000).resolveSync(); + clock.tick(500); + new AWSSecretsConfiguration("RSTREAMS_CONFIG_SECRET", 1000).resolveSync(); + assert.equal(getSecretValue.callCount, 1); + clock.tick(600); + new AWSSecretsConfiguration("RSTREAMS_CONFIG_SECRET", 1000).resolveSync(); + assert.equal(getSecretValue.callCount, 2); + }); + + it("the chain throws the last provider error when nothing resolves", function () { + let err: any; + try { resolved(); } catch (e) { err = e; } + assert.equal(err.message, "Secret not specified. Use ENV var RSTREAMS_CONFIG_SECRET."); + }); + }); + + describe("chain construction", function () { + it("Replace uses only the given providers", function () { + const chain = new ConfigProviderChain([() => new EnvironmentConfiguration("custom")], ProvidersInputType.Replace); + assert.equal(chain.providers.length, 1); + }); + + it("Prepend puts the given providers ahead of the defaults and Append puts them last", function () { + const custom = { Region: "custom" } as any; + const defaults = ConfigProviderChain.defaultProviders.length; + const prepended = new ConfigProviderChain(custom, ProvidersInputType.Prepend); + assert.equal(prepended.providers.length, defaults + 1); + assert.equal(prepended.providers[0], custom); + const appended = new ConfigProviderChain(custom, ProvidersInputType.Append); + assert.equal(appended.providers[defaults], custom); + }); + + it("the default chain has leo-config, 5 env, 3 process, 3 global, 1 file tree and 2 secret providers", function () { + assert.equal(ConfigProviderChain.defaultProviders.length, 15); + }); + }); +}); diff --git a/test/lib.stream.leo-stream.misc.utest.ts b/test/lib.stream.leo-stream.misc.utest.ts new file mode 100644 index 0000000..6c97f54 --- /dev/null +++ b/test/lib.stream.leo-stream.misc.utest.ts @@ -0,0 +1,594 @@ +import sinon from "sinon"; +import { assert } from "chai"; +import zlib from "zlib"; +import { Readable, Writable, PassThrough } from "stream"; +import utilFn from "../lib/stream/leo-stream"; + +describe("lib/stream/leo-stream.js misc", function () { + const sandbox = sinon.createSandbox(); + let ls: any; + let configure: any; + let putRecords: sinon.SinonStub; + let toS3Uploads: { bucket: string, key: string, chunks: Buffer[] }[]; + let onUpdateCallbacks: ((c: any) => void)[]; + + function build(extra: any = {}, stubToS3 = true) { + onUpdateCallbacks = []; + configure = { + onUpdate: (cb: any) => onUpdateCallbacks.push(cb), + validate: () => { }, + resources: { LeoCron: "mock-LeoCron", LeoEvent: "mock-LeoEvent", LeoStream: "mock-LeoStream", LeoS3: "mock-leos3", LeoSettings: "mock-LeoSettings" }, + aws: { region: "us-west-2" }, + stream: "mock-stream", + bus: { firehose: "mock-firehose" }, + ...extra + }; + ls = utilFn(configure); + putRecords = sandbox.stub(ls.kinesis, "putRecords").callsFake((params: any, cb: any) => cb(null, { FailedRecordCount: 0, Records: params.Records.map(() => ({})) })); + toS3Uploads = []; + if (stubToS3) { + sandbox.stub(ls, "toS3").callsFake((bucket: string, key: string) => { + const upload = { bucket, key, chunks: [] as Buffer[] }; + toS3Uploads.push(upload); + return new Writable({ write(chunk, enc, cb) { upload.chunks.push(Buffer.from(chunk)); cb(); } }); + }); + } + return ls; + } + + beforeEach(() => { + delete (process as any).__config; + build(); + }); + + afterEach(() => { + sandbox.restore(); + }); + + function decodeGzipRecord(data: Buffer) { + return zlib.gunzipSync(data).toString().trim().split("\n").map((l) => JSON.parse(l)); + } + + function readEvents(queue: string, count: number, extra: any = {}) { + return Array.from({ length: count }, (_, i) => ({ + id: "upstream", event: queue, eid: `z/2022/03/14/22/38/1647297513618-000000${i}`, payload: { n: i }, + event_source_timestamp: 1000 + i, timestamp: 2000 + i, ...extra + })); + } + + function collect(stream: any, events: any[]) { + const out: any[] = []; + return ls.pipeAsync(ls.eventstream.readArray(events), stream, ls.write({ hasCommands: false }, (o, done) => { out.push(o); done(); })).then(() => out); + } + + describe("toDynamoDB", function () { + let batchWrite: sinon.SinonStub; + beforeEach(() => { + batchWrite = sandbox.stub(ls.dynamodb.docClient, "batchWrite").callsFake((p: any, cb: any) => cb(null, { UnprocessedItems: {} })); + }); + + function itemsOf(call: any) { + return call.args[0].RequestItems.T.map((r: any) => r.PutRequest.Item); + } + + it("toDynamoDB writes records as PutRequests", async function () { + await ls.pipeAsync(ls.eventstream.readArray([{ id: "a" }, { id: "b" }]), ls.toDynamoDB("T", {})); + assert.equal(batchWrite.callCount, 1); + assert.deepEqual(batchWrite.getCall(0).args[0], { + RequestItems: { T: [{ PutRequest: { Item: { id: "a" } } }, { PutRequest: { Item: { id: "b" } } }] }, + ReturnConsumedCapacity: "TOTAL" + }); + }); + + it("toDynamoDB splits a flush into batches of 25", async function () { + const records = Array.from({ length: 60 }, (_, i) => ({ id: `r${i}` })); + await ls.pipeAsync(ls.eventstream.readArray(records), ls.toDynamoDB("T", { records: 100 })); + assert.deepEqual(batchWrite.getCalls().map((c: any) => c.args[0].RequestItems.T.length).sort((a: number, b: number) => b - a), [25, 25, 10]); + }); + + it("toDynamoDB flushes every opts.records records", async function () { + const records = Array.from({ length: 6 }, (_, i) => ({ id: `r${i}` })); + await ls.pipeAsync(ls.eventstream.readArray(records), ls.toDynamoDB("T", { records: 3 })); + assert.deepEqual(batchWrite.getCalls().map((c: any) => c.args[0].RequestItems.T.length), [3, 3]); + }); + + it("toDynamoDB keeps only the last record per hash key", async function () { + await ls.pipeAsync(ls.eventstream.readArray([{ id: "a", v: 1 }, { id: "b", v: 1 }, { id: "a", v: 2 }]), ls.toDynamoDB("T", { hash: "id" })); + assert.deepEqual(itemsOf(batchWrite.getCall(0)), [{ id: "a", v: 2 }, { id: "b", v: 1 }]); + }); + + it("toDynamoDB keys on hash and range together", async function () { + await ls.pipeAsync(ls.eventstream.readArray([{ id: "a", r: 1, v: 1 }, { id: "a", r: 2, v: 2 }, { id: "a", r: 1, v: 3 }]), ls.toDynamoDB("T", { hash: "id", range: "r" })); + assert.deepEqual(itemsOf(batchWrite.getCall(0)), [{ id: "a", r: 1, v: 3 }, { id: "a", r: 2, v: 2 }]); + }); + + it("toDynamoDB merge deep-merges duplicates within a batch", async function () { + sandbox.stub(ls.dynamodb, "batchGetTable").callsFake((t: string, keys: any[], o: any, cb: any) => cb(null, [])); + await ls.pipeAsync(ls.eventstream.readArray([{ id: "a", n: { x: 1 } }, { id: "a", n: { y: 2 } }]), ls.toDynamoDB("T", { hash: "id", merge: true })); + assert.deepEqual(itemsOf(batchWrite.getCall(0)), [{ id: "a", n: { x: 1, y: 2 } }]); + }); + + it("toDynamoDB merge reads existing items by key and merges the new data over them", async function () { + const batchGetTable = sandbox.stub(ls.dynamodb, "batchGetTable").callsFake((t: string, keys: any[], o: any, cb: any) => cb(null, [{ id: "a", old: true, n: { x: 0, z: 9 } }])); + await ls.pipeAsync(ls.eventstream.readArray([{ id: "a", n: { x: 1 } }, { id: "b" }]), ls.toDynamoDB("T", { hash: "id", merge: true })); + assert.equal(batchGetTable.getCall(0).args[0], "T"); + assert.deepEqual(batchGetTable.getCall(0).args[1].map((k: any) => k.id), ["a", "b"]); + assert.deepEqual(itemsOf(batchWrite.getCall(0)), [{ id: "a", old: true, n: { x: 1, z: 9 } }, { id: "b" }]); + }); + + it("toDynamoDB merge fails the stream when the existing-item lookup fails", async function () { + sandbox.stub(ls.dynamodb, "batchGetTable").callsFake((t: string, keys: any[], o: any, cb: any) => cb(new Error("lookup failed"))); + let err: any; + try { + await ls.pipeAsync(ls.eventstream.readArray([{ id: "a" }]), ls.toDynamoDB("T", { hash: "id", merge: true })); + } catch (e) { err = e; } + assert.exists(err); + assert.equal(batchWrite.callCount, 0); + }); + + it("toDynamoDB resends only unprocessed items", async function () { + batchWrite.onFirstCall().callsFake((p: any, cb: any) => cb(null, { UnprocessedItems: { T: [{ PutRequest: { Item: { id: "b" } } }] } })); + await ls.pipeAsync(ls.eventstream.readArray([{ id: "a" }, { id: "b" }]), ls.toDynamoDB("T", {})); + assert.equal(batchWrite.callCount, 2); + assert.deepEqual(itemsOf(batchWrite.getCall(1)), [{ id: "b" }]); + }); + + it("toDynamoDB retries after a retryable error", async function () { + const retryable: any = new Error("throttled"); + retryable.retryable = true; + batchWrite.onFirstCall().callsFake((p: any, cb: any) => cb(retryable)); + await ls.pipeAsync(ls.eventstream.readArray([{ id: "a" }]), ls.toDynamoDB("T", {})); + assert.equal(batchWrite.callCount, 2); + assert.deepEqual(itemsOf(batchWrite.getCall(1)), [{ id: "a" }]); + }); + + it("toDynamoDB fails the stream on a non-retryable error", async function () { + batchWrite.callsFake((p: any, cb: any) => cb(new Error("validation"))); + let err: any; + try { + await ls.pipeAsync(ls.eventstream.readArray([{ id: "a" }]), ls.toDynamoDB("T", {})); + } catch (e) { err = e; } + assert.exists(err); + assert.equal(batchWrite.callCount, 1); + }); + + it("toDynamoDB with no records never calls batchWrite", async function () { + await ls.pipeAsync(ls.eventstream.readArray([]), ls.toDynamoDB("T", {})); + assert.equal(batchWrite.callCount, 0); + }); + }); + + describe("autoSwitch", function () { + it("autoSwitch consumes events without emitting them or flushing the inner pipeline", async function () { + const out = await collect(ls.autoSwitch("q1", { recordsPerSecond: 1e9 }), [{ id: "bot1", event: "q1", payload: { n: 1 } }, { id: "bot1", event: "q1", payload: { n: 2 } }]); + assert.deepEqual(out, []); + assert.equal(putRecords.callCount, 0); + assert.equal(toS3Uploads.filter((u) => u.chunks.length > 0).length, 0); + }); + }); + + describe("magic", function () { + function passStream(seen: any[]) { + return ls.through((o, done) => { seen.push(o); done(null, o); }); + } + + function slowStream(seen: any[]) { + return ls.through((o, done) => { seen.push(o.n); setTimeout(() => done(null, o), 5); }); + } + + it("magic sends everything through the first stream that can sustain the load", async function () { + const seenA: any[] = []; + const seenB: any[] = []; + const out = await collect(ls.magic([ + { stream: passStream(seenA), canSustainLoad: () => true, label: "A" }, + { stream: passStream(seenB), canSustainLoad: () => true, label: "B" } + ], {}), [{ n: 1 }, { n: 2 }, { n: 3 }]); + assert.deepEqual(seenA.map((e) => e.n), [1, 2, 3]); + assert.deepEqual(seenB, []); + assert.deepEqual(out.map((e) => e.n), [1, 2, 3]); + }); + + it("magic falls back to the last stream when none can sustain the load", async function () { + const seenA: any[] = []; + const seenB: any[] = []; + await collect(ls.magic([ + { stream: passStream(seenA), label: "A" }, + { stream: passStream(seenB), label: "B" } + ], {}), [{ n: 1 }, { n: 2 }]); + assert.deepEqual(seenA, []); + assert.deepEqual(seenB.map((e) => e.n), [1, 2]); + }); + + it("magic numeric canSustainLoad compares against records per second", async function () { + const seenA: any[] = []; + const seenB: any[] = []; + await collect(ls.magic([ + { stream: passStream(seenA), canSustainLoad: 1000000 }, + { stream: passStream(seenB) } + ], {}), [{ n: 1 }, { n: 2 }]); + assert.deepEqual(seenA.map((e) => e.n), [1, 2]); + assert.deepEqual(seenB, []); + }); + + it("magic assigns labels from the stream index and a default minimum duration", function () { + const streams: any[] = [{ stream: passStream([]) }, { stream: passStream([]) }]; + ls.magic(streams, {}); + assert.deepEqual(streams.map((s) => s.label), ["0", "1"]); + assert.deepEqual(streams.map((s) => s.minDurationMS), [5000, 5000]); + }); + + it("magic switches streams once the current one can no longer sustain the load", async function () { + const seenA: any[] = []; + const seenB: any[] = []; + let calls = 0; + let clock = 1000000000000; + sandbox.stub(Date, "now").callsFake(() => (clock += 10)); + const out = await collect(ls.magic([ + { stream: slowStream(seenA), canSustainLoad: () => ++calls <= 2, minDurationMS: 1, label: "A" }, + { stream: slowStream(seenB), minDurationMS: 1, label: "B" } + ], {}), [{ n: 1 }, { n: 2 }, { n: 3 }, { n: 4 }]); + assert.equal(seenA[0], 1); + assert.isAbove(seenB.length, 0); + assert.deepEqual(seenA.concat(seenB), [1, 2, 3, 4]); + assert.deepEqual(out.map((e) => e.n), [1, 2, 3, 4]); + }); + }); + + describe("streams", function () { + let s3: any; + beforeEach(() => { + s3 = ls.s3; + }); + + function read(stream: any) { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + stream.on("data", (c: Buffer) => chunks.push(Buffer.from(c))); + stream.on("end", () => resolve(Buffer.concat(chunks).toString())); + stream.on("error", reject); + }); + } + + it("fromS3 reads the object using bucket, key and range", async function () { + const getObject = sandbox.stub(s3, "getObject").resolves({ Body: Readable.from([Buffer.from("hello")]) }); + const text = await read(ls.fromS3({ bucket: "b", key: "k", range: "bytes=0-4" })); + assert.equal(text, "hello"); + assert.deepEqual(getObject.getCall(0).args[0], { Bucket: "b", Key: "k", Range: "bytes=0-4" }); + }); + + it("fromS3 accepts capitalized Bucket and Key", async function () { + const getObject = sandbox.stub(s3, "getObject").resolves({ Body: Readable.from([Buffer.from("x")]) }); + await read(ls.fromS3({ Bucket: "B", Key: "K" })); + assert.deepEqual(getObject.getCall(0).args[0], { Bucket: "B", Key: "K", Range: undefined }); + }); + + it("fromS3 finalizes a partial multipart upload on NoSuchKey and reads again", async function () { + const failing = new Readable({ read() { const e: any = new Error("missing"); e.name = "NoSuchKey"; this.destroy(e); } }); + const getObject = sandbox.stub(s3, "getObject"); + getObject.onFirstCall().resolves({ Body: failing }); + getObject.onSecondCall().resolves({ Body: Readable.from([Buffer.from("recovered")]) }); + const finalize = sandbox.stub(ls, "tryFinalizeMultipartUpload").resolves(); + const text = await read(ls.fromS3({ bucket: "b", key: "k" })); + assert.equal(text, "recovered"); + assert.equal(finalize.callCount, 1); + assert.equal(getObject.callCount, 2); + }); + + it("fromS3 passes the NoSuchKey error along when finalizing fails", async function () { + const failing = new Readable({ read() { const e: any = new Error("missing"); e.name = "NoSuchKey"; this.destroy(e); } }); + sandbox.stub(s3, "getObject").resolves({ Body: failing }); + sandbox.stub(ls, "tryFinalizeMultipartUpload").rejects(new Error("no uploads")); + let err: any; + try { await read(ls.fromS3({ bucket: "b", key: "k" })); } catch (e) { err = e; } + assert.equal(err.message, "missing"); + }); + + it("fromS3 passes other errors along", async function () { + const failing = new Readable({ read() { this.destroy(new Error("network")); } }); + sandbox.stub(s3, "getObject").resolves({ Body: failing }); + let err: any; + try { await read(ls.fromS3({ bucket: "b", key: "k" })); } catch (e) { err = e; } + assert.equal(err.message, "network"); + }); + + it("tryFinalizeMultipartUpload completes the first upload with its listed parts", async function () { + sandbox.stub(s3, "listMultipartUploads").resolves({ Uploads: [{ Key: "bus/k", UploadId: "u1" }] }); + const listParts = sandbox.stub(s3, "listParts").resolves({ Parts: [{ ETag: "e1", PartNumber: 1, Size: 5 }, { ETag: "e2", PartNumber: 2, Size: 5 }] }); + const complete = sandbox.stub(s3, "completeMultipartUpload").resolves({}); + await ls.tryFinalizeMultipartUpload({ bucket: "b", key: "bus/k" }); + assert.deepEqual(s3.listMultipartUploads.getCall(0).args[0], { Bucket: "b", Prefix: "bus/k" }); + assert.deepEqual(listParts.getCall(0).args[0], { Bucket: "b", Key: "bus/k", UploadId: "u1" }); + assert.deepEqual(complete.getCall(0).args[0], { + Bucket: "b", Key: "bus/k", UploadId: "u1", + MultipartUpload: { Parts: [{ ETag: "e1", PartNumber: 1 }, { ETag: "e2", PartNumber: 2 }] } + }); + }); + + it("tryFinalizeMultipartUpload does nothing without uploads or parts", async function () { + const list = sandbox.stub(s3, "listMultipartUploads"); + const complete = sandbox.stub(s3, "completeMultipartUpload").resolves({}); + list.onFirstCall().resolves({ Uploads: [] }); + await ls.tryFinalizeMultipartUpload({ Bucket: "b", Key: "k" }); + list.onSecondCall().resolves({ Uploads: [{ Key: "k", UploadId: "u" }] }); + sandbox.stub(s3, "listParts").resolves({ Parts: [] }); + await ls.tryFinalizeMultipartUpload({ Bucket: "b", Key: "k" }); + assert.equal(complete.callCount, 0); + }); + }); + + describe("toS3", function () { + it("toS3 uploads written chunks with the bucket and key", async function () { + sandbox.restore(); + build({}, false); + const sent: any[] = []; + sandbox.stub(ls.s3, "send").callsFake(async (command: any) => { sent.push(command.input); return {}; }); + const stream = ls.toS3("bucket-1", "folder/file.gz"); + stream.write("hello "); + await new Promise((resolve, reject) => stream.end((err: any) => err ? reject(err) : resolve())); + assert.equal(sent.length, 1); + assert.equal(sent[0].Bucket, "bucket-1"); + assert.equal(sent[0].Key, "folder/file.gz"); + assert.equal(Buffer.from(sent[0].Body).toString(), "hello "); + }); + + it("toS3 reports an upload failure through the error event and not the end callback", async function () { + sandbox.restore(); + build({}, false); + sandbox.stub(ls.s3, "send").rejects(new Error("access denied")); + const stream = ls.toS3("bucket-1", "folder/file.gz"); + const errors: any[] = []; + stream.on("error", (e: any) => errors.push(e)); + stream.write("hello"); + const endArg = await new Promise((resolve) => stream.end(resolve)); + assert.isUndefined(endArg); + assert.isAtLeast(errors.length, 1); + errors.forEach((e) => assert.equal(e.message, "access denied")); + }); + }); + + describe("logSourceRead/logTargetWrite", function () { + let checkpoint: sinon.SinonStub; + beforeEach(() => { + sandbox.useFakeTimers({ now: 1700000000000, toFake: ["Date"] }); + checkpoint = sandbox.stub(ls.cron, "checkpoint").callsFake((id: string, event: string, cp: any, cb: any) => cb()); + }); + + it("logSourceRead with two strings uses them as bot id and system", async function () { + await new Promise((resolve) => ls.logSourceRead("bot1", "sysA", 5, {}, resolve)); + assert.deepEqual(checkpoint.getCall(0).args.slice(0, 3), ["bot1", "system.sysA", { + eid: 1700000000000, records: 5, started_timestamp: undefined, timestamp: 1700000000000, source_timestamp: 1700000000000 + }]); + }); + + it("logSourceRead with one string takes the bot id from the registry", async function () { + configure.registry.id = "registry-bot"; + await new Promise((resolve) => ls.logSourceRead("sysA", 7, resolve)); + assert.equal(checkpoint.getCall(0).args[0], "registry-bot"); + assert.equal(checkpoint.getCall(0).args[1], "system.sysA"); + assert.equal(checkpoint.getCall(0).args[2].records, 7); + }); + + it("logSourceRead applies subqueue and timestamps from the options", async function () { + await new Promise((resolve) => ls.logSourceRead("bot1", "sysA", 1, { subqueue: "sub", timestamp: 5, source_timestamp: 3 }, resolve)); + assert.equal(checkpoint.getCall(0).args[1], "system.sysA.sub"); + assert.deepEqual([checkpoint.getCall(0).args[2].started_timestamp, checkpoint.getCall(0).args[2].timestamp, checkpoint.getCall(0).args[2].source_timestamp], [5, 5, 3]); + }); + + it("logSourceRead falls back to event_source_timestamp and defaults records to 0", async function () { + await new Promise((resolve) => ls.logSourceRead("bot1", "sysA", { event_source_timestamp: 9 }, resolve)); + assert.equal(checkpoint.getCall(0).args[2].source_timestamp, 9); + assert.equal(checkpoint.getCall(0).args[2].records, 0); + }); + + it("logTargetWrite puts a records event on the system queue", function () { + const putEvent = sandbox.stub(ls, "putEvent").callsFake((id: string, event: string, obj: any, opts: any, cb: any) => cb()); + const cb = sandbox.stub(); + ls.logTargetWrite("bot1", "sysB", 12, {}, cb); + assert.deepEqual(putEvent.getCall(0).args.slice(0, 4), ["bot1", "system.sysB", { records: 12 }, { metadata: { units: 12 } }]); + assert.equal(cb.callCount, 1); + }); + + it("logTargetWrite applies the subqueue", function () { + const putEvent = sandbox.stub(ls, "putEvent").callsFake((id: string, event: string, obj: any, opts: any, cb: any) => cb()); + ls.logTargetWrite("bot1", "sysB", 1, { subqueue: "out" }, () => { }); + assert.equal(putEvent.getCall(0).args[1], "system.sysB.out"); + }); + }); + + describe("checkpoint and manual checkpoint", function () { + it("checkpoint tracks per-bot per-queue stats as events are visited", function () { + sandbox.useFakeTimers({ now: 5000, toFake: ["Date"] }); + const stream = ls.checkpoint({}); + stream.visit({ id: "bot1", event: "q1", eid: "z/1", event_source_timestamp: 100, timestamp: 4000, units: 2 }); + stream.visit({ id: "bot1", event: "q1", eid: "z/2", event_source_timestamp: 50, timestamp: 4500 }); + assert.deepEqual(stream.getCheckpoint("bot1", "q1"), { + eid: "z/2", units: 3, source_timestamp: 50, started_timestamp: 4500, ended_timestamp: 4500, start_eid: "z/1" + }); + }); + + it("checkpoint keeps separate stats per queue", function () { + const stream = ls.checkpoint({}); + stream.visit({ id: "bot1", event: "q1", eid: "z/1" }); + stream.visit({ id: "bot1", event: "q2", eid: "z/9" }); + assert.equal(stream.getCheckpoint("bot1", "q1").eid, "z/1"); + assert.equal(stream.getCheckpoint("bot1", "q2").eid, "z/9"); + }); + + it("checkpoint resets the stats after a checkpoint call", async function () { + const stream = ls.checkpoint({}); + stream.visit({ id: "bot1", event: "q1", eid: "z/1" }); + await new Promise((resolve) => stream.checkpoint(() => resolve())); + assert.equal(stream.getCheckpoint("bot1", "q1").units, 0); + }); + + it("checkpoint accepts params before the callback", async function () { + const stream = ls.checkpoint({}); + stream.visit({ id: "bot1", event: "q1", eid: "z/1" }); + const err = await new Promise((resolve) => stream.checkpoint({ extra: 1 }, resolve)); + assert.notExists(err); + }); + + it("toManualCheckpoint checkpoints what was written when finished", async function () { + const checkpoint = sandbox.stub(ls.cron, "checkpoint").callsFake((id: string, event: string, cp: any, cb: any) => cb()); + const stream = ls.toManualCheckpoint("bot1", {}); + stream.write({ id: "src", event: "queue1", eid: "z/1", payload: {}, event_source_timestamp: 5, timestamp: 6 }); + stream.write({ id: "src", event: "queue1", eid: "z/2", payload: {}, event_source_timestamp: 7, timestamp: 8 }); + await new Promise((r) => setTimeout(r, 20)); + assert.equal(stream.get("bot1", "queue1").eid, "z/2"); + await new Promise((resolve) => stream.finish(() => resolve())); + assert.equal(checkpoint.callCount, 1); + assert.deepEqual(checkpoint.getCall(0).args.slice(0, 2), ["bot1", "queue1"]); + assert.equal(checkpoint.getCall(0).args[2].eid, "z/2"); + assert.equal(checkpoint.getCall(0).args[2].records, 2); + }); + + it("toManualCheckpoint flush checkpoints without ending", async function () { + const checkpoint = sandbox.stub(ls.cron, "checkpoint").callsFake((id: string, event: string, cp: any, cb: any) => cb()); + const stream = ls.toManualCheckpoint("bot1", {}); + stream.write({ id: "src", event: "queue1", eid: "z/1", payload: {}, event_source_timestamp: 5, timestamp: 6 }); + await new Promise((r) => setTimeout(r, 20)); + await new Promise((resolve) => stream.flush(() => resolve())); + assert.equal(checkpoint.callCount, 1); + assert.equal(checkpoint.getCall(0).args[2].eid, "z/1"); + }); + }); + + describe("enrich and offload", function () { + let checkpoint: sinon.SinonStub; + beforeEach(() => { + checkpoint = sandbox.stub(ls.cron, "checkpoint").callsFake((id: string, event: string, cp: any, cb: any) => cb()); + }); + + it("enrich reads the in queue, writes transformed events to the out queue and checkpoints", async function () { + const fromLeo = sandbox.stub(ls, "fromLeo").callsFake(() => ls.eventstream.readArray(readEvents("in-q", 2))); + await new Promise((resolve, reject) => ls.enrich({ + id: "bot1", inQueue: "in-q", outQueue: "out-q", start: "z/start", + transform: (payload: any, event: any, done: any) => done(null, { doubled: payload.n * 2 }) + }, (err: any) => err ? reject(err) : resolve())); + assert.deepEqual(fromLeo.getCall(0).args.slice(0, 2), ["bot1", "in-q"]); + assert.equal(fromLeo.getCall(0).args[2].start, "z/start"); + const sent = putRecords.getCalls().flatMap((c: any) => c.args[0].Records.flatMap((r: any) => decodeRecord(r.Data))); + assert.deepEqual(sent.map((e: any) => [e.id, e.event, e.payload]), [["bot1", "out-q", { doubled: 0 }], ["bot1", "out-q", { doubled: 2 }]]); + assert.equal(checkpoint.callCount, 1); + assert.deepEqual(checkpoint.getCall(0).args.slice(0, 2), ["bot1", "in-q"]); + assert.equal(checkpoint.getCall(0).args[2].eid, "z/2022/03/14/22/38/1647297513618-0000001"); + assert.equal(checkpoint.getCall(0).args[2].records, 2); + }); + + it("enrich batches the input when opts.batch is set", async function () { + sandbox.stub(ls, "fromLeo").callsFake(() => ls.eventstream.readArray(readEvents("in-q", 3))); + const batches: any[] = []; + await new Promise((resolve, reject) => ls.enrich({ + id: "bot1", inQueue: "in-q", outQueue: "out-q", batch: { count: 2 }, + each: (payload: any, event: any, done: any) => { batches.push(payload); done(null, true); } + }, (err: any) => err ? reject(err) : resolve())); + assert.deepEqual(batches.map((b) => b.length), [2, 1]); + }); + + it("offload hands each event to the handler and checkpoints the in queue", async function () { + const fromLeo = sandbox.stub(ls, "fromLeo").callsFake(() => ls.eventstream.readArray(readEvents("in-q", 3))); + const seen: any[] = []; + await new Promise((resolve, reject) => ls.offload({ + id: "bot1", inQueue: "in-q", + each: (payload: any, event: any, done: any) => { seen.push(payload); done(null, true); } + }, (err: any) => err ? reject(err) : resolve())); + assert.equal(fromLeo.getCall(0).args[1], "in-q"); + assert.deepEqual(seen, [{ n: 0 }, { n: 1 }, { n: 2 }]); + assert.equal(checkpoint.getCall(0).args[1], "in-q"); + assert.equal(checkpoint.getCall(0).args[2].records, 3); + }); + + it("offload with a batch size passes arrays of payloads to the handler", async function () { + sandbox.stub(ls, "fromLeo").callsFake(() => ls.eventstream.readArray(readEvents("in-q", 4))); + const seen: any[] = []; + await new Promise((resolve, reject) => ls.offload({ + queue: "in-q", id: "bot1", batch: 2, + transform: (payload: any, event: any, done: any) => { seen.push(payload); done(null, true); } + }, (err: any) => err ? reject(err) : resolve())); + assert.deepEqual(seen.map((b) => b.map((e: any) => e.payload)), [[{ n: 0 }, { n: 1 }], [{ n: 2 }, { n: 3 }]]); + assert.equal(checkpoint.getCall(0).args[2].eid, "z/2022/03/14/22/38/1647297513618-0000003"); + }); + + it("offload drops events when the batch map callback returns nothing", async function () { + sandbox.stub(ls, "fromLeo").callsFake(() => ls.eventstream.readArray(readEvents("in-q", 3))); + const seen: any[] = []; + await new Promise((resolve, reject) => ls.offload({ + id: "bot1", inQueue: "in-q", + batch: { size: 1, map: (payload: any, meta: any, done: any) => done(null, payload.n === 1 ? null : payload) }, + each: (payload: any, event: any, done: any) => { seen.push(payload); done(null, true); } + }, (err: any) => err ? reject(err) : resolve())); + assert.deepEqual(seen, [{ n: 0 }, { n: 2 }]); + }); + + function decodeRecord(data: Buffer) { + return decodeGzipRecord(data); + } + }); + + describe("process flush", function () { + it("a flush function can push an aggregate event correlated to the last event", async function () { + const out = await collect(ls.process("bot1", (payload: any, event: any, done: any) => done(null, true), "out-q", function (this: any, done: any) { + this.push({ summary: true }); + done(); + }), readEvents("in-q", 2).map((e) => ({ ...e, units: 1 }))); + const last = out[out.length - 1]; + assert.deepEqual(last, { + id: "bot1", event: "out-q", payload: { summary: true }, + event_source_timestamp: 1001, + correlation_id: { source: "in-q", start: "z/2022/03/14/22/38/1647297513618-0000001", units: 1 } + }); + }); + + it("a flush callback result is pushed with its options", async function () { + const out = await collect(ls.process("bot1", (payload: any, event: any, done: any) => done(null, true), "out-q", (done: any) => done(null, { total: 2 }, { queue: "totals", partial: true })), readEvents("in-q", 1)); + const last = out[out.length - 1]; + assert.equal(last.event, "totals"); + assert.deepEqual(last.payload, { total: 2 }); + assert.equal(last.correlation_id.partial_start, "z/2022/03/14/22/38/1647297513618-0000000"); + }); + + it("a flush error fails the stream", async function () { + let err: any; + try { + await collect(ls.process("bot1", (payload: any, event: any, done: any) => done(null, true), "out-q", (done: any) => done(new Error("flush failed"))), readEvents("in-q", 1)); + } catch (e) { err = e; } + assert.equal(err.message, "flush failed"); + }); + + it("outQueue may be the flush function", async function () { + const flush = sandbox.stub().callsFake(function (this: any, done: any) { done(); }); + await collect(ls.process("bot1", (payload: any, event: any, done: any) => done(null, true), flush), readEvents("in-q", 1)); + assert.equal(flush.callCount, 1); + }); + }); + + describe("client configuration", function () { + it("awsResourceConfig maps httpOptions and maxRetries onto each client", async function () { + sandbox.restore(); + build({ + awsResourceConfig: { + s3Config: { httpOptions: { connectTimeout: 111, timeout: 222 }, maxRetries: 4 }, + kinesisConfig: { maxRetries: 2 }, + firehoseConfig: { maxRetries: 6 }, + dynamodbConfig: { maxRetries: 1 } + } + }); + assert.equal(await ls.s3.config.maxAttempts(), 5); + assert.equal(await ls.kinesis.config.maxAttempts(), 3); + assert.equal(await ls.firehose.config.maxAttempts(), 7); + assert.equal(await ls.dynamodb.docClient.config.maxAttempts(), 2); + }); + + it("an explicit request handler is kept when httpOptions is also given", function () { + sandbox.restore(); + const handler = { handle: () => { }, metadata: { handlerProtocol: "http/1.1" } }; + build({ awsResourceConfig: { s3Config: { httpOptions: { timeout: 1 }, requestHandler: handler } } }); + assert.strictEqual(ls.s3.config.requestHandler, handler); + }); + + it("the onUpdate listener throws a TypeError because v3 client configs have no update function", function () { + assert.isAtLeast(onUpdateCallbacks.length, 1); + assert.throws(() => onUpdateCallbacks[0]({ aws: { region: "eu-west-1" }, credentials: {} }), TypeError); + }); + }); +}); diff --git a/test/lib.stream.leo-stream.read.utest.ts b/test/lib.stream.leo-stream.read.utest.ts new file mode 100644 index 0000000..ea8f706 --- /dev/null +++ b/test/lib.stream.leo-stream.read.utest.ts @@ -0,0 +1,727 @@ +import sinon from "sinon"; +import { assert } from "chai"; +import zlib from "zlib"; +import { PassThrough } from "stream"; +import utilFn from "../lib/stream/leo-stream"; + +const EVENT_TABLE = "mock-LeoEvent"; +const CRON_TABLE = "mock-LeoCron"; +const QUEUE = "queue1"; + +describe("lib/stream/leo-stream.js read", function () { + const sandbox = sinon.createSandbox(); + let ls: any; + let configure: any; + let batchGet: sinon.SinonStub; + let query: sinon.SinonStub; + let fromS3: sinon.SinonStub; + let s3Files: Record; + + function build(extra: any = {}) { + delete (process as any).__config; + configure = { + onUpdate: () => { }, + validate: () => { }, + resources: { LeoCron: CRON_TABLE, LeoEvent: EVENT_TABLE, LeoStream: "mock-LeoStream", LeoS3: "mock-leos3", LeoSettings: "mock-LeoSettings" }, + aws: { region: "us-west-2" }, + ...extra + }; + ls = utilFn(configure); + batchGet = sandbox.stub(ls.dynamodb.docClient, "batchGet"); + query = sandbox.stub(ls.dynamodb.docClient, "query"); + s3Files = {}; + fromS3 = sandbox.stub(ls, "fromS3").callsFake((file: any) => { + const pass = new PassThrough(); + pass.end(s3Files[file.key]); + return pass; + }); + return ls; + } + + beforeEach(() => { + delete (process as any).__config; + build(); + }); + + afterEach(() => { + sandbox.restore(); + }); + + const BASE = 1647297513618; + + function eidPrefix(ms: number) { + return ls.eventIdFromTimestamp(ms) + "-"; + } + + function makeItem(n: number, opts: { ms?: number, asS3?: boolean, key?: string, offset?: number, label?: string } = {}) { + const ms = opts.ms || BASE; + const offset = opts.offset || 0; + const events = Array.from({ length: n }, (_, i) => ({ + id: "upstream", event: QUEUE, payload: { label: opts.label || "e", n: offset + i }, + event_source_timestamp: 1000 + i, timestamp: 2000 + i + })); + const text = events.map((e) => JSON.stringify(e)).join("\n") + "\n"; + const gzip = zlib.gzipSync(text); + const base: any = { + event: QUEUE, v: 2, size: Buffer.byteLength(text), gzipSize: gzip.length, records: n, + start: eidPrefix(ms) + String(offset).padStart(7, "0"), + end: eidPrefix(ms) + String(offset + n - 1).padStart(7, "0") + }; + if (opts.asS3) { + const key = opts.key || `file-${ms}-${offset}`; + s3Files[key] = gzip; + return { + ...base, + s3: { bucket: "mock-leos3", key }, + offsets: [{ size: base.size, gzipOffset: 0, gzipSize: gzip.length, records: n, start: 0, end: n - 1, event: QUEUE, offset: 0 }] + }; + } + return { ...base, gzip }; + } + + function leoEvent(items: any[], extra: any = {}) { + const last = items[items.length - 1]; + return { event: QUEUE, v: 2, max_eid: last ? last.end : ls.eventIdFromTimestamp(BASE), ...extra }; + } + + function setup(items: any[], opts: { event?: any, cron?: any, pages?: any[][] } = {}) { + batchGet.callsFake((params: any, cb: any) => cb(null, { + Responses: { + [EVENT_TABLE]: opts.event === null ? [] : [opts.event || leoEvent(items)], + [CRON_TABLE]: opts.cron ? [opts.cron] : [] + }, + UnprocessedKeys: {} + })); + const pages = opts.pages || [items]; + pages.forEach((page, i) => { + query.onCall(i).callsFake((params: any, cb: any) => cb(null, { Items: page, Count: page.length, ConsumedCapacity: { CapacityUnits: 1 } })); + }); + query.callsFake((params: any, cb: any) => cb(null, { Items: [], Count: 0 })); + } + + function readAll(opts: any = {}, id = "bot1", queue = QUEUE) { + return new Promise((resolve, reject) => { + const out: any[] = []; + const stream = ls.fromLeo(id, queue, { start: ls.eventIdFromTimestamp(BASE - 1000), ...opts }); + stream.on("error", reject); + stream.on("data", (e: any) => out.push(e)); + stream.on("end", () => resolve(out)); + }); + } + + describe("start position and query", function () { + it("fromLeo reads every event of an inline gzip item and assigns eids from the item start", async function () { + const item = makeItem(3); + setup([item]); + const events = await readAll(); + assert.deepEqual(events.map((e) => e.payload.n), [0, 1, 2]); + assert.deepEqual(events.map((e) => e.eid), [0, 1, 2].map((i) => eidPrefix(BASE) + String(i).padStart(7, "0"))); + assert.isTrue(events.every((e) => e.event === QUEUE)); + }); + + it("fromLeo looks up the cron row and event row together", async function () { + setup([makeItem(1)]); + await readAll(); + assert.deepEqual(batchGet.getCall(0).args[0].RequestItems, { + [CRON_TABLE]: { Keys: [{ id: "bot1" }] }, + [EVENT_TABLE]: { Keys: [{ event: QUEUE }] } + }); + }); + + it("fromLeo queries the stream table between start and the event max eid", async function () { + const item = makeItem(1); + setup([item]); + const startEid = ls.eventIdFromTimestamp(BASE - 1000); + await readAll({ start: startEid, stream_query_limit: 7 }); + const params = query.getCall(0).args[0]; + assert.equal(params.TableName, "mock-LeoStream"); + assert.equal(params.KeyConditionExpression, "#event = :event and #key between :start and :maxkey"); + assert.deepEqual(params.ExpressionAttributeNames, { "#event": "event", "#key": "end" }); + assert.equal(params.Limit, 7); + assert.deepEqual(params.ExpressionAttributeValues, { ":event": QUEUE, ":start": startEid + " ", ":maxkey": item.end }); + }); + + it("fromLeo defaults the query limit to 50 and to 1000 with fast_s3_read", async function () { + setup([makeItem(1)]); + await readAll(); + assert.equal(query.getCall(0).args[0].Limit, 50); + sandbox.restore(); + build(); + setup([makeItem(1, { asS3: true })]); + await readAll({ fast_s3_read: true }); + assert.equal(query.getCall(0).args[0].Limit, 1000); + }); + + it("fromLeo starts after the cron checkpoint when no start is given", async function () { + const item = makeItem(1); + setup([item], { cron: { id: "bot1", checkpoints: { read: { "queue:queue1": { checkpoint: "z/2022/03/14/22/00/1647295200000-0000005" } } } } }); + await readAll({ start: undefined }); + assert.equal(query.getCall(0).args[0].ExpressionAttributeValues[":start"], "z/2022/03/14/22/00/1647295200000-0000005 "); + }); + + it("fromLeo starts at the registry starteid when set", async function () { + setup([makeItem(1)]); + configure.registry.__cron = { starteid: { "queue:queue1": "z/2022/03/14/22/00/1647295200000-0000001" } }; + await readAll({ start: undefined }); + assert.equal(query.getCall(0).args[0].ExpressionAttributeValues[":start"], "z/2022/03/14/22/00/1647295200000-0000001 "); + }); + + it("fromLeo records the cron checkpoint in the registry", async function () { + const cp = { checkpoint: "z/2022/03/14/22/00/1-0", records: 3 }; + setup([makeItem(1)], { cron: { id: "bot1", checkpoints: { read: { "queue:queue1": cp } } } }); + await readAll(); + assert.deepEqual(configure.registry.__cron.checkpoints.read["queue:queue1"], cp); + }); + + it("fromLeo caps the read at the registry maxeid, then maxOverride, then the event max eid", async function () { + const item = makeItem(1); + setup([item]); + configure.registry.__cron = { maxeid: "z/9999/maxeid" }; + await readAll(); + assert.equal(query.getCall(0).args[0].ExpressionAttributeValues[":maxkey"], "z/9999/maxeid"); + sandbox.restore(); + build(); + setup([item]); + await readAll({ maxOverride: item.end }); + assert.equal(query.getCall(0).args[0].ExpressionAttributeValues[":maxkey"], item.end); + }); + + it("fromLeo applies subqueue to the queue name", async function () { + setup([makeItem(1)]); + await readAll({ subqueue: "sub" }); + assert.equal(query.getCall(0).args[0].ExpressionAttributeValues[":event"], "queue1.sub"); + }); + + it("fromLeo ends empty when the queue has no event row", async function () { + setup([], { event: null }); + assert.deepEqual(await readAll(), []); + assert.equal(query.callCount, 0); + }); + + it("fromLeo reads a phantom queue even without an event row", async function () { + setup([], { event: null }); + assert.deepEqual(await readAll({}, "bot1", "queue1/_phantom"), []); + }); + + it("fromLeo emits an error when the lookup fails", async function () { + batchGet.callsFake((params: any, cb: any) => cb(new Error("lookup failed"))); + let err: any; + try { await readAll(); } catch (e) { err = e; } + assert.match(err.message, /lookup failed/); + }); + + it("fromLeo emits an error when the lookup is throttled", async function () { + batchGet.callsFake((params: any, cb: any) => cb(null, { Responses: {}, UnprocessedKeys: { [EVENT_TABLE]: {} } })); + let err: any; + try { await readAll(); } catch (e) { err = e; } + assert.equal(err.message, "Not enough capacity to read"); + }); + + it("fromLeo emits an error when the stream query fails", async function () { + setup([makeItem(1)]); + query.reset(); + query.callsFake((params: any, cb: any) => cb(new Error("query failed"))); + let err: any; + try { await readAll(); } catch (e) { err = e; } + assert.equal(err.message, "query failed"); + }); + + it("fromLeo does not read when the start is already past the max eid", async function () { + const item = makeItem(1); + setup([item]); + const hooks = { onEnd: sandbox.stub().resolves() }; + assert.deepEqual(await readAll({ start: item.end, hooks }), []); + assert.equal(query.callCount, 0); + assert.equal(hooks.onEnd.callCount, 1); + }); + }); + + describe("limits", function () { + it("limit stops after that many events", async function () { + setup([makeItem(5)]); + const events = await readAll({ limit: 2 }); + assert.deepEqual(events.map((e) => e.payload.n), [0, 1]); + }); + + it("size stops once that many bytes were read", async function () { + setup([makeItem(5)]); + const events = await readAll({ size: 1 }); + assert.isBelow(events.length, 5); + assert.isAtLeast(events.length, 1); + }); + + it("events at or before start are skipped", async function () { + const item = makeItem(4); + setup([item]); + const events = await readAll({ start: eidPrefix(BASE) + "0000001" }); + assert.deepEqual(events.map((e) => e.payload.n), [2, 3]); + }); + + it("loops caps the number of query pages", async function () { + const pages = [[makeItem(1, { ms: BASE })], [makeItem(1, { ms: BASE + 1000 })], [makeItem(1, { ms: BASE + 2000 })]]; + setup(pages[0], { pages, event: leoEvent(pages[2]) }); + const events = await readAll({ loops: 2 }); + assert.equal(events.length, 2); + assert.equal(query.callCount >= 2, true); + }); + + it("multiple pages are read in order", async function () { + const pages = [[makeItem(2, { ms: BASE, label: "p1" })], [makeItem(2, { ms: BASE + 1000, label: "p2" })]]; + setup(pages[0], { pages, event: leoEvent(pages[1]) }); + const events = await readAll(); + assert.deepEqual(events.map((e) => `${e.payload.label}${e.payload.n}`), ["p10", "p11", "p20", "p21"]); + }); + + it("pass.get reports the stats of what was read and checkpoint writes them", async function () { + setup([makeItem(3)]); + const stream = ls.fromLeo("bot1", QUEUE, { start: ls.eventIdFromTimestamp(BASE - 1000) }); + stream.on("data", () => { }); + await new Promise((r) => stream.on("end", r)); + const stats = stream.get(); + assert.equal(stats.units, 3); + assert.equal(stats.eid, eidPrefix(BASE) + "0000002"); + assert.equal(stats.start_eid, eidPrefix(BASE) + "0000000"); + const checkpoint = sandbox.stub(ls.cron, "checkpoint").callsFake((id: string, queue: string, data: any, cb: any) => cb()); + const result = await new Promise((resolve) => stream.checkpoint((err: any, data: any) => resolve(data))); + assert.equal(checkpoint.getCall(0).args[0], "bot1"); + assert.equal(checkpoint.getCall(0).args[1], QUEUE); + assert.equal((result as any).units, 3); + assert.equal(stream.get().units, 0); + }); + + it("checkpoint with no events read skips the cron write", async function () { + setup([], { event: null }); + const stream = ls.fromLeo("bot1", QUEUE, {}); + stream.on("data", () => { }); + await new Promise((r) => stream.on("end", r)); + const checkpoint = sandbox.stub(ls.cron, "checkpoint"); + const data = await new Promise((resolve) => stream.checkpoint({ extra: 1 }, (err: any, d: any) => resolve(d))); + assert.equal(checkpoint.callCount, 0); + assert.equal((data as any).extra, 1); + }); + + it("destroying the stream stops reading", async function () { + setup([makeItem(50)]); + const stream = ls.fromLeo("bot1", QUEUE, { start: ls.eventIdFromTimestamp(BASE - 1000), buffer: 1 }); + const seen: any[] = []; + await new Promise((resolve) => { + stream.on("data", (e: any) => { + seen.push(e); + if (seen.length === 2) { stream.destroy(); resolve(); } + }); + }); + assert.isBelow(seen.length, 50); + }); + + it("getOpts returns the resolved options", function () { + setup([makeItem(1)]); + const stream = ls.fromLeo("bot1", QUEUE, { loops: 5 }); + assert.equal(stream.getOpts().loops, 5); + assert.equal(stream.getOpts().buffer, 16); + assert.equal(stream.getOpts().fast_s3_read_timeout_retry_count, 1); + assert.equal(stream.getOpts().fast_s3_read_parallel_fetch_max_bytes, 5 * 1024 * 1024); + stream.destroy(); + }); + + it("runTime sets the stop time from a duration", function () { + setup([makeItem(1)]); + sandbox.useFakeTimers({ now: 1700000000000, toFake: ["Date"] }); + const stream = ls.fromLeo("bot1", QUEUE, { runTime: { seconds: 30 } }); + assert.equal(stream.getOpts().stopTime, 1700000030000); + stream.destroy(); + }); + + it("the lambda context shortens the stop time to 80 percent of the remaining time", function () { + setup([makeItem(1)]); + sandbox.useFakeTimers({ now: 1700000000000, toFake: ["Date"] }); + configure.registry.context = { getRemainingTimeInMillis: () => 10000 }; + const stream = ls.fromLeo("bot1", QUEUE, {}); + assert.equal(stream.getOpts().stopTime, 1700000008000); + stream.destroy(); + }); + }); + + describe("s3 items", function () { + it("reads events from an S3 item through fromS3 with the byte range", async function () { + const item: any = makeItem(3, { asS3: true }); + setup([item]); + const events = await readAll(); + assert.deepEqual(events.map((e) => e.payload.n), [0, 1, 2]); + assert.equal(fromS3.callCount, 1); + assert.deepEqual(fromS3.getCall(0).args[0], { bucket: "mock-leos3", key: item.s3.key, range: `bytes=0-${item.gzipSize}` }); + }); + + it("jumps into the file at the offset that contains the start", async function () { + const item: any = makeItem(4, { asS3: true }); + item.offsets = [ + { size: 10, gzipOffset: 0, gzipSize: 10, records: 2, start: 0, end: 1, event: QUEUE, offset: 0 }, + { size: 10, gzipOffset: 10, gzipSize: 10, records: 2, start: 2, end: 3, event: QUEUE, offset: 10 } + ]; + setup([item]); + await readAll({ start: eidPrefix(BASE) + "0000001" }); + assert.equal(fromS3.getCall(0).args[0].range, `bytes=10-${item.gzipSize}`); + }); + + it("reads from a local tmp file when one exists for the range", async function () { + const fs = require("fs"); + const os = require("os"); + const path = require("path"); + const item: any = makeItem(2, { asS3: true }); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rstreams-tmp-")); + try { + ls.tmpDir = dir; + const local = path.resolve(dir, `s3/${item.event}/${item.end.replace(/\//g, "_")}-0-${item.gzipSize}.jsonl.gz`); + fs.mkdirSync(path.dirname(local), { recursive: true }); + fs.writeFileSync(local, s3Files[item.s3.key]); + setup([item]); + const events = await readAll(); + assert.equal(events.length, 2); + assert.equal(fromS3.callCount, 0); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + }); + + describe("fast_s3_read timeout retry", function () { + function timeoutError() { + const err: any = new Error("timed out"); + err.name = "TimeoutError"; + return err; + } + + function slowGood(key: string, ms = 30) { + const pass = new PassThrough(); + setTimeout(() => pass.end(s3Files[key]), ms); + return pass; + } + + function deferred() { + let resolve: () => void = () => { }; + const promise = new Promise((r) => { resolve = r; }); + return { promise, resolve }; + } + + async function flushEventLoop() { + for (let i = 0; i < 10; i++) { + await new Promise((r) => setImmediate(r)); + } + } + + function gatedGood(key: string, gate: Promise) { + const pass = new PassThrough(); + gate.then(() => pass.end(s3Files[key])); + return pass; + } + + function failing(err: any) { + const pass = new PassThrough(); + setImmediate(() => pass.emit("error", err)); + return pass; + } + + function twoFiles() { + const a: any = makeItem(2, { asS3: true, ms: BASE, key: "file-a", label: "a" }); + const b: any = makeItem(2, { asS3: true, ms: BASE + 1000, key: "file-b", label: "b" }); + setup([a, b]); + return [a, b]; + } + + function labels(events: any[]) { + return events.map((e) => `${e.payload.label}${e.payload.n}`); + } + + it("a pre-connected file that times out is connected again once", async function () { + twoFiles(); + let bCalls = 0; + fromS3.callsFake((file: any) => { + if (file.key === "file-a") { return slowGood("file-a"); } + bCalls++; + return bCalls === 1 ? failing(timeoutError()) : slowGood("file-b", 0); + }); + const events = await readAll({ fast_s3_read: true }); + assert.deepEqual(labels(events), ["a0", "a1", "b0", "b1"]); + assert.equal(bCalls, 2); + }); + + it("after the retry count is used up the file is connected when it is needed", async function () { + twoFiles(); + let bCalls = 0; + fromS3.callsFake((file: any) => { + if (file.key === "file-a") { return slowGood("file-a", 60); } + bCalls++; + return bCalls <= 2 ? failing(timeoutError()) : slowGood("file-b", 0); + }); + const events = await readAll({ fast_s3_read: true }); + assert.deepEqual(labels(events), ["a0", "a1", "b0", "b1"]); + assert.equal(bCalls, 3); + }); + + it("fast_s3_read_timeout_retry_count 0 does not reconnect early", async function () { + twoFiles(); + let bCalls = 0; + fromS3.callsFake((file: any) => { + if (file.key === "file-a") { return slowGood("file-a", 60); } + bCalls++; + return bCalls === 1 ? failing(timeoutError()) : slowGood("file-b", 0); + }); + const events = await readAll({ fast_s3_read: true, fast_s3_read_timeout_retry_count: 0 }); + assert.deepEqual(labels(events), ["a0", "a1", "b0", "b1"]); + assert.equal(bCalls, 2); + }); + + it("a non-timeout error on a pre-connected file is raised when the file is read", async function () { + twoFiles(); + fromS3.callsFake((file: any) => { + if (file.key === "file-a") { return slowGood("file-a", 60); } + const err: any = new Error("access denied"); + err.name = "AccessDenied"; + return failing(err); + }); + let err: any; + try { await readAll({ fast_s3_read: true }); } catch (e) { err = e; } + assert.equal(err.message, "access denied"); + }); + + it("without fast_s3_read files are fetched one at a time with no retry", async function () { + twoFiles(); + const order: string[] = []; + fromS3.callsFake((file: any) => { order.push(file.key); return slowGood(file.key, 0); }); + const events = await readAll(); + assert.deepEqual(labels(events), ["a0", "a1", "b0", "b1"]); + assert.deepEqual(order, ["file-a", "file-b"]); + }); + + it("fast_s3_read connects to following files while reading the first", async function () { + twoFiles(); + const connectedWhileReadingFirst: string[] = []; + const gateA = deferred(); + const bConnected = deferred(); + fromS3.callsFake((file: any) => { + connectedWhileReadingFirst.push(file.key); + if (file.key === "file-b") { bConnected.resolve(); } + return file.key === "file-a" ? gatedGood(file.key, gateA.promise) : slowGood(file.key, 0); + }); + const stream = ls.fromLeo("bot1", QUEUE, { start: ls.eventIdFromTimestamp(BASE - 1000), fast_s3_read: true }); + const first = new Promise((resolve) => stream.once("data", resolve)); + stream.on("data", () => { }); + await bConnected.promise; + assert.deepEqual(connectedWhileReadingFirst, ["file-a", "file-b"]); + gateA.resolve(); + await first; + await new Promise((r) => stream.on("end", r)); + }); + + it("fast_s3_read_parallel_fetch_max_bytes limits how many files connect ahead", async function () { + const a: any = makeItem(2, { asS3: true, ms: BASE, key: "file-a" }); + const b: any = makeItem(2, { asS3: true, ms: BASE + 1000, key: "file-b" }); + const c: any = makeItem(2, { asS3: true, ms: BASE + 2000, key: "file-c" }); + setup([a, b, c]); + const connected: string[] = []; + const gateA = deferred(); + const aConnected = deferred(); + fromS3.callsFake((file: any) => { + connected.push(file.key); + if (file.key === "file-a") { aConnected.resolve(); } + return file.key === "file-a" ? gatedGood(file.key, gateA.promise) : slowGood(file.key, 0); + }); + const stream = ls.fromLeo("bot1", QUEUE, { start: ls.eventIdFromTimestamp(BASE - 1000), fast_s3_read: true, fast_s3_read_parallel_fetch_max_bytes: 1 }); + stream.on("data", () => { }); + await aConnected.promise; + await flushEventLoop(); + assert.deepEqual(connected, ["file-a"]); + const ended = new Promise((r) => stream.on("end", r)); + gateA.resolve(); + await ended; + assert.deepEqual(connected, ["file-a", "file-b", "file-c"]); + }); + }); + + describe("read hooks", function () { + function hookLog() { + const log: string[] = []; + const hooks = { + onStart: (info: any) => { log.push(`start:${info.queue}`); }, + onGetEvents: (items: any[]) => { log.push(`getEvents:${items.length}`); }, + onBatchStart: (items: any[]) => { log.push(`batchStart:${items.length}`); }, + onRecordStart: (item: any, i: number) => { log.push(`recordStart:${i}`); }, + onRecordEnd: (item: any, i: number) => { log.push(`recordEnd:${i}`); }, + onBatchEnd: async (items: any[]) => { log.push(`batchEnd:${items.length}`); }, + onEnd: async () => { log.push("end"); } + }; + return { log, hooks }; + } + + it("hooks run in order, with onGetEvents also running for the prefetched page that is never read", async function () { + setup([makeItem(2), makeItem(1, { ms: BASE + 1000 })]); + const { log, hooks } = hookLog(); + await readAll({ hooks }); + assert.deepEqual(log, [ + "start:queue1", "getEvents:2", "getEvents:0", "batchStart:2", + "recordStart:0", "recordEnd:0", "recordStart:1", "recordEnd:1", + "batchEnd:2", "end" + ]); + }); + + it("onStart receives the queue and the start eid", async function () { + setup([makeItem(1)]); + const onStart = sandbox.stub(); + const start = ls.eventIdFromTimestamp(BASE - 1000); + await readAll({ start, hooks: { onStart } }); + assert.deepEqual(onStart.getCall(0).args[0], { queue: QUEUE, eid: start + " " }); + }); + + it("onGetEvents may replace the items returned by the query", async function () { + const real = makeItem(2, { label: "real" }); + setup([makeItem(2, { label: "ignored" })], { event: leoEvent([real]) }); + const events = await readAll({ hooks: { onGetEvents: (items: any[]) => items.length ? [real] : undefined } }); + assert.deepEqual(events.map((e) => e.payload.label), ["real", "real"]); + }); + + it("onBatchStart may replace the items to read", async function () { + const first = makeItem(1, { ms: BASE, label: "first" }); + const second = makeItem(1, { ms: BASE + 1000, label: "second" }); + setup([first, second]); + const events = await readAll({ hooks: { onBatchStart: (items: any[]) => items.length ? [items[1]] : undefined } }); + assert.deepEqual(events.map((e) => e.payload.label), ["second"]); + }); + + it("onBatchEnd and onEnd are awaited before the stream ends", async function () { + setup([makeItem(1)]); + const order: string[] = []; + let ended = false; + const stream = ls.fromLeo("bot1", QUEUE, { + start: ls.eventIdFromTimestamp(BASE - 1000), + hooks: { + onBatchEnd: async () => { await new Promise((r) => setTimeout(r, 20)); order.push("batchEnd"); }, + onEnd: async () => { await new Promise((r) => setTimeout(r, 20)); order.push("end"); } + } + }); + stream.on("data", () => { }); + stream.on("end", () => { ended = true; order.push("streamEnd"); }); + await new Promise((r) => stream.on("end", r)); + assert.isTrue(ended); + assert.deepEqual(order, ["batchEnd", "end", "streamEnd"]); + }); + + it("onEnd is not called when there is nothing to read", async function () { + setup([], { event: null }); + const onEnd = sandbox.stub().resolves(); + await readAll({ hooks: { onEnd } }); + assert.equal(onEnd.callCount, 0); + }); + + it("getS3Stream supplies the stream for an S3 item", async function () { + const item: any = makeItem(2, { asS3: true }); + setup([item]); + const stream: any = new PassThrough(); + stream.idOffset = 0; + stream.end(s3Files[item.s3.key]); + const getS3Stream = sandbox.stub().returns(stream); + const events = await readAll({ hooks: { getS3Stream } }); + assert.equal(events.length, 2); + assert.equal(fromS3.callCount, 0); + assert.equal(getS3Stream.getCall(0).args[1], 0); + }); + + it("createS3Stream supplies a wrapper whose get returns the stream", async function () { + const item: any = makeItem(2, { asS3: true }); + setup([item]); + const createS3Stream = sandbox.stub().callsFake(() => { + const stream: any = new PassThrough(); + stream.idOffset = 0; + stream.end(s3Files[item.s3.key]); + return { get: () => stream }; + }); + const events = await readAll({ hooks: { createS3Stream } }); + assert.equal(events.length, 2); + assert.equal(fromS3.callCount, 0); + }); + + it("createSplitParseStream replaces the line parser", async function () { + const item = makeItem(2); + setup([item]); + const createSplitParseStream = sandbox.stub().callsFake(() => { + const { Transform } = require("stream"); + return new Transform({ + objectMode: true, + transform(chunk: any, enc: any, cb: any) { + chunk.toString().split("\n").filter(Boolean).forEach((l: string) => this.push({ ...JSON.parse(l), custom: true })); + cb(); + } + }); + }); + const events = await readAll({ hooks: { createSplitParseStream } }); + assert.isTrue(events.every((e) => e.custom === true)); + assert.equal(events.length, 2); + }); + }); + + describe("snapshot and archive", function () { + it("starting at z/ with a snapshot reads the _snapshot queue first and the page prefetched at the switch still uses the snapshot queue", async function () { + const snapRaw: any = makeItem(1, { label: "snap", ms: BASE, offset: 1 }); + const snapItem = { ...snapRaw, start: snapRaw.start.replace(/^z\//, ""), end: snapRaw.end.replace(/^z\//, "") }; + const realItem: any = makeItem(1, { label: "real", ms: BASE + 5000 }); + batchGet.callsFake((params: any, cb: any) => cb(null, { + Responses: { + [EVENT_TABLE]: [{ event: QUEUE, v: 2, max_eid: realItem.end, snapshot: { start: "_snapshot/" + snapItem.start.replace(/-\d+$/, "-0000000"), next: "z/2022/03/14/22/38/1647297513618-0000000" } }], + [CRON_TABLE]: [] + }, + UnprocessedKeys: {} + })); + query.onFirstCall().callsFake((p: any, cb: any) => cb(null, { Items: [snapItem], Count: 1 })); + query.onSecondCall().callsFake((p: any, cb: any) => cb(null, { Items: [], Count: 0 })); + query.onThirdCall().callsFake((p: any, cb: any) => cb(null, { Items: [realItem], Count: 1 })); + query.callsFake((p: any, cb: any) => cb(null, { Items: [], Count: 0 })); + const events = await readAll({ start: "z/" }); + const first = query.getCall(0).args[0].ExpressionAttributeValues; + assert.equal(first[":event"], "queue1/_snapshot"); + assert.equal(first[":start"], snapItem.start.replace(/-\d+$/, "-0000000")); + assert.match(first[":maxkey"], /^2022\/03\/14\/22\/38\/99999999999$/); + assert.deepEqual(events.map((e) => e.payload.label), ["snap", "real"]); + assert.equal(events[0].eid, "_snapshot/" + snapItem.start); + assert.equal(query.getCall(2).args[0].ExpressionAttributeValues[":event"], "queue1/_snapshot"); + assert.equal(query.getCall(3).args[0].ExpressionAttributeValues[":event"], "queue1"); + }); + + it("an archive end after the start reads the _archive queue first", async function () { + const archived: any = makeItem(1, { label: "old", ms: BASE }); + archived.archive = true; + archived.gzip = zlib.gzipSync(JSON.stringify({ id: "u", event: QUEUE, payload: { label: "old", n: 0 }, eid: archived.end }) + "\n"); + const item = makeItem(1, { label: "new", ms: BASE + 5000 }); + batchGet.callsFake((params: any, cb: any) => cb(null, { + Responses: { [EVENT_TABLE]: [{ event: QUEUE, v: 2, max_eid: item.end, archive: { end: archived.end } }], [CRON_TABLE]: [] }, + UnprocessedKeys: {} + })); + query.onFirstCall().callsFake((p: any, cb: any) => cb(null, { Items: [archived], Count: 1 })); + query.onSecondCall().callsFake((p: any, cb: any) => cb(null, { Items: [], Count: 0 })); + query.onThirdCall().callsFake((p: any, cb: any) => cb(null, { Items: [item], Count: 1 })); + query.callsFake((p: any, cb: any) => cb(null, { Items: [], Count: 0 })); + const events = await readAll({ start: ls.eventIdFromTimestamp(BASE - 1000) }); + assert.equal(query.getCall(0).args[0].ExpressionAttributeValues[":event"], "queue1/_archive"); + assert.deepEqual(events.map((e) => e.payload.label), ["old", "new"]); + }); + }); + + describe("parsers", function () { + it("an unparsable line ends the stream with a push-after-EOF error", async function () { + const item: any = makeItem(2); + const text = JSON.stringify({ id: "u", event: QUEUE, payload: { n: 0 } }) + "\nnot json\n" + JSON.stringify({ id: "u", event: QUEUE, payload: { n: 1 } }) + "\n"; + item.gzip = zlib.gzipSync(text); + setup([item]); + let err: any; + try { await readAll(); } catch (e) { err = e; } + assert.equal(err.message, "stream.push() after EOF"); + }); + + it("setBaseEid commands change the eid prefix and restart the offset", async function () { + const item: any = makeItem(2); + const text = JSON.stringify({ _cmd: "setBaseEid", eid: "z/2030/01/01/00/00/1893456000000-0000010" }) + "\n" + JSON.stringify({ id: "u", event: QUEUE, payload: { n: 0 } }) + "\n"; + item.gzip = zlib.gzipSync(text); + item.end = "z/2031/01/01/00/00/1924992000000-0000000"; + setup([item]); + const events = await readAll(); + assert.equal(events[0].eid, "z/2030/01/01/00/00/1893456000000-0000010"); + }); + }); +}); diff --git a/test/lib.stream.leo-stream.write.utest.ts b/test/lib.stream.leo-stream.write.utest.ts new file mode 100644 index 0000000..cd8da77 --- /dev/null +++ b/test/lib.stream.leo-stream.write.utest.ts @@ -0,0 +1,475 @@ +import sinon from "sinon"; +import { assert } from "chai"; +import zlib from "zlib"; +import { Writable } from "stream"; +import utilFn from "../lib/stream/leo-stream"; + +describe("lib/stream/leo-stream.js write", function () { + const sandbox = sinon.createSandbox(); + let ls: any; + let configure: any; + let putRecords: sinon.SinonStub; + let putRecordBatch: sinon.SinonStub; + let toS3Uploads: { bucket: string, key: string, chunks: Buffer[] }[]; + + function build(extra: any = {}) { + configure = { + onUpdate: () => { }, + validate: () => { }, + resources: { LeoCron: "mock-LeoCron", LeoEvent: "mock-LeoEvent", LeoStream: "mock-LeoStream", LeoS3: "mock-leos3", LeoSettings: "mock-LeoSettings" }, + aws: { region: "us-west-2" }, + stream: "mock-stream", + bus: { firehose: "mock-firehose" }, + ...extra + }; + ls = utilFn(configure); + putRecords = sandbox.stub(ls.kinesis, "putRecords").callsFake((params: any, cb: any) => cb(null, { FailedRecordCount: 0, Records: params.Records.map(() => ({})) })); + putRecordBatch = sandbox.stub(ls.firehose, "putRecordBatch").callsFake((params: any, cb: any) => cb(null, { FailedPutCount: 0 })); + toS3Uploads = []; + sandbox.stub(ls, "toS3").callsFake((bucket: string, key: string) => { + const upload = { bucket, key, chunks: [] as Buffer[] }; + toS3Uploads.push(upload); + return new Writable({ + write(chunk, enc, cb) { upload.chunks.push(Buffer.from(chunk)); cb(); } + }); + }); + return ls; + } + + beforeEach(() => { + delete (process as any).__config; + build(); + }); + + afterEach(() => { + sandbox.restore(); + }); + + function evt(n: number, extra: any = {}) { + return { id: "bot1", event: "queue1", payload: { n }, event_source_timestamp: 1000 + n, timestamp: 2000 + n, ...extra }; + } + + function decodeGzipRecord(data: Buffer) { + return zlib.gunzipSync(data).toString().trim().split("\n").map((l) => JSON.parse(l)); + } + + function write(events: any[], stream: any) { + const out: any[] = []; + return ls.pipeAsync( + ls.eventstream.readArray(events), + stream, + ls.write({ hasCommands: false }, (o, done) => { out.push(o); done(); }) + ).then(() => out); + } + + function sink(events: any[], stream: any) { + return ls.pipeAsync(ls.eventstream.readArray(events), stream); + } + + describe("toLeo to kinesis", function () { + it("toLeo sends one gzipped putRecords with the default partition and hash key", async function () { + await write([evt(1), evt(2)], ls.toLeo("bot1", {})); + assert.equal(putRecords.callCount, 1); + const params = putRecords.getCall(0).args[0]; + assert.equal(params.StreamName, "mock-stream"); + assert.equal(params.Records.length, 1); + assert.equal(params.Records[0].PartitionKey, "0"); + assert.equal(params.Records[0].ExplicitHashKey, "0"); + const events = decodeGzipRecord(params.Records[0].Data); + assert.deepEqual(events.map((e) => e.payload), [{ n: 1 }, { n: 2 }]); + assert.deepEqual(events.map((e) => e.eid), [0, 1]); + }); + + it("toLeo emits a checkpoint command carrying the correlations after the write", async function () { + const correlation = { source: "src", start: "z/1", end: "z/2", units: 2 }; + const out = await write([evt(1, { correlation_id: correlation })], ls.toLeo("bot1", {})); + assert.equal(out.length, 1); + assert.equal(out[0].__cmd, "checkpoint"); + assert.equal(out[0].id, "bot1"); + assert.deepEqual(out[0].correlations, [{ + src: { source: "src", start: "z/1", end: "z/2", records: 2, source_timestamp: 1001, timestamp: 2001 } + }]); + }); + + it("toLeo does not call kinesis when nothing is written", async function () { + await write([], ls.toLeo("bot1", {})); + assert.equal(putRecords.callCount, 0); + }); + }); + + describe("useQueuePartition", function () { + it("uses the queue name as the partition key and drops the explicit hash key", async function () { + await write([evt(1)], ls.toLeo("bot1", { useQueuePartition: true })); + const record = putRecords.getCall(0).args[0].Records[0]; + assert.equal(record.PartitionKey, "queue1"); + assert.isUndefined(record.ExplicitHashKey); + }); + + it("uses the first queue seen when events target several queues", async function () { + await write([evt(1, { event: "qa" }), evt(2, { event: "qb" })], ls.toLeo("bot1", { useQueuePartition: true })); + assert.equal(putRecords.getCall(0).args[0].Records[0].PartitionKey, "qa"); + }); + + it("partitions per record on queue only when the option is set", async function () { + await write([evt(1, { event: "qa" })], ls.toLeo("bot1", {})); + assert.equal(putRecords.getCall(0).args[0].Records[0].PartitionKey, "0"); + }); + + it("uses the event name qa as partition key for a single event with the option set, since every event adds queue stats", async function () { + await write([evt(1, { event: "qa" })], ls.toLeo("bot1", { useQueuePartition: true })); + assert.equal(putRecords.getCall(0).args[0].Records[0].PartitionKey, "qa"); + }); + }); + + describe("partitionHashKey", function () { + it("sends partitionHashKey as the string explicit hash key", async function () { + await write([evt(1)], ls.toLeo("bot1", { partitionHashKey: 12345 })); + assert.strictEqual(putRecords.getCall(0).args[0].Records[0].ExplicitHashKey, "12345"); + }); + + it("a zero partitionHashKey stays the string 0", async function () { + await write([evt(1)], ls.toLeo("bot1", { partitionHashKey: 0 })); + assert.strictEqual(putRecords.getCall(0).args[0].Records[0].ExplicitHashKey, "0"); + }); + + it("useQueuePartition takes precedence over partitionHashKey", async function () { + await write([evt(1)], ls.toLeo("bot1", { partitionHashKey: 99, useQueuePartition: true })); + const record = putRecords.getCall(0).args[0].Records[0]; + assert.isUndefined(record.ExplicitHashKey); + assert.equal(record.PartitionKey, "queue1"); + }); + }); + + describe("backoff", function () { + const fast = { backoff: { initialDelay: 1, maxDelay: 2 } }; + + it("retries the whole batch when putRecords errors", async function () { + putRecords.onFirstCall().callsFake((p: any, cb: any) => cb(new Error("throttled"))); + await write([evt(1)], ls.toLeo("bot1", fast)); + assert.equal(putRecords.callCount, 2); + assert.deepEqual(putRecords.getCall(0).args[0].Records.length, putRecords.getCall(1).args[0].Records.length); + }); + + it("retries only the records that failed", async function () { + sandbox.restore(); + build(); + putRecords.onFirstCall().callsFake((p: any, cb: any) => cb(null, { FailedRecordCount: 1, Records: [{ ErrorCode: "ProvisionedThroughputExceededException" }] })); + await write([evt(1)], ls.toLeo("bot1", fast)); + assert.equal(putRecords.callCount, 2); + assert.equal(putRecords.getCall(1).args[0].Records.length, 1); + }); + + it("fails the stream after ten failed attempts", async function () { + putRecords.callsFake((p: any, cb: any) => cb(new Error("always"))); + let err: any; + try { await write([evt(1)], ls.toLeo("bot1", fast)); } catch (e) { err = e; } + assert.exists(err); + assert.isAtLeast(putRecords.callCount, 10); + }); + }); + + describe("gzipFirehose", function () { + const fh = { firehose: true, time: { milliseconds: 5 } }; + + it("sends raw newline-delimited json to firehose by default", async function () { + await write([evt(1), evt(2)], ls.toLeo("bot1", fh)); + assert.equal(putRecords.callCount, 0); + const params = putRecordBatch.getCall(0).args[0]; + assert.equal(params.DeliveryStreamName, "mock-firehose"); + assert.equal(params.Records.length, 1); + const lines = params.Records[0].Data.toString().trim().split("\n").map((l: string) => JSON.parse(l)); + assert.deepEqual(lines.map((e: any) => e.payload), [{ n: 1 }, { n: 2 }]); + }); + + it("gzipFirehose sends one base64 gzip chunk per line", async function () { + await write([evt(1), evt(2)], ls.toLeo("bot1", { ...fh, gzipFirehose: true })); + const data = putRecordBatch.getCall(0).args[0].Records[0].Data.toString(); + assert.isTrue(data.endsWith("\n")); + const lines = data.trim().split("\n"); + assert.equal(lines.length, 1); + const events = decodeGzipRecord(Buffer.from(lines[0], "base64")); + assert.deepEqual(events.map((e) => e.payload), [{ n: 1 }, { n: 2 }]); + }); + + it("firehose retries only the failed records of a partial failure", async function () { + putRecordBatch.onFirstCall().callsFake((p: any, cb: any) => cb(null, { FailedPutCount: 1, RequestResponses: [{ ErrorCode: "ServiceUnavailableException" }] })); + await write([evt(1)], ls.toLeo("bot1", { ...fh, backoff: { initialDelay: 1, maxDelay: 2 } })); + assert.equal(putRecordBatch.callCount, 2); + }); + + it("firehose retries after an error", async function () { + putRecordBatch.onFirstCall().callsFake((p: any, cb: any) => cb(new Error("down"))); + await write([evt(1)], ls.toLeo("bot1", { ...fh, backoff: { initialDelay: 1, maxDelay: 2 } })); + assert.equal(putRecordBatch.callCount, 2); + }); + }); + + describe("useS3 and s3Opts", function () { + it("load with useS3 uploads to bus///z/ and sends the s3 pointer to kinesis", async function () { + await sink([{ n: 1 }, { n: 2 }], ls.load("bot1", "queue1", { useS3: true })); + assert.isAtLeast(toS3Uploads.length, 1); + const upload = toS3Uploads.find((u) => u.chunks.length > 0)!; + assert.equal(upload.bucket, "mock-leos3"); + assert.match(upload.key, /^bus\/queue1\/bot1\/z\/\d{4}\/\d\d\/\d\d\/\d\d\/\d\d\/\d+-0000001-[0-9a-f-]{36}\.gz$/); + const stored = decodeGzipRecord(Buffer.concat(upload.chunks)); + assert.deepEqual(stored.map((e) => e.payload), [{ n: 1 }, { n: 2 }]); + + const pointers = putRecords.getCalls().flatMap((c: any) => c.args[0].Records.map((r: any) => decodeGzipRecord(r.Data)[0])); + assert.equal(pointers.length, 1); + assert.equal(pointers[0].s3.bucket, "mock-leos3"); + assert.equal(pointers[0].s3.key, upload.key); + assert.equal(pointers[0].records, 2); + }); + + it("prefix replaces the bot id in the S3 key", async function () { + await sink([{ n: 1 }], ls.load("bot1", "queue1", { useS3: true, prefix: "custom" })); + assert.match(toS3Uploads[0].key, /^bus\/queue1\/custom\/z\//); + }); + + it("s3Opts can switch the file to the archive layout", async function () { + await sink([{ n: 1 }], ls.load("bot1", "queue1", { useS3: true, s3Opts: { archive: true } })); + assert.match(toS3Uploads[0].key, /^bus\/_archive\/q=queue1\/dt=\d{4}-\d\d-\d\d\/\d+-0000001-[0-9a-f-]{36}\.gz$/); + }); + + it("s3Opts values override the defaults passed by load", async function () { + await sink([{ n: 1 }], ls.load("bot1", "queue1", { useS3: true, s3Opts: { prefix: "from-s3-opts" } })); + assert.match(toS3Uploads[0].key, /^bus\/queue1\/from-s3-opts\/z\//); + }); + + it("toLeo with useS3 forwards an existing s3 pointer record to kinesis", async function () { + await sink([{ id: "bot1", event: "queue1", payload: { n: 1 }, s3: { bucket: "b", key: "k" }, records: 1, size: 10, gzipSize: 10, start: 0, end: 0, stats: {}, correlations: [], offsets: [] }], ls.toLeo("bot1", { useS3: true })); + assert.equal(putRecords.callCount, 1); + const sent = decodeGzipRecord(putRecords.getCall(0).args[0].Records[0].Data); + assert.equal(sent[0].s3.key, "k"); + }); + }); + + describe("automatic S3 switch for large events", function () { + const THRESHOLD = 1024 * 200 * 3; + + function sizedEvent(totalBytes: number) { + const base = evt(1, { payload: { d: "" } }); + const baseSize = Buffer.byteLength(JSON.stringify(base)); + return evt(1, { payload: { d: "x".repeat(totalBytes - baseSize) } }); + } + + it("an event below the threshold stays inline", async function () { + const e = sizedEvent(THRESHOLD - 1000); + await write([e], ls.toLeo("bot1", {})); + assert.equal(toS3Uploads.length, 0); + assert.equal(decodeGzipRecord(putRecords.getCall(0).args[0].Records[0].Data)[0].payload.d.length, e.payload.d.length); + }); + + it("an event of exactly 614400 bytes is not moved to S3 but is replaced by an error event once the eid is added", async function () { + const e = sizedEvent(THRESHOLD); + assert.equal(Buffer.byteLength(JSON.stringify(e)), THRESHOLD); + await write([e], ls.toLeo("bot1", {})); + assert.equal(toS3Uploads.length, 0); + const sent = decodeGzipRecord(putRecords.getCall(0).args[0].Records[0].Data)[0]; + assert.match(sent.payload.error, /^Event was too large \d+ > 614400$/); + assert.equal(sent.payload.maxSize, THRESHOLD); + assert.deepEqual([sent.event.type, sent.event.id], ["queue", "bot.bot1.error"]); + }); + + it("an event of 614401 bytes is moved to S3 and replaced by a pointer", async function () { + const e = sizedEvent(THRESHOLD + 1); + await write([e], ls.toLeo("bot1", {})); + const upload = toS3Uploads.find((u) => u.chunks.length > 0)!; + assert.match(upload.key, /^bus\/queue1\/bot1\/z\//); + const stored = decodeGzipRecord(Buffer.concat(upload.chunks)); + assert.equal(stored[0].payload.d.length, e.payload.d.length); + const sent = putRecords.getCalls().flatMap((c: any) => c.args[0].Records.map((r: any) => decodeGzipRecord(r.Data)[0])); + assert.equal(sent.length, 1); + assert.equal(sent[0].s3.key, upload.key); + assert.isUndefined(sent[0].payload); + }); + + it("the S3 prefix defaults to the event id and honors opts.prefix", async function () { + await write([sizedEvent(THRESHOLD + 1)], ls.toLeo("bot1", { prefix: "pre" })); + assert.match(toS3Uploads[0].key, /^bus\/queue1\/pre\/z\//); + }); + + it("load applies the same threshold before toLeo", async function () { + const e = sizedEvent(THRESHOLD + 1); + await sink([e], ls.load("bot1", "queue1", {})); + assert.isAtLeast(toS3Uploads.length, 1); + assert.match(toS3Uploads[0].key, /^bus\/queue1\/bot1\/z\//); + }); + + it("load with firehose still moves large events to S3 through toLeo", async function () { + const e = sizedEvent(THRESHOLD + 1); + await sink([e], ls.load("bot1", "queue1", { firehose: true })); + assert.equal(toS3Uploads.filter((u) => u.chunks.length > 0).length, 1); + }); + }); + + describe("onCheckpoint", function () { + function update(id: string, correlations: any[]) { + return { id, correlations }; + } + function corr(queue: string, start: string, end: string, records: number) { + return { [queue]: { start, end, records, source_timestamp: 10, timestamp: 20 } }; + } + + let checkpoint: sinon.SinonStub; + beforeEach(() => { + checkpoint = sandbox.stub(ls.cron, "checkpoint").callsFake((id: string, event: string, cp: any, cb: any) => cb()); + }); + + it("a sync hook receives the checkpointed eid per bot and queue ref", async function () { + const hook = sandbox.stub(); + await sink([update("bot1", [corr("queue1", "z/1", "z/5", 5)])], ls.toCheckpoint({ onCheckpoint: hook })); + assert.equal(hook.callCount, 1); + assert.deepEqual(hook.getCall(0).args[0], { error: null, checkpoints: { bot1: { "queue:queue1": "z/5" } } }); + }); + + it("an async hook is awaited before the stream finishes", async function () { + let finished = false; + const hook = async () => { await new Promise((r) => setTimeout(r, 20)); finished = true; }; + await sink([update("bot1", [corr("queue1", "z/1", "z/5", 5)])], ls.toCheckpoint({ onCheckpoint: hook })); + assert.isTrue(finished); + }); + + it("the hook sees the checkpoint error and the stream fails with it", async function () { + checkpoint.callsFake((id: string, event: string, cp: any, cb: any) => cb("Stale Checkpoint")); + const hook = sandbox.stub(); + let err: any; + try { + await sink([update("bot1", [corr("queue1", "z/1", "z/5", 5)])], ls.toCheckpoint({ onCheckpoint: hook })); + } catch (e) { err = e; } + assert.equal(hook.getCall(0).args[0].error, "Stale Checkpoint"); + assert.exists(err); + }); + + it("a hook that rejects fails the stream", async function () { + const boom = new Error("hook failed"); + let err: any; + try { + await sink([update("bot1", [corr("queue1", "z/1", "z/5", 5)])], ls.toCheckpoint({ onCheckpoint: async () => { throw boom; } })); + } catch (e) { err = e; } + assert.equal(err, boom); + }); + + it("without a hook the checkpoint still runs", async function () { + await sink([update("bot1", [corr("queue1", "z/1", "z/5", 5)])], ls.toCheckpoint({})); + assert.equal(checkpoint.callCount, 1); + assert.deepEqual(checkpoint.getCall(0).args.slice(0, 3), ["bot1", "queue1", { + eid: "z/5", records: 5, source_timestamp: 10, ended_timestamp: 20, started_timestamp: 20, force: undefined + }]); + }); + + it("force is carried on the checkpoint", async function () { + await sink([update("bot1", [corr("queue1", "z/1", "z/5", 5)])], ls.toCheckpoint({ force: true })); + assert.isTrue(checkpoint.getCall(0).args[2].force); + }); + + it("correlations for the same queue are merged and the last end wins", async function () { + await sink([ + update("bot1", [corr("queue1", "z/1", "z/5", 5)]), + update("bot1", [corr("queue1", "z/6", "z/9", 4)]) + ], ls.toCheckpoint({})); + assert.equal(checkpoint.callCount, 1); + const cp = checkpoint.getCall(0).args[2]; + assert.equal(cp.eid, "z/9"); + assert.equal(cp.records, 9); + }); + + it("a correlation_id on a plain event is converted to a correlation", async function () { + await sink([{ id: "bot1", event: "queue2", payload: 1, event_source_timestamp: 5, timestamp: 8, correlation_id: { source: "queue1", start: "z/1", end: "z/3", units: 3 } }], ls.toCheckpoint({})); + assert.deepEqual(checkpoint.getCall(0).args.slice(0, 3), ["bot1", "queue1", { + eid: "z/3", records: 3, source_timestamp: 5, ended_timestamp: 8, started_timestamp: 8, force: undefined + }]); + }); + + it("the checkpoint is written for the last bot and queue seen for every task", async function () { + await sink([update("bot1", [corr("queue1", "z/1", "z/5", 5), corr("queue2", "z/1", "z/7", 7)])], ls.toCheckpoint({})); + assert.equal(checkpoint.callCount, 2); + assert.deepEqual(checkpoint.getCalls().map((c: any) => c.args[1]), ["queue2", "queue2"]); + assert.deepEqual(checkpoint.getCalls().map((c: any) => c.args[2].eid), ["z/7", "z/7"]); + }); + + it("toCheckpoint get returns the pending checkpoint for a bot and queue", async function () { + const stream = ls.toCheckpoint({ records: Number.POSITIVE_INFINITY, time: { days: 1 } }); + await new Promise((r) => stream.write(update("bot1", [corr("queue1", "z/1", "z/5", 5)]), r)); + try { + assert.equal(stream.get("bot1", "queue1").eid, "z/5"); + assert.isUndefined(stream.get("bot1")); + assert.isUndefined(stream.get("bot9", "queue1")); + } finally { + await new Promise((r) => stream.end(r)); + } + }); + }); + + describe("load", function () { + it("load throws when the configuration is invalid", function () { + build({ validate: undefined }); + assert.throws(() => ls.load("bot1", "queue1", {}), "Invalid Settings: Missing config"); + }); + + it("load wraps plain objects as payloads and fills id, event and timestamps", async function () { + sandbox.useFakeTimers({ now: 1700000000000, toFake: ["Date"] }); + await sink([{ n: 1 }], ls.load("bot1", "queue1", {})); + const sent = decodeGzipRecord(putRecords.getCall(0).args[0].Records[0].Data)[0]; + assert.equal(sent.id, "bot1"); + assert.equal(sent.event, "queue1"); + assert.deepEqual(sent.payload, { n: 1 }); + assert.equal(sent.event_source_timestamp, 1700000000000); + assert.equal(sent.timestamp, 1700000000000); + }); + + it("load keeps objects that already carry a payload and their id, event and timestamps", async function () { + await sink([{ id: "other", event: "q9", payload: { a: 1 }, event_source_timestamp: 5, timestamp: 6 }], ls.load("bot1", "queue1", {})); + const sent = decodeGzipRecord(putRecords.getCall(0).args[0].Records[0].Data)[0]; + assert.deepEqual([sent.id, sent.event, sent.event_source_timestamp, sent.timestamp], ["other", "q9", 5, 6]); + }); + + it("load treats an object with a correlation_id as an event", async function () { + sandbox.stub(ls.cron, "checkpoint").callsFake((id: string, event: string, cp: any, cb: any) => cb()); + await sink([{ correlation_id: { source: "s", start: "z/1" }, payload: { a: 1 } }], ls.load("bot1", "queue1", {})); + const sent = decodeGzipRecord(putRecords.getCall(0).args[0].Records[0].Data)[0]; + assert.equal(sent.correlation_id.source, "s"); + }); + + it("load with autoDetectPayload false always wraps the object", async function () { + await sink([{ payload: { a: 1 } }], ls.load("bot1", "queue1", { autoDetectPayload: false })); + const sent = decodeGzipRecord(putRecords.getCall(0).args[0].Records[0].Data)[0]; + assert.deepEqual(sent.payload, { payload: { a: 1 } }); + }); + + it("load with autoCheckpoint false does not checkpoint", async function () { + const checkpoint = sandbox.stub(ls.cron, "checkpoint").callsFake((id: string, event: string, cp: any, cb: any) => cb()); + await sink([{ id: "bot1", event: "queue1", payload: 1, correlation_id: { source: "s", start: "z/1", end: "z/1" } }], ls.load("bot1", "queue1", { autoCheckpoint: false })); + assert.equal(checkpoint.callCount, 0); + }); + + it("load checkpoints the correlation sources by default", async function () { + const checkpoint = sandbox.stub(ls.cron, "checkpoint").callsFake((id: string, event: string, cp: any, cb: any) => cb()); + await sink([{ id: "bot1", event: "queue1", payload: 1, correlation_id: { source: "src", start: "z/1", end: "z/1" } }], ls.load("bot1", "queue1", {})); + assert.equal(checkpoint.callCount, 1); + assert.equal(checkpoint.getCall(0).args[0], "bot1"); + assert.equal(checkpoint.getCall(0).args[1], "src"); + assert.equal(checkpoint.getCall(0).args[2].eid, "z/1"); + }); + + it("load calls the onCheckpoint hook", async function () { + sandbox.stub(ls.cron, "checkpoint").callsFake((id: string, event: string, cp: any, cb: any) => cb()); + const hook = sandbox.stub(); + await sink([{ id: "bot1", event: "queue1", payload: 1, correlation_id: { source: "src", start: "z/1", end: "z/1" } }], ls.load("bot1", "queue1", { onCheckpoint: hook })); + assert.equal(hook.callCount, 1); + assert.deepEqual(hook.getCall(0).args[0].checkpoints, { bot1: { "queue:src": "z/1" } }); + }); + + it("putEvent writes the object through load and calls back", async function () { + const err = await new Promise((resolve) => ls.putEvent("bot1", "queue1", { n: 7 }, {}, resolve)); + assert.isUndefined(err); + const sent = decodeGzipRecord(putRecords.getCall(0).args[0].Records[0].Data)[0]; + assert.deepEqual(sent.payload, { n: 7 }); + }); + + it("putEvent accepts the callback in place of options", async function () { + await new Promise((resolve) => ls.putEvent("bot1", "queue1", { n: 8 }, resolve)); + assert.equal(putRecords.callCount, 1); + }); + }); +}); diff --git a/test/lib.streams.eid.utest.ts b/test/lib.streams.eid.utest.ts new file mode 100644 index 0000000..d4aa26a --- /dev/null +++ b/test/lib.streams.eid.utest.ts @@ -0,0 +1,111 @@ +import { assert } from "chai"; +import streams from "../lib/streams"; + +describe("lib/streams.js event ids", function () { + const cases: [number, string, number | null, string][] = [ + [1647297513618, "full", null, "z/2022/03/14/22/38/1647297513618"], + [1641022663000, "full", 35, "z/2022/01/01/07/37/1641022663000-0000035"], + [0, "year", null, "z/1970/"], + [1, "month", null, "z/1970/01/"], + [946684800000, "date", null, "z/2000/01/01/"], + [1709251199999, "hour", null, "z/2024/02/29/23/"], + [1735689600000, "minute", null, "z/2025/01/01/00/00/"], + [1767225599999, "second", null, "z/2025/12/31/23/59/1767225599000"], + [1700000000000, "millisecond", null, "z/2023/11/14/22/13/1700000000000"], + [1234567890123, "millisecond", 12, "z/2009/02/13/23/31/1234567890123-0000012"], + [1583020800000, "full", null, "z/2020/03/01/00/00/1583020800000"], + [1678886400000, "full", 35, "z/2023/03/15/13/20/1678886400000-0000035"], + [1704067199000, "year", null, "z/2023/"], + [1893456000000, "month", null, "z/2030/01/"], + [1650907126722, "date", null, "z/2022/04/25/"], + [1477958400000, "hour", null, "z/2016/11/01/00/"], + [1456790400000, "minute", null, "z/2016/03/01/00/00/"], + [1609459200000, "second", null, "z/2021/01/01/00/00/1609459200000"], + [1620000000000, "millisecond", null, "z/2021/05/03/00/00/1620000000000"], + [1999999999999, "millisecond", 12, "z/2033/05/18/03/33/1999999999999-0000012"] + ]; + + describe("eventIdFromTimestamp", function () { + cases.forEach(([timestamp, granularity, record, expected], i) => { + it(`eventIdFromTimestamp case ${i + 1}: ${timestamp} ${granularity}${record == null ? "" : " record " + record}`, function () { + assert.equal(streams.eventIdFromTimestamp(timestamp, granularity as any, record as any), expected); + }); + }); + + it("eventIdFromTimestamp defaults to full granularity", function () { + assert.equal(streams.eventIdFromTimestamp(1647297513618), "z/2022/03/14/22/38/1647297513618"); + }); + + it("eventIdFromTimestamp falls back to full for an unknown granularity", function () { + assert.equal(streams.eventIdFromTimestamp(1647297513618, "decade" as any), "z/2022/03/14/22/38/1647297513618"); + }); + + it("eventIdFromTimestamp ignores the record for coarse granularities", function () { + assert.equal(streams.eventIdFromTimestamp(1647297513618, "hour", 5 as any), "z/2022/03/14/22/"); + assert.equal(streams.eventIdFromTimestamp(1647297513618, "second", 5 as any), "z/2022/03/14/22/38/1647297513000"); + }); + + it("eventIdFromTimestamp pads the record to seven digits and truncates beyond that", function () { + assert.equal(streams.eventIdFromTimestamp(1647297513618, "full", 0 as any), "z/2022/03/14/22/38/1647297513618-0000000"); + assert.equal(streams.eventIdFromTimestamp(1647297513618, "full", 123456789 as any), "z/2022/03/14/22/38/1647297513618-3456789"); + }); + }); + + describe("eventIdToTimestamp", function () { + it("eventIdToTimestamp reads the millisecond timestamp from a full event id", function () { + assert.equal(streams.eventIdToTimestamp("z/2022/03/14/22/38/1647297513618-0000035"), 1647297513618); + }); + + it("eventIdToTimestamp reads the timestamp when there is no record suffix", function () { + assert.equal(streams.eventIdToTimestamp("z/2022/03/14/22/38/1647297513618"), 1647297513618); + }); + + it("eventIdToTimestamp uses the minute prefix when the timestamp part is empty", function () { + assert.equal(streams.eventIdToTimestamp("z/2022/03/14/22/38/"), Date.UTC(2022, 2, 14, 22, 38)); + }); + + it("eventIdToTimestamp uses the hour prefix", function () { + assert.equal(streams.eventIdToTimestamp("z/2022/03/14/22/"), Date.UTC(2022, 2, 14, 22)); + }); + + it("eventIdToTimestamp uses the date prefix", function () { + assert.equal(streams.eventIdToTimestamp("z/2022/03/14/"), Date.UTC(2022, 2, 14)); + }); + + it("eventIdToTimestamp uses the month prefix", function () { + assert.equal(streams.eventIdToTimestamp("z/2022/03/"), Date.UTC(2022, 2, 1)); + }); + + it("eventIdToTimestamp uses the year prefix", function () { + assert.equal(streams.eventIdToTimestamp("z/2022/"), Date.UTC(2022, 0, 1)); + }); + + it("eventIdToTimestamp treats a timestamp part of 10000 or less as not a timestamp", function () { + assert.equal(streams.eventIdToTimestamp("z/2022/03/14/22/38/5"), Date.UTC(2022, 2, 14, 22, 38)); + }); + + it("eventIdToTimestamp round trips every case that carries a millisecond timestamp", function () { + cases.filter(([, g]) => g === "full" || g === "millisecond").forEach(([timestamp, granularity, record]) => { + const eid = streams.eventIdFromTimestamp(timestamp, granularity as any, record as any); + assert.equal(streams.eventIdToTimestamp(eid), timestamp); + }); + }); + + it("eventIdToTimestamp round trips second granularity to the whole second", function () { + const eid = streams.eventIdFromTimestamp(1767225599999, "second"); + assert.equal(streams.eventIdToTimestamp(eid), 1767225599000); + }); + + it("eventIdToTimestamp throws for a non-string id", function () { + assert.throws(() => streams.eventIdToTimestamp(123 as any), "Invalid Event Id. Should be a string."); + }); + + it("eventIdToTimestamp throws when the id does not start with z", function () { + assert.throws(() => streams.eventIdToTimestamp("x/2022/03/14/22/38/1647297513618"), "Unable to get timestamp for Event Id: x/2022/03/14/22/38/1647297513618"); + }); + + it("eventIdToTimestamp throws when no usable parts remain", function () { + assert.throws(() => streams.eventIdToTimestamp("z/"), "Unable to get timestamp for Event Id: z/"); + }); + }); +}); diff --git a/test/wrappers.fanout.utest.ts b/test/wrappers.fanout.utest.ts new file mode 100644 index 0000000..193e088 --- /dev/null +++ b/test/wrappers.fanout.utest.ts @@ -0,0 +1,485 @@ +import sinon from "sinon"; +import { expect, assert } from "chai"; +import { Lambda } from "@aws-sdk/client-lambda"; +import childProcess from "child_process"; + +const leo = require("../index.js"); +const fanout = require("../wrappers/fanout.js"); + +describe("wrappers/fanout.js", function () { + const sandbox = sinon.createSandbox(); + const originals: any = {}; + let originalCheckpoint: sinon.SinonStub; + let originalCheckLock: sinon.SinonStub; + let originalReportComplete: sinon.SinonStub; + let originalFromLeo: sinon.SinonStub; + let envKeys = ["FANOUT_iid", "FANOUT_icount", "FANOUT_maxeid", "AWS_LAMBDA_FUNCTION_NAME", "AWS_LAMBDA_FUNCTION_VERSION", "AWS_DEFAULT_REGION"]; + let savedEnv: Record; + + beforeEach(() => { + originals.fromLeo = leo.streams.fromLeo; + originals.read = leo.read; + originals.checkpoint = leo.bot.checkpoint; + originals.checkLock = leo.bot.checkLock; + originals.reportComplete = leo.bot.reportComplete; + savedEnv = {}; + envKeys.forEach((k) => { savedEnv[k] = process.env[k]; delete process.env[k]; }); + + originalCheckpoint = sandbox.stub().callsFake((id, event, params, done) => done()); + originalCheckLock = sandbox.stub().callsFake((cron, runid, remaining, cb) => cb(null, { original: true })); + originalReportComplete = sandbox.stub().callsFake((cron, runid, status, log, opts, cb) => cb(null, { original: true })); + originalFromLeo = sandbox.stub(); + sandbox.stub(childProcess, "fork").throws(new Error("fanout test must not fork a process")); + leo.bot.checkpoint = originalCheckpoint; + leo.bot.checkLock = originalCheckLock; + leo.bot.reportComplete = originalReportComplete; + leo.streams.fromLeo = originalFromLeo; + }); + + afterEach(() => { + leo.streams.fromLeo = originals.fromLeo; + leo.read = originals.read; + leo.bot.checkpoint = originals.checkpoint; + leo.bot.checkLock = originals.checkLock; + leo.bot.reportComplete = originals.reportComplete; + envKeys.forEach((k) => { + if (savedEnv[k] === undefined) { delete process.env[k]; } else { process.env[k] = savedEnv[k]; } + }); + sandbox.restore(); + }); + + function context(remaining = 60000) { + return { getRemainingTimeInMillis: () => remaining }; + } + + function workerEvent(iid: number, icount: number, extra: any = {}) { + return { __cron: { iid, icount, maxeid: "z/2023/11/14/22/13/1700000000000", ...extra } }; + } + + function collect(stream) { + return new Promise((resolve, reject) => { + leo.streams.pipe(stream, leo.streams.eventstream.writeArray((err, results) => err ? reject(err) : resolve(results)), (err) => err && reject(err)); + }); + } + + describe("process safety", function () { + it("master with instances above 1 and no Lambda environment forks through child_process", async function () { + sandbox.useFakeTimers({ now: 1700000000000, toFake: ["Date"] }); + const handler = fanout((event, ctx, cb) => cb(null), { instances: 2 }); + let err; + try { await handler({ __cron: {} }, context()); } catch (e) { err = e; } + assert.equal((childProcess.fork as sinon.SinonStub).callCount, 1); + assert.match(err.message, /must not fork/); + }); + }); + + describe("reduceCheckpoints", function () { + it("reduceCheckpoints returns the single worker checkpoints with eid set", function () { + const result = fanout.reduceCheckpoints([{ + iid: 0, + checkpoints: { bot1: { read: { "queue:q": { checkpoint: "z/2", records: 3 } }, write: {} } } + }]); + assert.deepEqual(result, [{ + bot1: { read: { "queue:q": { checkpoint: "z/2", records: 3, eid: "z/2" } }, write: {} } + }]); + }); + + it("reduceCheckpoints keeps the lowest checkpoint across workers", function () { + const result = fanout.reduceCheckpoints([ + { checkpoints: { bot1: { read: { "queue:q": { checkpoint: "z/5", records: 1 } }, write: {} } } }, + { checkpoints: { bot1: { read: { "queue:q": { checkpoint: "z/3", records: 2 } }, write: {} } } }, + { checkpoints: { bot1: { read: { "queue:q": { checkpoint: "z/4", records: 3 } }, write: {} } } } + ]); + assert.deepEqual(result[0].bot1.read["queue:q"], { checkpoint: "z/3", records: 2, eid: "z/3" }); + }); + + it("reduceCheckpoints keeps the first worker checkpoint when it is already lowest", function () { + const result = fanout.reduceCheckpoints([ + { checkpoints: { bot1: { read: { "queue:q": { checkpoint: "z/1", records: 1 } }, write: {} } } }, + { checkpoints: { bot1: { read: { "queue:q": { checkpoint: "z/9", records: 2 } }, write: {} } } } + ]); + assert.deepEqual(result[0].bot1.read["queue:q"], { checkpoint: "z/1", records: 1, eid: "z/1" }); + }); + + it("reduceCheckpoints takes write checkpoints from the first worker only", function () { + const result = fanout.reduceCheckpoints([ + { checkpoints: { bot1: { read: {}, write: { "queue:w": { checkpoint: "z/1" } } } } }, + { checkpoints: { bot1: { read: {}, write: { "queue:w": { checkpoint: "z/0" } } } } } + ]); + assert.deepEqual(result[0].bot1.write, { "queue:w": { checkpoint: "z/1" } }); + }); + + it("reduceCheckpoints throws when any worker reported an error", function () { + assert.throws(() => fanout.reduceCheckpoints([ + { checkpoints: { bot1: { read: {}, write: {} } } }, + { error: "worker failed" } + ]), "errors from sub lambdas"); + }); + + it("reduceCheckpoints ignores empty worker responses", function () { + assert.deepEqual(fanout.reduceCheckpoints([null, {}]), [{}]); + }); + + it("reduceCheckpoints throws a TypeError when a later worker has a queue the first lacks", function () { + assert.throws(() => fanout.reduceCheckpoints([ + { checkpoints: { bot1: { read: { "queue:a": { checkpoint: "z/1" } }, write: {} } } }, + { checkpoints: { bot1: { read: { "queue:b": { checkpoint: "z/2" } }, write: {} } } } + ]), TypeError); + }); + }); + + describe("callCheckpointOnResponses", function () { + it("callCheckpointOnResponses checkpoints read then write queues with the type set", function (done) { + const checkpoint = sandbox.stub().callsFake((id, queue, params, cb) => cb(null, "ok")); + fanout.callCheckpointOnResponses(checkpoint, (err, results) => { + assert.isNull(err); + assert.deepEqual(results, ["ok", "ok"]); + assert.equal(checkpoint.callCount, 2); + assert.deepEqual(checkpoint.getCall(0).args.slice(0, 3), ["bot1", "queue:r", { checkpoint: "z/2", records: 4, eid: "z/2", type: "read" }]); + assert.deepEqual(checkpoint.getCall(1).args.slice(0, 3), ["bot1", "queue:w", { checkpoint: "z/7", type: "write" }]); + done(); + })([{ checkpoints: { bot1: { read: { "queue:r": { checkpoint: "z/2", records: 4 } }, write: { "queue:w": { checkpoint: "z/7" } } } } }]); + }); + + it("callCheckpointOnResponses calls back true when no worker processed events", function (done) { + const checkpoint = sandbox.stub(); + fanout.callCheckpointOnResponses(checkpoint, (err, result) => { + assert.isNull(err); + assert.isTrue(result); + assert.equal(checkpoint.callCount, 0); + done(); + })([{ checkpoints: {} }]); + }); + + it("callCheckpointOnResponses only checkpoints the first bot", function (done) { + const checkpoint = sandbox.stub().callsFake((id, queue, params, cb) => cb()); + fanout.callCheckpointOnResponses(checkpoint, () => { + assert.deepEqual(checkpoint.getCalls().map((c) => c.args[0]), ["bot1"]); + done(); + })([{ + checkpoints: { + bot1: { read: { "queue:r": { checkpoint: "z/1" } }, write: {} }, + bot2: { read: { "queue:r": { checkpoint: "z/1" } }, write: {} } + } + }]); + }); + + it("callCheckpointOnResponses reports a checkpoint failure", function (done) { + const checkpoint = sandbox.stub().callsFake((id, queue, params, cb) => cb("Stale Checkpoint")); + fanout.callCheckpointOnResponses(checkpoint, (err) => { + assert.equal(err, "Stale Checkpoint"); + done(); + })([{ checkpoints: { bot1: { read: { "queue:r": { checkpoint: "z/1" } }, write: {} } } }]); + }); + }); + + describe("eventPartition hashing", function () { + async function readThroughWorker(iid: number, icount: number, events: any[], partition?: any, maxOverride?: any) { + const reader: any = leo.streams.eventstream.readArray(events); + reader.checkpoint = () => "cp"; + reader.get = () => "got"; + originalFromLeo.returns(reader); + leo.streams.fromLeo = originalFromLeo; + let readerOut: any; + let stream: any; + const inner = fanout((event, ctx, cb) => { + stream = leo.streams.fromLeo("bot1", "queue:q", { maxOverride }); + collect(stream).then((r) => { readerOut = r; cb(null, "done"); }, cb); + }, partition, { instances: icount }); + await inner(workerEvent(iid, icount), context()); + return { out: readerOut, stream }; + } + + it("string eventPartition values are hashed and only the matching instance processes the event", async function () { + const events = [{ eid: "e1", k: "a" }, { eid: "e2", k: "b" }, { eid: "e3", k: "c" }]; + const part = (e) => e.k; + assert.deepEqual((await readThroughWorker(1, 3, events, part)).out.map((e) => e.eid), ["e1"]); + assert.deepEqual((await readThroughWorker(2, 3, events, part)).out.map((e) => e.eid), ["e2"]); + }); + + it("numeric and array partition values hash to their value and the sum of their parts", async function () { + const events = [{ eid: "n", k: 100 }, { eid: "arr", k: ["a", "b"] }]; + const part = (e) => e.k; + assert.deepEqual((await readThroughWorker(1, 3, events, part)).out.map((e) => e.eid), ["n"]); + assert.deepEqual((await readThroughWorker(2, 3, events, part)).out, []); + }); + + it("partition defaults to the event eid and handles negative hashes", async function () { + const events = [{ eid: "order-1" }, { eid: "order-2" }, { eid: "order-3" }, { eid: "order-4" }]; + assert.deepEqual((await readThroughWorker(1, 3, events)).out.map((e) => e.eid), ["order-2"]); + assert.deepEqual((await readThroughWorker(2, 3, events)).out.map((e) => e.eid), ["order-1", "order-4"]); + }); + + it("a partition object argument supplies eventPartition", async function () { + const events = [{ eid: "e1", k: "a" }, { eid: "e2", k: "b" }]; + const reader: any = leo.streams.eventstream.readArray(events); + originalFromLeo.returns(reader); + let out; + await fanout((event, ctx, cb) => { + collect(leo.streams.fromLeo("bot1", "queue:q")).then((r) => { out = r; cb(null); }, cb); + }, { eventPartition: (e) => e.k, instances: 3 })(workerEvent(2, 3), context()); + assert.deepEqual(out.map((e) => e.eid), ["e2"]); + }); + + it("the wrapped reader keeps the original checkpoint and get functions", async function () { + const { stream } = await readThroughWorker(1, 2, [{ eid: "e1" }]); + assert.equal(stream.checkpoint(), "cp"); + assert.equal(stream.get(), "got"); + }); + + it("reads are capped at the lowest of the requested max and the master maxeid", async function () { + await readThroughWorker(1, 2, [], undefined, undefined); + assert.equal(originalFromLeo.getCall(0).args[2].maxOverride, "z/2023/11/14/22/13/1700000000000"); + await readThroughWorker(1, 2, [], undefined, "z/2020/01/01/00/00/1"); + assert.equal(originalFromLeo.getCall(1).args[2].maxOverride, "z/2020/01/01/00/00/1"); + await readThroughWorker(1, 2, [], undefined, "z/2030/01/01/00/00/1"); + assert.equal(originalFromLeo.getCall(2).args[2].maxOverride, "z/2023/11/14/22/13/1700000000000"); + }); + + it("the override is installed on both leo.streams.fromLeo and leo.read", function () { + fanout(() => { }); + assert.equal(leo.streams.fromLeo, leo.read); + assert.notEqual(leo.read, originalFromLeo); + }); + }); + + describe("checkpoint/lock overrides", function () { + it("checkLock runs the real lock check only on instance 0", function (done) { + fanout(() => { }); + leo.bot.checkLock({ iid: 0 }, "run1", 1000, (err, data) => { + assert.deepEqual(data, { original: true }); + assert.deepEqual(originalCheckLock.getCall(0).args.slice(0, 3), [{ iid: 0 }, "run1", 1000]); + done(); + }); + }); + + it("checkLock is a no-op on worker instances", function (done) { + fanout(() => { }); + leo.bot.checkLock({ iid: 2 }, "run1", 1000, (err, data) => { + assert.isNull(err); + assert.deepEqual(data, {}); + assert.equal(originalCheckLock.callCount, 0); + done(); + }); + }); + + it("checkLock takes the instance from FANOUT_iid when running locally", function (done) { + fanout(() => { }); + process.env.FANOUT_iid = "3"; + process.env.FANOUT_icount = "4"; + process.env.FANOUT_maxeid = "z/9"; + const cron: any = { iid: 0 }; + leo.bot.checkLock(cron, "run1", 1000, (err, data) => { + assert.deepEqual(data, {}); + assert.equal(cron.iid, 3); + assert.equal(cron.icount, 4); + assert.equal(cron.maxeid, "z/9"); + assert.equal(originalCheckLock.callCount, 0); + done(); + }); + }); + + it("reportComplete reports only from instance 0", function (done) { + fanout(() => { }); + leo.bot.reportComplete({ iid: 0 }, "run1", "complete", "", {}, (err, data) => { + assert.deepEqual(data, { original: true }); + assert.equal(originalReportComplete.callCount, 1); + leo.bot.reportComplete({ iid: 1 }, "run1", "complete", "", {}, (err2, data2) => { + assert.deepEqual(data2, {}); + assert.equal(originalReportComplete.callCount, 1); + done(); + }); + }); + }); + + it("options passed as a third argument are ignored when eventPartition is not a function", function () { + fanout(() => { }, undefined, { allowCheckpoint: true }); + assert.notEqual(leo.bot.checkpoint, originalCheckpoint); + }); + + it("options passed as a third argument are honored when eventPartition is a function", function () { + fanout(() => { }, (e) => e.eid, { allowCheckpoint: true }); + assert.equal(leo.bot.checkpoint, originalCheckpoint); + }); + + it("allowCheckpoint leaves the real checkpoint function in place", function () { + fanout(() => { }, { allowCheckpoint: true }); + assert.equal(leo.bot.checkpoint, originalCheckpoint); + }); + + it("checkpoint is captured for the master instead of written", async function () { + let resolved; + const handler = fanout((event, ctx, cb) => { + leo.bot.checkpoint("bot1_reader", "queue:q", { eid: "z/1", units: 2, source_timestamp: 10, started_timestamp: 20, ended_timestamp: 30 }, () => { + leo.bot.checkpoint("bot1", "queue:q", { kinesis_number: "k/2", records: 3 }, () => { + leo.bot.checkpoint("bot1", "queue:w", { type: "write", eid: "z/9" }, () => cb(null, "data")); + }); + }); + }); + resolved = await handler(workerEvent(1, 2), context()); + assert.equal(originalCheckpoint.callCount, 0); + assert.deepEqual(resolved.checkpoints, { + bot1: { + read: { + "queue:q": { records: 5, checkpoint: "k/2", source_timestamp: undefined, started_timestamp: undefined, ended_timestamp: undefined } + }, + write: { + "queue:w": { records: 0, checkpoint: "z/9", source_timestamp: undefined, started_timestamp: undefined, ended_timestamp: undefined } + } + } + }); + }); + }); + + describe("worker invocation", function () { + it("worker returns error, checkpoints, data and iid and trims 3s from the remaining time", async function () { + let remaining; + const handler = fanout((event, ctx, cb) => { + remaining = ctx.getRemainingTimeInMillis(); + cb(null, "worker-data"); + }); + const response = await handler(workerEvent(1, 2), context(10000)); + assert.equal(remaining, 7000); + assert.deepEqual(response, { error: null, checkpoints: {}, data: "worker-data", iid: 1 }); + }); + + it("worker resolves from a promise returned by the handler", async function () { + const handler = fanout(() => Promise.resolve("promised")); + const response = await handler(workerEvent(2, 3), context()); + assert.equal(response.data, "promised"); + assert.equal(response.iid, 2); + }); + + it("worker reports a rejected promise as the response error", async function () { + const boom = new Error("handler failed"); + const handler = fanout(() => Promise.reject(boom)); + const response = await handler(workerEvent(1, 2), context()); + assert.equal(response.error, boom); + }); + + it("worker only responds once when the handler calls back and returns a promise", async function () { + const handler = fanout((event, ctx, cb) => { + cb(null, "first"); + return Promise.resolve("second"); + }); + const response = await handler(workerEvent(1, 2), context()); + assert.equal(response.data, "first"); + }); + + it("FANOUT_ environment variables turn an unmarked event into a worker", async function () { + process.env.FANOUT_iid = "2"; + process.env.FANOUT_icount = "4"; + process.env.FANOUT_maxeid = "z/local"; + let seen; + const handler = fanout((event, ctx, cb) => { seen = event; cb(null, "x"); }); + const response = await handler({}, context()); + assert.deepEqual(seen, {}); + assert.equal(response.iid, 2); + }); + }); + + describe("master invocation", function () { + beforeEach(() => { + sandbox.useFakeTimers({ now: 1700000000000, toFake: ["Date"] }); + }); + + it("master with a single instance runs the handler as iid 0 with the maxeid stamp", async function () { + let seen; + const handler = fanout((event, ctx, cb) => { seen = event.__cron; cb(null, "data"); }, { instances: 1 }); + const result = await handler({ __cron: { id: "bot1" } }, context()); + assert.deepEqual(seen, { id: "bot1", maxeid: "z/2023/11/14/22/13/1700000000000", iid: 0, icount: 1 }); + assert.isTrue(result); + }); + + it("master clamps instances to at least 1 and uses an instances function", async function () { + let seen; + const instances = sandbox.stub().returns(0); + const handler = fanout((event, ctx, cb) => { seen = event.__cron.icount; cb(null); }, { instances }); + await handler({ __cron: { id: "bot1" } }, context()); + assert.equal(seen, 1); + assert.equal(instances.getCall(0).args[0].__cron.id, "bot1"); + assert.equal(instances.getCall(0).args[1].id, "bot1"); + }); + + it("master writes the reduced checkpoints through the real checkpoint function", async function () { + const handler = fanout((event, ctx, cb) => { + leo.bot.checkpoint("bot1", "queue:q", { eid: "z/5", units: 7, source_timestamp: 1 }, () => cb(null)); + }, { instances: 1 }); + const result = await handler({ __cron: { id: "bot1" } }, context()); + assert.equal(originalCheckpoint.callCount, 1); + assert.deepEqual(originalCheckpoint.getCall(0).args.slice(0, 3), ["bot1", "queue:q", { + records: 7, + checkpoint: "z/5", + source_timestamp: 1, + started_timestamp: undefined, + ended_timestamp: undefined, + eid: "z/5", + type: "read" + }]); + }); + + it("master surfaces a handler error as a failure", async function () { + const handler = fanout((event, ctx, cb) => cb(new Error("bad")), { instances: 1 }); + let err; + try { await handler({ __cron: {} }, context()); } catch (e) { err = e; } + assert.equal(err.message, "errors from sub lambdas"); + }); + + it("master invokes the other instances through Lambda and merges their checkpoints", async function () { + process.env.AWS_LAMBDA_FUNCTION_NAME = "my-fn"; + process.env.AWS_LAMBDA_FUNCTION_VERSION = "7"; + process.env.AWS_DEFAULT_REGION = "us-west-2"; + const invoke = sandbox.stub(Lambda.prototype, "invoke").callsFake((params: any, cb: any) => { + const payload = JSON.parse(params.Payload); + cb(null, { + Payload: JSON.stringify({ + error: null, + iid: payload.__cron.iid, + checkpoints: { bot1: { read: { "queue:q": { checkpoint: `z/${10 - payload.__cron.iid}`, records: 1 } }, write: {} } } + }) + }); + }); + const handler = fanout((event, ctx, cb) => { + leo.bot.checkpoint("bot1", "queue:q", { eid: "z/8", units: 1 }, () => cb(null)); + }, { instances: 3 }); + await handler({ __cron: { id: "bot1" }, other: "keep" }, context()); + assert.equal(invoke.callCount, 2); + const payloads = invoke.getCalls().map((c) => JSON.parse((c.args[0] as any).Payload)); + assert.deepEqual(payloads.map((p) => [p.__cron.iid, p.__cron.icount]).sort(), [[1, 3], [2, 3]]); + assert.equal(payloads[0].other, "keep"); + const first: any = invoke.getCall(0).args[0]; + assert.equal(first.FunctionName, "my-fn"); + assert.equal(first.InvocationType, "RequestResponse"); + assert.equal(first.Qualifier, "7"); + assert.equal(originalCheckpoint.callCount, 1); + assert.equal(originalCheckpoint.getCall(0).args[2].checkpoint, "z/8"); + }); + + it("master fails when a Lambda instance returns a FunctionError", async function () { + process.env.AWS_LAMBDA_FUNCTION_NAME = "my-fn"; + sandbox.stub(Lambda.prototype, "invoke").callsFake((params: any, cb: any) => cb(null, { FunctionError: "Unhandled", Payload: "boom" })); + const handler = fanout((event, ctx, cb) => cb(null), { instances: 2 }); + let err; + try { await handler({ __cron: {} }, context()); } catch (e) { err = e; } + assert.equal(err, "boom"); + }); + + it("master fails when the Lambda invoke itself errors", async function () { + process.env.AWS_LAMBDA_FUNCTION_NAME = "my-fn"; + const boom = new Error("invoke failed"); + sandbox.stub(Lambda.prototype, "invoke").callsFake((params: any, cb: any) => cb(boom)); + const handler = fanout((event, ctx, cb) => cb(null), { instances: 2 }); + let err; + try { await handler({ __cron: {} }, context()); } catch (e) { err = e; } + assert.equal(err, boom); + }); + + it("master caps instances at maxInstances", async function () { + process.env.AWS_LAMBDA_FUNCTION_NAME = "my-fn"; + const invoke = sandbox.stub(Lambda.prototype, "invoke").callsFake((params: any, cb: any) => cb(null, { Payload: "null" })); + const handler = fanout((event, ctx, cb) => cb(null), { instances: 10, maxInstances: 3 }); + await handler({ __cron: {} }, context()); + assert.equal(invoke.callCount, 2); + }); + }); +});