diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 27d6c91..8725dee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,10 @@ jobs: test: runs-on: ${{ matrix.os }} timeout-minutes: 15 + # redis-memory-server would build Redis from source on install, which breaks on new macOS + # SDKs; the stores job below runs the stores against real servers instead + env: + REDISMS_DISABLE_POSTINSTALL: "1" strategy: fail-fast: false matrix: @@ -30,3 +34,67 @@ jobs: - run: npm install --no-audit --no-fund - run: npm run typecheck - run: npm test + + # the stores against real servers: redis, postgres and mysql as service containers, mongo + # as a single-node replica set (change streams need one). The test job above runs redis and + # postgres on servers they download and skips mongo and mysql. + stores: + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + node: [22, 26] + services: + redis: + image: redis:7 + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 5s + --health-timeout 3s + --health-retries 10 + postgres: + image: postgres:17 + env: + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 3s + --health-retries 10 + mysql: + image: mysql:8.4 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + MYSQL_DATABASE: inkan_test + ports: + - 3306:3306 + options: >- + --health-cmd "mysqladmin ping -h 127.0.0.1" + --health-interval 2s + --health-timeout 5s + --health-retries 30 + env: + REDIS_URL: redis://localhost:6379 + DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres + REDISMS_DISABLE_POSTINSTALL: "1" + MONGO_URL: mongodb://127.0.0.1:27017/?replicaSet=rs0&directConnection=true + MONGO_REPLSET: "1" + MYSQL_URL: mysql://root@127.0.0.1:3306/inkan_test + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-node@v5 + with: + node-version: ${{ matrix.node }} + - name: Start MongoDB 7 as a single-node replica set + run: | + docker run -d --name mongo -p 27017:27017 mongo:7 --replSet rs0 --bind_ip_all + for i in $(seq 1 30); do docker exec mongo mongosh --quiet --eval 'db.runCommand({ ping: 1 }).ok' && break; sleep 1; done + docker exec mongo mongosh --quiet --eval 'rs.initiate({ _id: "rs0", members: [{ _id: 0, host: "127.0.0.1:27017" }] })' + for i in $(seq 1 30); do [ "$(docker exec mongo mongosh --quiet --eval 'db.hello().isWritablePrimary')" = "true" ] && break; sleep 1; done + - run: npm install --no-audit --no-fund + - run: npm test -w packages/redis -w packages/postgres -w packages/mongo -w packages/mysql diff --git a/README.md b/README.md index 896b4e0..b7e1b2e 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,13 @@ itself, every package keeps no dependencies, but `@inkanjs/otel`, which needs | [`@inkanjs/proxy`](packages/proxy) | A streaming reverse proxy: a prefix or one route handed on to an upstream. | | [`@inkanjs/circuit-breaker`](packages/circuit-breaker) | Circuit breakers for outgoing calls and for routes: fail fast while something is down. | | [`@inkanjs/otel`](packages/otel) | OpenTelemetry tracing: a server span per request, with the HTTP semantic conventions. | +| [`@inkanjs/mongo`](packages/mongo) | A `Store` on MongoDB, on your own client: jobs, rate limits and sessions shared by every process. | +| [`@inkanjs/mysql`](packages/mysql) | A `Store` on MySQL 8, on your own `mysql2` pool: jobs, rate limits and sessions shared by every process. | +| [`@inkanjs/redis`](packages/redis) | A `Store` on Redis, on your own `redis` or `ioredis` client: jobs, rate limits and sessions shared by every process, with pub/sub. | +| [`@inkanjs/postgres`](packages/postgres) | A `Store` on PostgreSQL, on your own `pg` pool or postgres.js: jobs, rate limits and sessions shared by every process, with LISTEN/NOTIFY. | -Every package is tested in CI on Node 22, 24 and 26: +Every package is tested in CI on Node 22, 24 and 26; the stores against MongoDB 7 and +MySQL 8.4 (locally they download a server once, or use `MONGO_URL` and `MYSQL_URL`): ```sh npm install diff --git a/package-lock.json b/package-lock.json index a3326b2..1eab9ec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,150 @@ "node": ">=22.18" } }, + "node_modules/@embedded-postgres/darwin-arm64": { + "version": "18.4.0-beta.17", + "resolved": "https://registry.npmjs.org/@embedded-postgres/darwin-arm64/-/darwin-arm64-18.4.0-beta.17.tgz", + "integrity": "sha512-Kg1ZMNFzkVIJs3g4V2UYEc5X005km9rGinxFBH8R1sOU2Rblvz4pt2Uf93Pu8Rk273Ue9HIdrbMPpgUqwjUi0Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@embedded-postgres/darwin-x64": { + "version": "18.4.0-beta.17", + "resolved": "https://registry.npmjs.org/@embedded-postgres/darwin-x64/-/darwin-x64-18.4.0-beta.17.tgz", + "integrity": "sha512-4tShSYWMxUQTSWoQAdLnHISvRj6L9gTch9/31ASJPg6wgyN64LDRwMcOINEWybM7N0ropJ3nTYhFT3UX1bKY5Q==", + "cpu": [ + "x64" + ], + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@embedded-postgres/linux-arm": { + "version": "18.4.0-beta.17", + "resolved": "https://registry.npmjs.org/@embedded-postgres/linux-arm/-/linux-arm-18.4.0-beta.17.tgz", + "integrity": "sha512-VuD1nFasN7yG57zpJcp6MBXM0nXqGfb8GBQV+f1FGJ17+VMNYLIFLhFp99UlY4xnWG74FQC4NPYw8eCwSTJ6qg==", + "cpu": [ + "arm" + ], + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@embedded-postgres/linux-arm64": { + "version": "18.4.0-beta.17", + "resolved": "https://registry.npmjs.org/@embedded-postgres/linux-arm64/-/linux-arm64-18.4.0-beta.17.tgz", + "integrity": "sha512-TIzjtGnDGSD/LbdW0OWCEcbI+YVT0WKSfSr20TCkkP4UR8zeKe+QCZbO0/CM4hS9EWyZ4cDebjRvpRc3iMi6wg==", + "cpu": [ + "arm64" + ], + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@embedded-postgres/linux-ia32": { + "version": "18.4.0-beta.17", + "resolved": "https://registry.npmjs.org/@embedded-postgres/linux-ia32/-/linux-ia32-18.4.0-beta.17.tgz", + "integrity": "sha512-aSRe4kknqRHuedtpo/rDIaqa8r9VrKIgW9p5FkJyIFqUUkIkxRqND1dg8xrDRREcSUXRJKS0x+j/AuxflgYIfg==", + "cpu": [ + "ia32" + ], + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@embedded-postgres/linux-ppc64": { + "version": "18.4.0-beta.17", + "resolved": "https://registry.npmjs.org/@embedded-postgres/linux-ppc64/-/linux-ppc64-18.4.0-beta.17.tgz", + "integrity": "sha512-zO2RRUFdKRPplrdhmglG39VXJZzDXu3P3ONAG1sFPhGh89vbk24Btd5P7uOmQLIpWehzL0s+UqUWoeE/ZUKvgw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@embedded-postgres/linux-x64": { + "version": "18.4.0-beta.17", + "resolved": "https://registry.npmjs.org/@embedded-postgres/linux-x64/-/linux-x64-18.4.0-beta.17.tgz", + "integrity": "sha512-jVw/MdDtIX/vICH/DKIe6/mHpiCggdx6QVyza4vt/NbcZFsL0KhwglF6F1Koqx3gRBZ9XtN+vi63EsqSyqOSxA==", + "cpu": [ + "x64" + ], + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@embedded-postgres/windows-x64": { + "version": "18.4.0-beta.17", + "resolved": "https://registry.npmjs.org/@embedded-postgres/windows-x64/-/windows-x64-18.4.0-beta.17.tgz", + "integrity": "sha512-AwRerliA4IGWyW5jWBvHt5vidVUSB0QQW9Tt2y7ScnmifnCb/awfxZr4BkBSTv+gNt8Djcddqs+xSF5Z6/CkTg==", + "cpu": [ + "x64" + ], + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16" + } + }, "node_modules/@inkanjs/auth": { "resolved": "packages/auth", "link": true @@ -33,14 +177,30 @@ "resolved": "packages/idempotency", "link": true }, + "node_modules/@inkanjs/mongo": { + "resolved": "packages/mongo", + "link": true + }, + "node_modules/@inkanjs/mysql": { + "resolved": "packages/mysql", + "link": true + }, "node_modules/@inkanjs/otel": { "resolved": "packages/otel", "link": true }, + "node_modules/@inkanjs/postgres": { + "resolved": "packages/postgres", + "link": true + }, "node_modules/@inkanjs/proxy": { "resolved": "packages/proxy", "link": true }, + "node_modules/@inkanjs/redis": { + "resolved": "packages/redis", + "link": true + }, "node_modules/@inkanjs/secure-headers": { "resolved": "packages/secure-headers", "link": true @@ -49,6 +209,54 @@ "resolved": "packages/session", "link": true }, + "node_modules/@ioredis/commands": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-2.0.0.tgz", + "integrity": "sha512-vrx0AE/T0h7cRZwfo1M39Cr+ZhZrkf0V8mQN75wucKCxCLD9l/VX6no3gFvrLqD1IlG/1LtzWovqEw3t0Vr9zg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "dev": true, + "license": "ISC", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@mongodb-js/saslprep": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.6.1.tgz", + "integrity": "sha512-D4CQtk9IwjCLAvCf1xvHEod3960q8XMUzfmdRU9qjUNlHnesGuTOvJcOTaPm9koZ1caUKA2aTbszZiAkcmSgmg==", + "dev": true, + "license": "MIT", + "dependencies": { + "sparse-bitfield": "^3.0.3" + } + }, "node_modules/@opentelemetry/api": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", @@ -59,6 +267,91 @@ "node": ">=8.0.0" } }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "dev": true, + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@redis/bloom": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-6.3.0.tgz", + "integrity": "sha512-NQ5poYpZr0jv6zazN1cy6JY5kIipKFZ3R9GTL4kOUTX3a9YAfY29U8bsE4lWhVQEM+G4V0BXVC57mubIdDijeA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 20.0.0" + }, + "peerDependencies": { + "@redis/client": "^6.3.0" + } + }, + "node_modules/@redis/client": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-6.3.0.tgz", + "integrity": "sha512-fHrjwGCBvAANGi+TuRuwKi83sIOzgHr8eh+Uf48w3fSX7Hhrz174NrWqfJhAC2vnN0d/tfaZpdr6+o6JC3QHwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 20.0.0" + }, + "peerDependencies": { + "@node-rs/xxhash": "^1.1.0", + "@opentelemetry/api": ">=1 <2" + }, + "peerDependenciesMeta": { + "@node-rs/xxhash": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + } + } + }, + "node_modules/@redis/json": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-6.3.0.tgz", + "integrity": "sha512-yt6vRfPBXtW/3qm53cF81TSQpLMjWdW4/2E9WQ2GgbvsKkuCKCFQIcMiKlil1ZMnezK+IQuS795xKU5psbaegw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 20.0.0" + }, + "peerDependencies": { + "@redis/client": "^6.3.0" + } + }, + "node_modules/@redis/search": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-6.3.0.tgz", + "integrity": "sha512-RPXAZmKjZZG7Tm7m1w9keFdXzbL91PiSHQBfQRe9xwW40jNsTRbG0FwhIz4QlHPU4gKDAsCy8IQHj4yZl4H9rw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 20.0.0" + }, + "peerDependencies": { + "@redis/client": "^6.3.0" + } + }, + "node_modules/@redis/time-series": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-6.3.0.tgz", + "integrity": "sha512-+fJOB8mN1z5WRakCJNh5lVdz1KqvHGSOistsKeitD24QsCtPwyRQXWC0iq6T/i/ubLE/YcguZqPm8F598MKk4A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 20.0.0" + }, + "peerDependencies": { + "@redis/client": "^6.3.0" + } + }, "node_modules/@types/node": { "version": "26.6.4", "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.4.tgz", @@ -69,6 +362,35 @@ "undici-types": "~8.9.0" } }, + "node_modules/@types/pg": { + "version": "8.23.1", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.23.1.tgz", + "integrity": "sha512-fKVHpikPdg4GKks3JuLEhvwSyvwzF23hnabPy6DD8ljVbC7+6J5dQzdv4arV6jqq57djnMgs1HKBxX4P8aBI3A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^2.2.0" + } + }, + "node_modules/@types/webidl-conversions": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz", + "integrity": "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/whatwg-url": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-13.0.0.tgz", + "integrity": "sha512-N8WXpbE6Wgri7KUSvrmQcqrMllKZ9uxkYWMt+mCSGwNc0Hsw9VQTW7ApqI4XNrx6/SaM2QQJCzMPDEXE058s+Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/webidl-conversions": "*" + } + }, "node_modules/@typescript/typescript-aix-ppc64": { "version": "7.0.2", "resolved": "https://registry.npmjs.org/@typescript/typescript-aix-ppc64/-/typescript-aix-ppc64-7.0.2.tgz", @@ -410,9 +732,9 @@ } }, "node_modules/@vxnsin/inkan": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/@vxnsin/inkan/-/inkan-0.7.0.tgz", - "integrity": "sha512-d9RavATzolWCk/WHy4ktZVHNT/obYwfllRHHuwUgmEAqYI6f/JoWl6TVPGQ9T8iHwc48pOB1+5opELAzIWzaSw==", + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@vxnsin/inkan/-/inkan-0.8.0.tgz", + "integrity": "sha512-hwCyNL0VQcvTvWxjIVCT8QFaQ92fR0dXB7aca0AGXqdBzd7HpeirLg/o2QI16MIkHroxSGuPaerM/f3lO78zBA==", "dev": true, "license": "MIT", "bin": { @@ -422,160 +744,2086 @@ "node": ">=22" } }, - "node_modules/typescript": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.0.2.tgz", - "integrity": "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==", + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", "dev": true, - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc" - }, + "license": "MIT", "engines": { - "node": ">=16.20.0" - }, - "optionalDependencies": { - "@typescript/typescript-aix-ppc64": "7.0.2", - "@typescript/typescript-darwin-arm64": "7.0.2", - "@typescript/typescript-darwin-x64": "7.0.2", - "@typescript/typescript-freebsd-arm64": "7.0.2", - "@typescript/typescript-freebsd-x64": "7.0.2", - "@typescript/typescript-linux-arm": "7.0.2", - "@typescript/typescript-linux-arm64": "7.0.2", - "@typescript/typescript-linux-loong64": "7.0.2", - "@typescript/typescript-linux-mips64el": "7.0.2", - "@typescript/typescript-linux-ppc64": "7.0.2", - "@typescript/typescript-linux-riscv64": "7.0.2", - "@typescript/typescript-linux-s390x": "7.0.2", - "@typescript/typescript-linux-x64": "7.0.2", - "@typescript/typescript-netbsd-arm64": "7.0.2", - "@typescript/typescript-netbsd-x64": "7.0.2", - "@typescript/typescript-openbsd-arm64": "7.0.2", - "@typescript/typescript-openbsd-x64": "7.0.2", - "@typescript/typescript-sunos-x64": "7.0.2", - "@typescript/typescript-win32-arm64": "7.0.2", - "@typescript/typescript-win32-x64": "7.0.2" + "node": ">= 14" } }, - "node_modules/undici-types": { - "version": "8.9.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz", - "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==", + "node_modules/ansi-regex": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.4.0.tgz", + "integrity": "sha512-KzTVk2tCWAHtYrvvvaP8bJKJq2pVinhLcGEQdtLIYPbmNGNyYe8QwNaTUYQp2J7/vIsUKt5QCqAfUkYyG9DkOw==", "dev": true, - "license": "MIT" - }, - "packages/auth": { - "name": "@inkanjs/auth", - "version": "0.1.0", "license": "MIT", - "devDependencies": { - "@vxnsin/inkan": "^0.7.0" - }, "engines": { - "node": ">=22.18" + "node": ">=12" }, - "peerDependencies": { - "@vxnsin/inkan": ">=0.7.0 <0.8.0" + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" } }, - "packages/circuit-breaker": { - "name": "@inkanjs/circuit-breaker", - "version": "0.1.0", + "node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "dev": true, "license": "MIT", - "devDependencies": { - "@vxnsin/inkan": "^0.7.0" - }, "engines": { - "node": ">=22.18" + "node": ">=12" }, - "peerDependencies": { - "@vxnsin/inkan": ">=0.7.0 <0.8.0" + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "packages/etag": { - "name": "@inkanjs/etag", - "version": "0.1.0", + "node_modules/async-exit-hook": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/async-exit-hook/-/async-exit-hook-2.0.1.tgz", + "integrity": "sha512-NW2cX8m1Q7KPA7a5M2ULQeZ2wR5qI5PAbw5L0UOMxdioVk9PMZ0h1TmyZEkPYrCvYjDlFICusOu1dlEKAAeXBw==", + "dev": true, "license": "MIT", - "devDependencies": { - "@vxnsin/inkan": "^0.7.0" - }, "engines": { - "node": ">=22.18" - }, - "peerDependencies": { - "@vxnsin/inkan": ">=0.7.0 <0.8.0" + "node": ">=0.12.0" } }, - "packages/idempotency": { - "name": "@inkanjs/idempotency", - "version": "0.1.0", + "node_modules/async-mutex": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/async-mutex/-/async-mutex-0.5.0.tgz", + "integrity": "sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA==", + "dev": true, "license": "MIT", - "devDependencies": { - "@vxnsin/inkan": "^0.7.0" - }, - "engines": { - "node": ">=22.18" - }, - "peerDependencies": { - "@vxnsin/inkan": ">=0.7.0 <0.8.0" + "dependencies": { + "tslib": "^2.4.0" } }, - "packages/otel": { - "name": "@inkanjs/otel", - "version": "0.1.0", + "node_modules/aws-ssl-profiles": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/aws-ssl-profiles/-/aws-ssl-profiles-1.1.2.tgz", + "integrity": "sha512-NZKeq9AfyQvEeNlN0zSYAaWrmBffJh3IELMZfRpJVWgrpEbtEpnjvzqBPf+mxoI287JohRDoa+/nsfqqiZmF6g==", + "dev": true, "license": "MIT", - "devDependencies": { + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/b4a": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.9.0.tgz", + "integrity": "sha512-dpfcF9fDNR6++cthXR67iyhgqWy9CBouAvIWhIntzBG6cvK/cnIPiZQjBwi/ZqjjBEDGfoNDtmB0kTjroOJ3pQ==", + "dev": true, + "license": "Apache-2.0", + "peerDependencies": { + "react-native-b4a": "*" + }, + "peerDependenciesMeta": { + "react-native-b4a": { + "optional": true + } + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/bare-events": { + "version": "2.9.2", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.9.2.tgz", + "integrity": "sha512-AIPKioV7/Y/8KfZ3AAhjPJxLLbY49S64Ym5DakZlUg75qQiTgUq9hEJoEwa4eUezPUlXRy/i5NpsKvo9jgKmoA==", + "dev": true, + "license": "Apache-2.0", + "peerDependencies": { + "bare-abort-controller": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + } + } + }, + "node_modules/bare-fs": { + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.8.4.tgz", + "integrity": "sha512-MPC2eyrpbk3A873sK4y6rk+Iv4o2enxTjHZ8zH3HO0mjrpm5XjnguY2XmfidIsDV6vnjpbMBKUAJpV0ISD2SsQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.5.4", + "bare-path": "^3.0.0", + "bare-stream": "^2.6.4", + "bare-url": "^2.2.2", + "fast-fifo": "^1.3.2" + }, + "engines": { + "bare": ">=1.28.0" + }, + "peerDependencies": { + "bare-buffer": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + } + } + }, + "node_modules/bare-path": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.1.2.tgz", + "integrity": "sha512-ZyKbsuuqK6Ag0K8pX6V5Txq6XeJRvY+wXucnFGRjiyVYP9YWDpIQugk/b+enRYrEYBJaqLzghRQpXPMR7341Nw==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/bare-stream": { + "version": "2.13.4", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.13.4.tgz", + "integrity": "sha512-PcrQ8lVLbiJscNm1Kez+Yp4Gy4AHGcN1lzwjvf5NybWen7VvEgUfyfnXYJ2zNqWnzOfCb1Abq6lH8ti0syQszA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.8.1", + "streamx": "^2.25.0", + "teex": "^1.0.1" + }, + "peerDependencies": { + "bare-abort-controller": "*", + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + }, + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, + "node_modules/bare-url": { + "version": "2.5.5", + "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.5.5.tgz", + "integrity": "sha512-iM9qJCruvpt2rPaLsrumVDru9pogI7PHziHIa9kTNRFt8C8HuSa+kI9kFezK+NvHEFnnz71rKGZ1EAilq/3V/Q==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bare-path": "^3.0.0" + } + }, + "node_modules/bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==", + "dev": true, + "license": "MIT" + }, + "node_modules/brace-expansion": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.7.tgz", + "integrity": "sha512-uZbew1NqdmPDTMJ8ah1y+b+9QEJrfkXFk3RcTQw3X0jW/xRUvFKsg1CfQdSYGdTbXZWExtU3J3ccxtnfw1Fi0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/bson": { + "version": "7.3.3", + "resolved": "https://registry.npmjs.org/bson/-/bson-7.3.3.tgz", + "integrity": "sha512-oz3LwE3qGEhTCSyhzQRLY3Bj1NLOsjFMf6tcmY5J8M+3R7EVcmoBfSHWvjUK3yXCLDUsGpIuB3aQL4nDZ1EV/g==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/chownr": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/cluster-key-slot": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.1.tgz", + "integrity": "sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/commondir": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", + "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==", + "dev": true, + "license": "MIT" + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/duplexer2": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", + "integrity": "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "readable-stream": "^2.0.2" + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true, + "license": "MIT" + }, + "node_modules/embedded-postgres": { + "version": "18.4.0-beta.17", + "resolved": "https://registry.npmjs.org/embedded-postgres/-/embedded-postgres-18.4.0-beta.17.tgz", + "integrity": "sha512-ORT/A1YiO6ecpRHpyIIBgyCR/8ASqqYZuFw11aX9PkqTX3FUM5+9sNOXY1mGIixxsm2TWSIA5WghJEk7A5ZxVw==", + "dev": true, + "license": "MIT", + "dependencies": { + "async-exit-hook": "^2.0.1", + "pg": "^8.7.3" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "@embedded-postgres/darwin-arm64": "^18.4.0-beta.17", + "@embedded-postgres/darwin-x64": "^18.4.0-beta.17", + "@embedded-postgres/linux-arm": "^18.4.0-beta.17", + "@embedded-postgres/linux-arm64": "^18.4.0-beta.17", + "@embedded-postgres/linux-ia32": "^18.4.0-beta.17", + "@embedded-postgres/linux-ppc64": "^18.4.0-beta.17", + "@embedded-postgres/linux-x64": "^18.4.0-beta.17", + "@embedded-postgres/windows-x64": "^18.4.0-beta.17" + } + }, + "node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true, + "license": "MIT" + }, + "node_modules/events-universal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz", + "integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.7.0" + } + }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/find-cache-dir": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.2.tgz", + "integrity": "sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig==", + "dev": true, + "license": "MIT", + "dependencies": { + "commondir": "^1.0.1", + "make-dir": "^3.0.2", + "pkg-dir": "^4.1.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/avajs/find-cache-dir?sponsor=1" + } + }, + "node_modules/find-package-json": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/find-package-json/-/find-package-json-1.2.0.tgz", + "integrity": "sha512-+SOGcLGYDJHtyqHd87ysBhmaeQ95oWspDKnMXBrnQ9Eq4OkLNqejgoaD8xVWu6GPa0B6roa6KinCMEMcVeqONw==", + "dev": true, + "license": "MIT" + }, + "node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/follow-redirects": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.1.tgz", + "integrity": "sha512-FNvFGzoMLWmE6Yj9spb/zjd7yiNCHiAW9/Tg9CXrQ8wuu32HtlJOwWO11OJafl5FfY3DxTdQ0vj42zU1kvv5jg==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "dev": true, + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/fs-extra": { + "version": "11.3.1", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.1.tgz", + "integrity": "sha512-eXvGGwZ5CL17ZSwHWd3bbgk7UUpF6IFHtP57NYYakPvHOs8GDgDe5KJI36jIJzDkJ6eJjuzRA8eBQb6SkKue0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, + "node_modules/generate-function": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.3.1.tgz", + "integrity": "sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-property": "^1.0.2" + } + }, + "node_modules/get-port": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz", + "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/glob": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "dev": true, + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/ioredis": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-6.0.0.tgz", + "integrity": "sha512-f+Dtubxfpf6KYFq7WVXJoOLn0bk4TJrMrN9SzeE+jrWrCWj7XX3fA6vkryafhADX+GMymRxgDJDOI33COkJc0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@ioredis/commands": "2.0.0", + "cluster-key-slot": "1.1.1", + "debug": "4.4.3", + "denque": "2.1.0", + "redis-errors": "1.2.0", + "standard-as-callback": "2.1.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-property": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", + "integrity": "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==", + "dev": true, + "license": "MIT" + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, + "node_modules/jsonfile": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.1.tgz", + "integrity": "sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/lockfile": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/lockfile/-/lockfile-1.0.4.tgz", + "integrity": "sha512-cvbTwETRfsFh4nHsL1eGWapU1XFi5Ot9E85sWAwia7Y7EgB7vfqcZhTKZ+l7hCGxSPoushMv5GKhT5PdLv03WA==", + "dev": true, + "license": "ISC", + "dependencies": { + "signal-exit": "^3.0.2" + } + }, + "node_modules/lockfile/node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/lru.min": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/lru.min/-/lru.min-1.1.5.tgz", + "integrity": "sha512-5J9ysMYUpYIg9RF2vJpy9SinEmSviFSe0GyPpCQ4L5QSkLAgeLXlTAOu2ZwWUU5m+0SBl6gUU1R1ZQB3aKypfA==", + "dev": true, + "license": "MIT", + "engines": { + "bun": ">=1.0.0", + "deno": ">=1.30.0", + "node": ">=8.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wellwelwel" + } + }, + "node_modules/make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^6.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/memory-pager": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", + "dev": true, + "license": "MIT" + }, + "node_modules/minimatch": { + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.2" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/minizlib": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "minipass": "^7.1.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/mongodb": { + "version": "7.7.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-7.7.0.tgz", + "integrity": "sha512-XG1xXEqyouAV8GCb3PuRaujofeXKpAJAPvXYWZgHl6Dz4QHSJ1nRLl0+GpqaLTwKvC0RFE9truOur34RNFY1nw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@mongodb-js/saslprep": "^1.5.0", + "bson": "^7.3.2", + "mongodb-connection-string-url": "^7.0.2" + }, + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "@aws-sdk/credential-providers": "^3.806.0", + "@mongodb-js/zstd": "^7.0.0", + "gcp-metadata": "^7.0.1", + "kerberos": "^7.0.0", + "mongodb-client-encryption": "^7.2.0", + "snappy": "^7.3.2", + "socks": "^2.8.6" + }, + "peerDependenciesMeta": { + "@aws-sdk/credential-providers": { + "optional": true + }, + "@mongodb-js/zstd": { + "optional": true + }, + "gcp-metadata": { + "optional": true + }, + "kerberos": { + "optional": true + }, + "mongodb-client-encryption": { + "optional": true + }, + "snappy": { + "optional": true + }, + "socks": { + "optional": true + } + } + }, + "node_modules/mongodb-connection-string-url": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-7.0.2.tgz", + "integrity": "sha512-ZoS07RoFqpKYQwAk59qmrx8+jJHNHU30UjlU96QktiGn1ltvDr+vCznLX5DiUBLEpMAHatHNWV1nM/74ul66kA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/whatwg-url": "^13.0.0", + "whatwg-url": "^14.1.0" + }, + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/mongodb-memory-server-core": { + "version": "11.3.0", + "resolved": "https://registry.npmjs.org/mongodb-memory-server-core/-/mongodb-memory-server-core-11.3.0.tgz", + "integrity": "sha512-QBu/RsRpBfcNd5CrBDfCzqcaFPaYpivUzGYUy9YgVc3up78ff0gIHQVD9PLmBLOOxFX6BM9hU6YT0HocZH+s1g==", + "dev": true, + "license": "MIT", + "dependencies": { + "async-mutex": "^0.5.0", + "camelcase": "^6.3.0", + "debug": "^4.4.3", + "find-cache-dir": "^3.3.2", + "follow-redirects": "^1.16.0", + "https-proxy-agent": "^7.0.6", + "mongodb": "~7.5.0", + "new-find-package-json": "^2.0.0", + "semver": "^7.8.5", + "tar-stream": "^3.2.1", + "tslib": "^2.8.1", + "yauzl": "^3.4.0" + }, + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/mongodb-memory-server-core/node_modules/mongodb": { + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-7.5.0.tgz", + "integrity": "sha512-5FnrEDLnvp6ycUOGLNLLU33BfCx2qmp2mJjGPDwKLruYsVzXVSK5fsGpoDXvsXJwBfBsD7ebMRdawbDxC2814g==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@mongodb-js/saslprep": "^1.4.11", + "bson": "^7.2.0", + "mongodb-connection-string-url": "^7.0.1" + }, + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "@aws-sdk/credential-providers": "^3.806.0", + "@mongodb-js/zstd": "^7.0.0", + "gcp-metadata": "^7.0.1", + "kerberos": "^7.0.0", + "mongodb-client-encryption": "^7.2.0", + "snappy": "^7.3.2", + "socks": "^2.8.6" + }, + "peerDependenciesMeta": { + "@aws-sdk/credential-providers": { + "optional": true + }, + "@mongodb-js/zstd": { + "optional": true + }, + "gcp-metadata": { + "optional": true + }, + "kerberos": { + "optional": true + }, + "mongodb-client-encryption": { + "optional": true + }, + "snappy": { + "optional": true + }, + "socks": { + "optional": true + } + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/mysql-memory-server": { + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/mysql-memory-server/-/mysql-memory-server-1.17.0.tgz", + "integrity": "sha512-dSWIrTGAPlqMVb4gB6S42X0GGq3HxH9btF+qUcWDKjgIQ0A/7U18jKaAKUF170GGTQJTSYE/9s0l1H/Vv8U7/w==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.8.5", + "signal-exit": "^4.1.0", + "unzipper": "^0.12.5" + }, + "bin": { + "mysql-memory-server": "dist/src/cli.js" + }, + "engines": { + "bun": ">=1.0.0", + "node": ">=16.6.0" + } + }, + "node_modules/mysql2": { + "version": "3.24.6", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.24.6.tgz", + "integrity": "sha512-enu73KhbAua2L3dIxjc6qg6ZJ7H3pt1BImZASWNWY5xSsmJ1ZuUPULTDGXgmJbI61vb+nJAG19IZclIq19UidQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "aws-ssl-profiles": "^1.1.2", + "generate-function": "^2.3.1", + "iconv-lite": "^0.7.3", + "long": "^5.3.2", + "lru.min": "^1.1.4", + "named-placeholders": "^1.1.6", + "sql-escaper": "^1.5.1" + }, + "engines": { + "node": ">= 8.0" + }, + "peerDependencies": { + "@types/node": ">= 8" + } + }, + "node_modules/named-placeholders": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.6.tgz", + "integrity": "sha512-Tz09sEL2EEuv5fFowm419c1+a/jSMiBjI9gHxVLrVdbUkkNUUfjsVYs9pVZu5oCon/kmRh9TfLEObFtkVxmY0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "lru.min": "^1.1.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/new-find-package-json": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/new-find-package-json/-/new-find-package-json-2.0.0.tgz", + "integrity": "sha512-lDcBsjBSMlj3LXH2v/FW3txlh2pYTjmbOXPYJD93HI5EwuLzI11tdHSIpUMmfq/IOsldj4Ps8M8flhm+pCK4Ew==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">=12.22.0" + } + }, + "node_modules/node-int64": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", + "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "dev": true, + "license": "BlueOak-1.0.0" + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", + "dev": true, + "license": "MIT" + }, + "node_modules/pg": { + "version": "8.23.1", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.23.1.tgz", + "integrity": "sha512-aL96AHANtWjPLDOLqnhx+ngp9+UK7ETEU8VJrDCGvsSSi/mGLcWYsS6Herg7lmaBJe4uwrfqsa7gTEFaSizDoQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.14.1", + "pg-pool": "^3.14.0", + "pg-protocol": "^1.16.1", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.4.1" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.1.tgz", + "integrity": "sha512-6PQbsFWZcp9EmJEwy5cGQ2La+AMWpP46lgbb8X+U/XsHIUweYDNCpeuKck5RxL2MdVFi7krbbEi5nX4Zh7JhrQ==", + "dev": true, + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.1.tgz", + "integrity": "sha512-qR3kGNPBLpCNtz0evbKA0Y/MRFXwSSdT+pTJvYp/bXTcReZbvX1kzF0IyTc1QnxqF7AZbOeBhNL8R5mYQZV/MA==", + "dev": true, + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", + "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.16.1.tgz", + "integrity": "sha512-p9VOFMiHB/ZbJATetbg+99PxssTVSQRnyuPSQ67mN1+1KBOjZaZ83ZQzltnxPhJwSsC3nwVjJ10DVJlerbFzLg==", + "dev": true, + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "dev": true, + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "dev": true, + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "find-up": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/postgres": { + "version": "3.4.9", + "resolved": "https://registry.npmjs.org/postgres/-/postgres-3.4.9.tgz", + "integrity": "sha512-GD3qdB0x1z9xgFI6cdRD6xu2Sp2WCOEoe3mtnyB5Ee0XrrL5Pe+e4CCnJrRMnL1zYtRDZmQQVbvOttLnKDLnaw==", + "dev": true, + "license": "Unlicense", + "engines": { + "node": ">=12" + }, + "funding": { + "type": "individual", + "url": "https://github.com/sponsors/porsager" + } + }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dev": true, + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/redis": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/redis/-/redis-6.3.0.tgz", + "integrity": "sha512-XFQbPie1lGpKeUZ8ySYY43yLnQy/iIQtHNvIr/1XdaABpb96rUSdm3Mfl+98FEVbv9ZJIzINMno5s0Nh8E5LEw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@redis/bloom": "6.3.0", + "@redis/client": "6.3.0", + "@redis/json": "6.3.0", + "@redis/search": "6.3.0", + "@redis/time-series": "6.3.0" + }, + "engines": { + "node": ">= 20.0.0" + } + }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-memory-server": { + "version": "0.17.1", + "resolved": "https://registry.npmjs.org/redis-memory-server/-/redis-memory-server-0.17.1.tgz", + "integrity": "sha512-obdMVFkzLRM+NaCeuZObYNUY5klmAvv91Hq8UBtGsz6YbZd1s5DauaZpIbmjkDoMgxagWIP3/UJiSkuarXTlBg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "camelcase": "^6.3.0", + "cross-spawn": "^7.0.3", + "debug": "^4.3.4", + "find-cache-dir": "^3.3.2", + "find-package-json": "^1.2.0", + "get-port": "^5.1.1", + "https-proxy-agent": "^7.0.0", + "lockfile": "^1.0.4", + "rimraf": "^5.0.1", + "semver": "^7.5.3", + "tar": "^7.5.4", + "tmp": "^0.2.1" + }, + "bin": { + "redis-memory-server": "bin/index.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/rimraf": { + "version": "5.0.10", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.10.tgz", + "integrity": "sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "glob": "^10.3.7" + }, + "bin": { + "rimraf": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true, + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true, + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/sparse-bitfield": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", + "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "memory-pager": "^1.0.2" + } + }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, + "node_modules/sql-escaper": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/sql-escaper/-/sql-escaper-1.5.2.tgz", + "integrity": "sha512-6CKD38c31SENivxOADeMNLdukOnUxUcflKtzVWzace7Riv1v7cAEym5Cx9Q7gYZ3ezIJI7ZpqecQq8cqNeSSFg==", + "dev": true, + "license": "MIT", + "engines": { + "bun": ">=1.0.0", + "deno": ">=2.0.0", + "node": ">=12.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/mysqljs/sql-escaper?sponsor=1" + } + }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "dev": true, + "license": "MIT" + }, + "node_modules/streamx": { + "version": "2.28.1", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.28.1.tgz", + "integrity": "sha512-zEzXb0s5Cds7tqMH6rhZ05lcJydCWiQPEwiNngVqzsxCc962vLY4Uw+mW7od8kDH258k2Uz/JrOkdIAAhSh9VA==", + "dev": true, + "license": "MIT", + "dependencies": { + "events-universal": "^1.0.0", + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + } + }, + "node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/string-width-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/tar": { + "version": "7.5.22", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.22.tgz", + "integrity": "sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.2", + "minizlib": "^3.1.0", + "yallist": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/tar-stream": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.2.2.tgz", + "integrity": "sha512-+8NeqHRjQWH9nYlwo2gamAMImZCVzI4UoEgDpWorBt9OEfppiZn+uSkskzQKPWIZyji/C8fpWO7u69G0DX0tbg==", + "dev": true, + "license": "MIT", + "dependencies": { + "b4a": "^1.9.0", + "bare-fs": "^4.8.2", + "fast-fifo": "^1.3.2", + "streamx": "^2.28.1" + } + }, + "node_modules/teex": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/teex/-/teex-1.0.1.tgz", + "integrity": "sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "streamx": "^2.12.5" + } + }, + "node_modules/text-decoder": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.7.tgz", + "integrity": "sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, + "node_modules/tmp": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.7.tgz", + "integrity": "sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.14" + } + }, + "node_modules/tr46": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz", + "integrity": "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==", + "dev": true, + "license": "MIT", + "dependencies": { + "punycode": "^2.3.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD" + }, + "node_modules/typescript": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.0.2.tgz", + "integrity": "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc" + }, + "engines": { + "node": ">=16.20.0" + }, + "optionalDependencies": { + "@typescript/typescript-aix-ppc64": "7.0.2", + "@typescript/typescript-darwin-arm64": "7.0.2", + "@typescript/typescript-darwin-x64": "7.0.2", + "@typescript/typescript-freebsd-arm64": "7.0.2", + "@typescript/typescript-freebsd-x64": "7.0.2", + "@typescript/typescript-linux-arm": "7.0.2", + "@typescript/typescript-linux-arm64": "7.0.2", + "@typescript/typescript-linux-loong64": "7.0.2", + "@typescript/typescript-linux-mips64el": "7.0.2", + "@typescript/typescript-linux-ppc64": "7.0.2", + "@typescript/typescript-linux-riscv64": "7.0.2", + "@typescript/typescript-linux-s390x": "7.0.2", + "@typescript/typescript-linux-x64": "7.0.2", + "@typescript/typescript-netbsd-arm64": "7.0.2", + "@typescript/typescript-netbsd-x64": "7.0.2", + "@typescript/typescript-openbsd-arm64": "7.0.2", + "@typescript/typescript-openbsd-x64": "7.0.2", + "@typescript/typescript-sunos-x64": "7.0.2", + "@typescript/typescript-win32-arm64": "7.0.2", + "@typescript/typescript-win32-x64": "7.0.2" + } + }, + "node_modules/undici-types": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz", + "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==", + "dev": true, + "license": "MIT" + }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/unzipper": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.12.5.tgz", + "integrity": "sha512-tXYOi9R57Uj/2Z25SOs5RRSzq886MBQj2gY8dPL+xl/kv6s6SvByoKfAtvfVeEuhntWDgjd2o9p2lb4TVPAz0A==", + "dev": true, + "license": "MIT", + "dependencies": { + "bluebird": "~3.7.2", + "duplexer2": "~0.1.4", + "fs-extra": "11.3.1", + "graceful-fs": "^4.2.2", + "node-int64": "^0.4.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true, + "license": "MIT" + }, + "node_modules/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/whatwg-url": { + "version": "14.2.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz", + "integrity": "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tr46": "^5.1.0", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, + "node_modules/yallist": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/yauzl": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-3.4.0.tgz", + "integrity": "sha512-jIH9yLR9wqr0wOS0TpBvo/g/2UgZH5qePVbjgRliiF0BYvOZyaBknKsF+x9Iht0O6sqgnB93rCICdOZFecJuDw==", + "dev": true, + "license": "MIT", + "dependencies": { + "pend": "~1.2.0" + }, + "engines": { + "node": ">=12" + } + }, + "packages/auth": { + "name": "@inkanjs/auth", + "version": "0.2.0", + "license": "MIT", + "devDependencies": { + "@vxnsin/inkan": "^0.8.0" + }, + "engines": { + "node": ">=22.18" + }, + "peerDependencies": { + "@vxnsin/inkan": ">=0.8.0 <1.0.0" + } + }, + "packages/circuit-breaker": { + "name": "@inkanjs/circuit-breaker", + "version": "0.2.0", + "license": "MIT", + "devDependencies": { + "@vxnsin/inkan": "^0.8.0" + }, + "engines": { + "node": ">=22.18" + }, + "peerDependencies": { + "@vxnsin/inkan": ">=0.8.0 <1.0.0" + } + }, + "packages/etag": { + "name": "@inkanjs/etag", + "version": "0.2.0", + "license": "MIT", + "devDependencies": { + "@vxnsin/inkan": "^0.8.0" + }, + "engines": { + "node": ">=22.18" + }, + "peerDependencies": { + "@vxnsin/inkan": ">=0.8.0 <1.0.0" + } + }, + "packages/idempotency": { + "name": "@inkanjs/idempotency", + "version": "0.2.0", + "license": "MIT", + "devDependencies": { + "@vxnsin/inkan": "^0.8.0" + }, + "engines": { + "node": ">=22.18" + }, + "peerDependencies": { + "@vxnsin/inkan": ">=0.8.0 <1.0.0" + } + }, + "packages/mongo": { + "name": "@inkanjs/mongo", + "version": "0.1.0", + "license": "MIT", + "devDependencies": { + "@vxnsin/inkan": "^0.8.0", + "mongodb": "^7.7.0", + "mongodb-memory-server-core": "^11.3.0" + }, + "engines": { + "node": ">=22.18" + }, + "peerDependencies": { + "@vxnsin/inkan": ">=0.8.0 <1.0.0" + } + }, + "packages/mysql": { + "name": "@inkanjs/mysql", + "version": "0.1.0", + "license": "MIT", + "devDependencies": { + "@vxnsin/inkan": "^0.8.0", + "mysql-memory-server": "^1.17.0", + "mysql2": "^3.24.6" + }, + "engines": { + "node": ">=22.18" + }, + "peerDependencies": { + "@vxnsin/inkan": ">=0.8.0 <1.0.0" + } + }, + "packages/otel": { + "name": "@inkanjs/otel", + "version": "0.2.0", + "license": "MIT", + "devDependencies": { "@opentelemetry/api": "^1.9.1", - "@vxnsin/inkan": "^0.7.0" + "@vxnsin/inkan": "^0.8.0" }, "engines": { "node": ">=22.18" }, "peerDependencies": { "@opentelemetry/api": "^1.4.0", - "@vxnsin/inkan": ">=0.7.0 <0.8.0" + "@vxnsin/inkan": ">=0.8.0 <1.0.0" + } + }, + "packages/postgres": { + "name": "@inkanjs/postgres", + "version": "0.1.0", + "license": "MIT", + "devDependencies": { + "@types/pg": "^8.23.1", + "@vxnsin/inkan": "^0.8.0", + "embedded-postgres": "^18.4.0-beta.17", + "pg": "^8.23.1", + "postgres": "^3.4.9" + }, + "engines": { + "node": ">=22.18" + }, + "peerDependencies": { + "@vxnsin/inkan": ">=0.8.0 <1.0.0" } }, "packages/proxy": { "name": "@inkanjs/proxy", + "version": "0.2.0", + "license": "MIT", + "devDependencies": { + "@vxnsin/inkan": "^0.8.0" + }, + "engines": { + "node": ">=22.18" + }, + "peerDependencies": { + "@vxnsin/inkan": ">=0.8.0 <1.0.0" + } + }, + "packages/redis": { + "name": "@inkanjs/redis", "version": "0.1.0", "license": "MIT", "devDependencies": { - "@vxnsin/inkan": "^0.7.0" + "@vxnsin/inkan": "^0.8.0", + "ioredis": "^6.0.0", + "redis": "^6.3.0", + "redis-memory-server": "^0.17.1" }, "engines": { "node": ">=22.18" }, "peerDependencies": { - "@vxnsin/inkan": ">=0.7.0 <0.8.0" + "@vxnsin/inkan": ">=0.8.0 <1.0.0" } }, "packages/secure-headers": { "name": "@inkanjs/secure-headers", - "version": "0.1.0", + "version": "0.2.0", "license": "MIT", "devDependencies": { - "@vxnsin/inkan": "^0.7.0" + "@vxnsin/inkan": "^0.8.0" }, "engines": { "node": ">=22.18" }, "peerDependencies": { - "@vxnsin/inkan": ">=0.7.0 <0.8.0" + "@vxnsin/inkan": ">=0.8.0 <1.0.0" } }, "packages/session": { "name": "@inkanjs/session", - "version": "0.1.0", + "version": "0.2.0", "license": "MIT", "devDependencies": { - "@vxnsin/inkan": "^0.7.0" + "@vxnsin/inkan": "^0.8.0" }, "engines": { "node": ">=22.18" }, "peerDependencies": { - "@vxnsin/inkan": ">=0.7.0 <0.8.0" + "@vxnsin/inkan": ">=0.8.0 <1.0.0" } } } diff --git a/packages/auth/CHANGELOG.md b/packages/auth/CHANGELOG.md index d01ba24..cfc268b 100644 --- a/packages/auth/CHANGELOG.md +++ b/packages/auth/CHANGELOG.md @@ -1,5 +1,33 @@ # Changelog +## 0.2.0 + +- **Any inkan `Store`** (inkan 0.8) for `store` and `bruteForce.store`: `memoryStore()` from + @vxnsin/inkan, or an adapter such as @inkanjs/redis. The plugin's own store interface and + `memoryStore` are gone; `memoryStore`, `Store` and `AuthStore` are re-exported from the + core, deprecated. +- **TTLs are in milliseconds**, as `Store` has them (they were seconds). A store of your own + that takes seconds keeps keys a thousand times too long: switch it to ms. +- `add`, `take` and `increment` are required: a store without them is refused when the + plugin is made, instead of falling back to a read and a write. +- Brute-force counters use `increment`, and a counter over its free attempts claims its lock + with `add`, so parallel guesses are counted and held back across processes. The count is + a fixed window (it starts over `window` after its first attempt), but the **escalation is + kept**: next to each counter its level (locks claimed) is stored, every attempt after a + lock claims the next one, twice as long up to `maxDelay`, and the level is forgotten only + after `window` of quiet past the last lock. A slow guesser stays at one guess per + `maxDelay`, as with the old sliding window, instead of getting the free attempts back every + `window`. +- `bruteForce.window` must be at least `bruteForce.maxDelay` (a TypeError otherwise). +- A successful login no longer gives the address its attempt back: a guesser could log into + an account of their own between guesses and never be locked. Successful logins count + toward `ipFreeAttempts`; the login's own counters are still cleared by a success. +- A TOTP code is claimed with `add`: it works once across processes too. +- Needs inkan `>=0.8.0 <1.0.0`. +- On the inkan 0.8.0 names, so the plugin prints no deprecation warning: `onOperation`, + `decorateRequest(…, { resolve: "preHandler" })`, and `problem(status, type, { detail, headers })` + for the answers with headers. + ## 0.1.0 - The user is loaded lazily, with inkan 0.7's `decorateRequest(…, { before: "handler", when })`: diff --git a/packages/auth/README.md b/packages/auth/README.md index a26cb83..4736c83 100644 --- a/packages/auth/README.md +++ b/packages/auth/README.md @@ -9,7 +9,7 @@ OpenAPI document and covered by `inkan check`. npm install @inkanjs/auth ``` -Node 22.18 or newer, inkan 0.7 (`register()` refuses other versions). MIT licensed. +Node 22.18 or newer, inkan 0.8 (`register()` refuses other versions). MIT licensed. ## Quick start @@ -134,7 +134,9 @@ cookie or token signed with it can still be alive (the session `maxAge`). `retry-after`): - the client address, `ipFreeAttempts` (50). An IPv6 address counts by its /64, since one client usually holds a whole /64. It is checked first: a locked address writes no more - counters. A successful login gives the address its attempt back. + counters. Every password attempt from it counts, successful ones too: a success never + resets the address, else a guesser would log into an account of their own between + guesses. Behind a big NAT, raise `ipFreeAttempts`. - the login from that address, `freeAttempts` (5): the fast lock against one guesser. - the login from everywhere, `loginFreeAttempts` (100). Why two for the login: with only a per-login lock, anyone who knows your email could lock you out from one machine. Here @@ -142,9 +144,18 @@ cookie or token signed with it can still be alive (the session `maxAge`). which takes many addresses, and still holds against a distributed guesser. A lockout skips the user lookup and the hash; a success clears the login's counters. - Counters forget after `window` (15 minutes) of quiet. They live in `bruteForce.store`, by - default a bounded memoryStore of their own, so a flood of made-up logins never pushes - sessions or refresh tokens out of `store`. With several processes, give it a shared one. + + **Escalation.** The count is a fixed window: it starts over `window` (15 minutes) after + its first attempt. The lockout does not: each counter keeps its level (how many locks it + has claimed), and once it has one, every attempt after a lock claims the next, twice as + long, up to `maxDelay`, whatever the count. The level is forgotten only after `window` of + quiet following the end of the last lock. So a guesser who keeps going, however slowly, + stays at one guess per `maxDelay` (15 minutes) for as long as they keep going; a fixed + window alone would give them the free attempts back every `window`. `window` must be at + least `maxDelay`. Counters live in + `bruteForce.store`, a `Store` of their own (by default a bounded memoryStore), so a flood + of made-up logins never pushes sessions or refresh tokens out of `store`. With several + processes, give it a shared one. - With `users.updatePasswordHash(id, hash)`, a login whose hash was made with older scrypt parameters stores a fresh one. @@ -283,26 +294,23 @@ typed their address. Without `emailVerified`, make a new account or refuse. ## Store -Sessions, refresh tokens and TOTP steps live in `store`, by default in memory, per process. -For several processes, give a shared one: +Sessions, refresh tokens and TOTP steps live in `store`; the brute-force counters in +`bruteForce.store`, a second one. Both are inkan `Store`s, by default `memoryStore()` from +@vxnsin/inkan: in memory, per process. Any `Store` works, such as @inkanjs/redis, @inkanjs/mongo, @inkanjs/mysql and @inkanjs/postgres (coming with inkan 0.8), or one you +write against the interface (`storeConformance` from `@vxnsin/inkan/testing` checks it): ```js -const store = { - get: (key) => redis.get(key).then((v) => (v ? JSON.parse(v) : undefined)), - set: (key, value, ttlSeconds) => redis.set(key, JSON.stringify(value), "EX", ttlSeconds), - delete: (key) => redis.del(key), - // these two must each be ONE atomic step of the store: - take: (key) => redis.getdel(key).then((v) => (v ? JSON.parse(v) : undefined)), - add: (key, value, ttlSeconds) => redis.set(key, JSON.stringify(value), "EX", ttlSeconds, "NX").then((r) => r === "OK"), -}; -``` +import { redisStore } from "@inkanjs/redis"; -`take(key)` reads and removes; `add(key, value, ttl)` writes only when the key is free and -says whether it did. They are what makes a refresh token, a recovery code or a pending TOTP -secret usable once across processes. A store without them still works, but those steps are -then a read and a write, atomic only within one process. +const store = redisStore(redis); +app.register(auth({ secret, users, store, bruteForce: { store } })); +``` -`memoryStore({ max })` keeps at most `max` entries (100 000), dropping the oldest first. +What the plugin needs of it: `add` and `take` make a refresh token, a recovery code, a TOTP +code and a pending TOTP secret usable once across processes; `increment` counts failed logins, +so a burst of parallel guesses is counted in full. TTLs are in milliseconds, as in every +`Store`. The default `store` keeps at most 100 000 keys, `bruteForce.store` 50 000 of its own, +so a flood of made-up logins never pushes sessions out. ## Options @@ -319,7 +327,7 @@ then a read and a write, atomic only within one process. | `routes` | `"/auth"` | Where the routes go; `false` for none. They reach `client`. | | `example` | none | `{ login, password }` of a test account (no TOTP): `inkan check` logs in with it, then calls `/me` and `/logout` with the session cookie. | | `origins` | `[]` | Other origins of yours for the CSRF check. | -| `store` | `memoryStore()` | `get`, `set(key, value, ttl)`, `delete`, and atomic `take`, `add` when shared. | +| `store` | `memoryStore()` | Any inkan `Store`: sessions, refresh tokens, TOTP steps. | | `publicUser` | `{ id, roles }` | What `/login` and `/me` show. | | `userSchema` | matches the default | The contract of `publicUser`. | | `roles` | `user.roles` | A user's roles. | @@ -344,12 +352,11 @@ Tested against the official vectors (RFC 7515, 8037, 6238, 4226, 4648, 7636). - **A process-local store is per process.** Behind several processes, use a shared store (and a shared `bruteForce.store`), or sessions, lockouts and refresh rotation do not hold across - them. A shared store needs atomic `take` and `add` for the one-time steps (refresh tokens, - recovery codes, the pending TOTP secret). -- **Refresh rotation is atomic across processes only with a store that has an atomic `add`.** - Without it, two uses of one token at the same moment in two processes may both pass. A - client that sends the same refresh token twice at once loses its family (by design: it - looks like theft). + them. Its `add`, `take` and `increment` must be atomic across them, as the `Store` + interface asks. +- **Refresh rotation is atomic across processes**, by the store's `add`: of two uses of one + token at the same moment, in any processes, one passes. A client that sends the same + refresh token twice at once loses its family (by design: it looks like theft). - **Access tokens cannot be revoked** before they expire; keep `expiresIn` short. Sessions can: logout deletes them on the server. - **No "log out everywhere"**: sessions are not indexed by user. After a password change, diff --git a/packages/auth/index.d.ts b/packages/auth/index.d.ts index 5e46d91..8b93418 100644 --- a/packages/auth/index.d.ts +++ b/packages/auth/index.d.ts @@ -1,4 +1,4 @@ -import type { Context, Plugin, ProblemBody, Schema, Security } from "@vxnsin/inkan"; +import type { Context, memoryStore as coreMemoryStore, Plugin, ProblemBody, Schema, Security, Store as CoreStore } from "@vxnsin/inkan"; import type { AsymmetricKey, Duration, JwksOptions, JwtAlgorithm, JwtPayload, Secret } from "./lib/jwt.js"; import type { OAuthTokens, OAuthUser, Provider } from "./lib/oauth.js"; import type { ScryptParams } from "./lib/password.js"; @@ -54,26 +54,12 @@ export interface UserSource { updatePasswordHash?(id: any, hash: string): Awaitable; } -/** - * Any key-value store with a time to live (Redis, a database table, …). A store shared by - * several processes should have `take` and `add`, each one atomic step of the store (Redis - * GETDEL and SET … NX EX); without them both are a read and a write, atomic in one process only. - */ -export interface AuthStore { - get(key: string): Awaitable; - set(key: string, value: unknown, ttlSeconds: number): Awaitable; - delete(key: string): Awaitable; - /** Reads and removes in one step: the value, or undefined when there was none. */ - take?(key: string): Awaitable; - /** Writes only when the key is free, and says whether it did. */ - add?(key: string, value: unknown, ttlSeconds: number): Awaitable; -} - -export function memoryStore(options?: { max?: number; clock?: () => number }): Required & { - take(key: string): unknown; - add(key: string, value: unknown, ttlSeconds: number): boolean; - readonly size: number; -}; +/** @deprecated Use `Store` from "@vxnsin/inkan"; TTLs are in milliseconds since 0.2. */ +export type Store = CoreStore; +/** @deprecated Use `Store` from "@vxnsin/inkan"; TTLs are in milliseconds since 0.2. */ +export type AuthStore = CoreStore; +/** @deprecated Use `memoryStore` from "@vxnsin/inkan": this is the same function, re-exported. */ +export declare const memoryStore: typeof coreMemoryStore; export interface SessionOptions { /** Default "sid". */ @@ -86,7 +72,7 @@ export interface SessionOptions { sameSite?: "Lax" | "Strict" | "None"; domain?: string; /** Sessions only; default: the plugin's `store`. */ - store?: AuthStore; + store?: CoreStore; } export type JwtIssueOptions = { @@ -146,8 +132,11 @@ export interface AuthOptions { /** At least 32 bytes. Signs session cookies, seals OAuth cookies, signs own JWTs. A list rotates: the first signs, all verify. */ secret?: string | Uint8Array | readonly (string | Uint8Array)[]; users?: UserSource; - /** Sessions, refresh tokens, login counters, TOTP steps. Default: memoryStore(), per process. */ - store?: AuthStore; + /** + * Sessions, refresh tokens and TOTP steps: any inkan `Store` (Redis, a database, …), whose + * `add` and `take` are atomic across processes. Default: `memoryStore()`, per process. + */ + store?: CoreStore; /** Cookie sessions; on by default. */ session?: SessionOptions | false; jwt?: JwtOptions; @@ -179,11 +168,15 @@ export interface AuthOptions { /** * Failed logins before an exponential lockout: `freeAttempts` (5) per login from one address, * `loginFreeAttempts` (100) per login from everywhere, `ipFreeAttempts` (50) per address (an - * IPv6 address by its /64). Counted in `store`, by default a memoryStore of its own. false turns it off. + * IPv6 address by its /64). Counted with `increment` in `store`, a `Store` of its own, by default a + * memoryStore; `window` (900) and `maxDelay` (900) are seconds. A count starts over `window` after + * its first attempt; the lockout escalates (1s, 2s, … maxDelay) and is forgotten only after + * `window` of quiet past the last lock. `window` must be at least `maxDelay`. A success clears the + * login's counters, never the address's. false turns it off. */ bruteForce?: | false - | { freeAttempts?: number; loginFreeAttempts?: number; ipFreeAttempts?: number; maxDelay?: number; window?: number; store?: AuthStore }; + | { freeAttempts?: number; loginFreeAttempts?: number; ipFreeAttempts?: number; maxDelay?: number; window?: number; store?: CoreStore }; /** Milliseconds now; for tests. */ clock?: () => number; } diff --git a/packages/auth/index.js b/packages/auth/index.js index c6866d6..c37569d 100644 --- a/packages/auth/index.js +++ b/packages/auth/index.js @@ -1,7 +1,8 @@ // @inkanjs/auth: the inkan plugin, and the building blocks it is made of for apps that need // them on their own (password reset tokens, tokens for other services, account linking). export { auth } from "./lib/plugin.js"; -export { memoryStore } from "./lib/store.js"; +/** @deprecated the core's `memoryStore` from @vxnsin/inkan, re-exported for 0.1 code. */ +export { memoryStore } from "@vxnsin/inkan"; export { AuthError } from "./lib/errors.js"; export { signJwt, verifyJwt, jwksSource } from "./lib/jwt.js"; export { hashPassword, verifyPassword, needsRehash } from "./lib/password.js"; diff --git a/packages/auth/lib/plugin.js b/packages/auth/lib/plugin.js index 0a8af16..09c5b15 100644 --- a/packages/auth/lib/plugin.js +++ b/packages/auth/lib/plugin.js @@ -7,14 +7,14 @@ // for a scheme this plugin cannot verify is refused, never waved through. import { Buffer } from "node:buffer"; import { randomBytes } from "node:crypto"; -import { HttpProblem, plugin, reply, t } from "@vxnsin/inkan"; +import { HttpProblem, memoryStore, plugin, problem, reply, t } from "@vxnsin/inkan"; import { AuthError } from "./errors.js"; import { isObject, secureUrl } from "./http.js"; import { jwksSource, signJwt, verifyJwt } from "./jwt.js"; import { deriveKeys, readSessionId, seal, signSessionId, unseal } from "./keys.js"; import { buildAuthorizeUrl, exchangeCode, github, google, pkcePair, randomState } from "./oauth.js"; import { hashPassword, needsRehash, verifyPassword } from "./password.js"; -import { keyLock, memoryStore } from "./store.js"; +import { keyLock } from "./store.js"; import { hashToken, randomToken, safeEqual } from "./tokens.js"; import { base32Encode, generateTotpSecret, otpauthUrl, verifyTotp } from "./totp.js"; @@ -55,12 +55,13 @@ function seconds(value, what) { return Number(m[1]) * { s: 1, m: 60, h: 3600, d: 86400 }[m[2]]; } +const STORE_METHODS = ["get", "set", "delete", "add", "take", "increment"]; + +/** An inkan `Store` (TTLs in ms; add, take and increment atomic), or a TypeError that says what is missing. */ function checkStore(store, what) { - if (!isObject(store) || !fn(store.get) || !fn(store.set) || !fn(store.delete)) { - throw new TypeError(`auth: ${what} must have get(key), set(key, value, ttlSeconds) and delete(key)`); - } - if ((store.take !== undefined && !fn(store.take)) || (store.add !== undefined && !fn(store.add))) { - throw new TypeError(`auth: ${what}: take(key) and add(key, value, ttlSeconds) are functions when given`); + const missing = isObject(store) ? STORE_METHODS.filter((m) => !fn(store[m])) : STORE_METHODS; + if (missing.length) { + throw new TypeError(`auth: ${what} must be an inkan Store (get, set, delete, add, take, increment; TTLs in ms), missing: ${missing.join(", ")}`); } return store; } @@ -72,7 +73,7 @@ function configure(o) { const keys = o.secret === undefined ? undefined : deriveKeys(o.secret); const users = o.users ?? {}; if (!isObject(users)) throw new TypeError("auth: users must be an object with find and byId"); - const store = checkStore(o.store ?? memoryStore({ clock }), "store"); + const store = checkStore(o.store ?? memoryStore({ clock, max: 100_000 }), "store"); const c = { clock, keys, @@ -254,6 +255,9 @@ function configure(o) { for (const k of ["freeAttempts", "loginFreeAttempts", "ipFreeAttempts", "maxDelay", "window"]) { if (!Number.isInteger(b[k]) || b[k] < 1) throw new TypeError(`auth: bruteForce.${k} must be a positive integer`); } + // the escalation is forgotten `window` after a lock ends: shorter than a lock, it would be + // forgotten while the lock still holds + if (b.window < b.maxDelay) throw new TypeError("auth: bruteForce.window must be at least bruteForce.maxDelay"); // counters live apart from sessions and refresh tokens: a flood of made-up logins fills // (and evicts within) this store only b.store = checkStore(b.store ?? memoryStore({ clock, max: 50_000 }), "bruteForce.store"); @@ -276,7 +280,7 @@ function slots(max, waitMs) { let active = 0; /** @type {{ go: () => void, timer: NodeJS.Timeout }[]} */ const queue = []; - const busy = () => new HttpProblem(503, "busy", "Too many logins at once; try again in a moment", {}, { "retry-after": "1" }); + const busy = () => problem(503, "busy", { detail: "Too many logins at once; try again in a moment", headers: { "retry-after": "1" } }); return async function run(work) { if (active < max) active++; else { @@ -332,7 +336,7 @@ const forbidden = (detail) => new HttpProblem(403, "forbidden", detail); */ export function auth(options) { const c = configure(options); - const p = plugin((app) => install(app, c), { name: "auth", shared: true, inkan: ">=0.7.0 <0.8.0" }); + const p = plugin((app) => install(app, c), { name: "auth", shared: true, inkan: ">=0.8.0 <1.0.0" }); Object.defineProperty(p, "session", { value: c.sessionScheme, enumerable: true }); return p; } @@ -340,26 +344,12 @@ export function auth(options) { function install(app, c) { const { clock, keys, users, store, session } = c; const lock = keyLock(); - /** Reads and removes in one step: the store's own take, or a read and a delete under the process lock. */ - const take = (s, key) => - fn(s.take) - ? s.take(key) - : lock("take:" + key, async () => { - const v = await s.get(key); - if (v !== undefined) await s.delete(key); - return v; - }); - /** Writes only when the key is free, true when it did: the store's own add, or a read and a write under the process lock. */ - const add = async (s, key, value, ttl) => - fn(s.add) - ? (await s.add(key, value, ttl)) === true - : lock("add:" + key, async () => { - const v = await s.get(key); - if (v !== undefined && v !== null) return false; - await s.set(key, value, ttl); - return true; - }); - const ttlUntil = (ms) => Math.max(1, Math.ceil((ms - clock()) / 1000)); + /** Writes only when the key is free, true when it did: one atomic step of the store. */ + const add = async (s, key, value, ttl) => (await s.add(key, value, ttl)) === true; + /** Reads and removes in one step: one atomic step of the store. */ + const take = (s, key) => s.take(key); + /** The TTL in ms of a record that ends at `ms`. */ + const ttlUntil = (ms) => Math.max(1, Math.ceil(ms - clock())); /** @type {WeakMap} */ const states = new WeakMap(); const stateOf = (ctx) => { @@ -394,7 +384,7 @@ function install(app, c) { if (old) await session.store.delete(sessionKey(old)); const id = randomToken(32); const now = clock(); - await session.store.set(sessionKey(id), { u: user.id, iat: now, exp: now + session.maxAge * 1000 }, session.maxAge); + await session.store.set(sessionKey(id), { u: user.id, iat: now, exp: now + session.maxAge * 1000 }, session.maxAge * 1000); ctx.setCookie(session.cookie, signSessionId(keys, id), { maxAge: session.maxAge, path: "/", @@ -588,7 +578,7 @@ function install(app, c) { } if (failure && (schemes.length > 0 || needsUser) && !lenient) { const code = publicCode(failure.err); - throw new HttpProblem(401, code, DETAIL[code], {}, challenges(schemes.length ? schemes : [failure.s], c, code)); + throw problem(401, code, { detail: DETAIL[code], headers: challenges(schemes.length ? schemes : [failure.s], c, code) }); } // missing credentials on a route with security: the core answers 401 right after the onRequest hooks if (needsUser && !schemes.length) throw new HttpProblem(401, "unauthorized", "Log in first"); @@ -603,7 +593,7 @@ function install(app, c) { // Awaited before the handler, so hooks after the input check and the handler see the user // itself; a route outside needsAuth never touches the store. - app.decorateRequest("user", authenticate, { before: "handler", when: needsAuth }); + app.decorateRequest("user", authenticate, { resolve: "preHandler", when: needsAuth }); app.decorateRequest("auth", (ctx) => { const st = stateOf(ctx); return Object.freeze({ @@ -648,7 +638,7 @@ function install(app, c) { // ---- OpenAPI ---- const sessionSchemeName = session ? `${session.cookie}-cookie`.replace(/[^\w.-]/g, "_") : undefined; - app.describe((op, route, components) => { + app.onOperation((op, route, components) => { const schemes = components.securitySchemes ?? {}; if (c.jwt && schemes.bearer) { schemes.bearer.bearerFormat ??= "JWT"; @@ -679,9 +669,21 @@ function install(app, c) { // from everywhere. The login from one address locks fast (freeAttempts); the login as such // only after many more failures (loginFreeAttempts), which takes many addresses, so a // stranger who knows a login cannot lock its owner out from one machine. Each attempt is - // counted before the password is checked, so a burst of parallel guesses cannot all slip - // through; a success gives the address its attempt back and clears the login's counters. + // counted (an atomic increment) before the password is checked, and a counter over its + // free attempts claims its lock with an atomic add: of a burst of parallel guesses only the + // one that claims the lock goes on, in every process. + // + // Escalation: next to each counter `key:s` holds its level, the number of locks claimed. + // The count itself is a fixed window (it starts over `window` after its first attempt), but + // once a counter has a level, every attempt after a lock claims the next one, twice as long + // (1s, 2s, 4s, … up to maxDelay), whatever the count. The level is kept `window` past the + // end of the last lock: a guesser who keeps going, however slowly, stays at one attempt per + // maxDelay; only `window` of quiet after a lock forgets it. A success clears the login's + // counters and levels, never the address's: else a guesser would log into an account of + // their own between guesses. const bf = c.bruteForce; + const lockOf = (key) => key + ":lock"; + const levelOf = (key) => key + ":s"; const guard = bf && { keysFor(login, ip) { const l = hashToken(login.trim().toLowerCase()); @@ -693,45 +695,54 @@ function install(app, c) { const now = clock(); let until = 0; for (const [key, free] of keyList) { - if (until) break; - await lock(key, async () => { - const rec = (await bf.store.get(key)) ?? { n: 0, until: 0 }; - if (rec.until > now) return void (until = rec.until); - rec.n += 1; - const over = rec.n - free; - const delay = over > 0 ? Math.min(2 ** (over - 1), bf.maxDelay) : 0; - rec.until = delay ? now + delay * 1000 : 0; - await bf.store.set(key, rec, bf.window + delay); - }); + const locked = await bf.store.get(lockOf(key)); + if (typeof locked === "number" && locked > now) { + until = locked; + break; + } + const s = await bf.store.get(levelOf(key)); + const level = typeof s === "number" && s > 0 ? Math.min(s, 64) : 0; + const n = await bf.store.increment(key, 1, bf.window * 1000); + if (level === 0 && n <= free) continue; + const delay = Math.min(2 ** level, bf.maxDelay) * 1000; + // the attempt that claims the lock goes on; one that finds it claimed is turned away + if (!(await bf.store.add(lockOf(key), now + delay, delay))) { + const held = await bf.store.get(lockOf(key)); + until = typeof held === "number" && held > now ? held : now + delay; + break; + } + await bf.store.set(levelOf(key), level + 1, delay + bf.window * 1000); } if (until) { const wait = Math.max(1, Math.ceil((until - now) / 1000)); - throw new HttpProblem(429, "too-many-attempts", `Too many failed logins; try again in ${wait}s`, {}, { "retry-after": String(wait) }); + throw problem(429, "too-many-attempts", { detail: `Too many failed logins; try again in ${wait}s`, headers: { "retry-after": String(wait) } }); } }, - /** A success: the address gets its attempt back, the login's counters go. */ + /** A success: the login's counters go, the address's stay (its attempt counted as one). */ async succeeded(keys) { - const [ipKey, free] = keys.ip; - await lock(ipKey, async () => { - const rec = await bf.store.get(ipKey); - if (!isObject(rec) || typeof rec.n !== "number") return; - rec.n = Math.max(0, rec.n - 1); - if (rec.n <= free) rec.until = 0; - await bf.store.set(ipKey, rec, bf.window); - }); - await bf.store.delete(keys.loginIp[0]); - await bf.store.delete(keys.login[0]); + await guard.clear(keys.loginIp[0]); + await guard.clear(keys.login[0]); + }, + /** Forgets a counter, its lock and its level. */ + async clear(key) { + await bf.store.delete(key); + await bf.store.delete(lockOf(key)); + await bf.store.delete(levelOf(key)); }, }; - /** Uses a TOTP code once: the matched step is kept, so the same code never works twice. */ + /** + * Uses a TOTP code once: its step is claimed with an atomic add, so the same code never + * works twice, in any process; the last step is kept too, so an older code is refused. + */ async function useTotpCode(user, secret, code) { const key = "t:" + hashToken(String(user.id)); return lock(key, async () => { const last = await store.get(key); const step = verifyTotp(code, secret, { now: clock(), window: c.totp?.window ?? 1, lastUsedStep: typeof last === "number" ? last : null }); if (step === null) return null; - await store.set(key, step, 600); + if (!(await add(store, key + ":" + step, true, 600_000))) return null; + if (!(typeof last === "number" && last >= step)) await store.set(key, step, 600_000); return step; }); } @@ -751,7 +762,7 @@ function install(app, c) { const list = Array.isArray(user.recoveryCodes) ? user.recoveryCodes.filter((h) => typeof h === "string") : []; const h = recoveryHash(norm); if (!list.some((x) => safeEqual(x, h))) return false; - if (!(await add(store, "rc:" + hashToken(`${user.id}\n${h}`), true, 86400))) return false; + if (!(await add(store, "rc:" + hashToken(`${user.id}\n${h}`), true, 86_400_000))) return false; await users.setRecoveryCodes(user.id, list.filter((x) => x !== h)); return true; } @@ -777,7 +788,7 @@ function install(app, c) { if (guard) await guard.attempt([key]); const ok = stored !== undefined && (await hashSlot(() => verifyPassword(password, stored))); if (!ok) throw new HttpProblem(403, "reauth-failed", "The password is wrong"); - if (guard) await bf.store.delete(key[0]); + if (guard) await guard.clear(key[0]); return; } if (strict && stored) throw new HttpProblem(403, "reauth-required", "Confirm this with your password"); @@ -817,7 +828,7 @@ function install(app, c) { const now = clock(); if (!family) { family = { id: randomToken(16), at: now, exp: now + c.issue.refreshMaxAge * 1000 }; - const made = await add(store, "f:" + family.id, { u: user.id, revoked: false, at: family.at, exp: family.exp }, c.issue.refreshMaxAge); + const made = await add(store, "f:" + family.id, { u: user.id, revoked: false, at: family.at, exp: family.exp }, c.issue.refreshMaxAge * 1000); if (!made) throw new Error("auth: a new refresh token family found its id taken"); } const [key] = keys; @@ -1003,7 +1014,7 @@ function install(app, c) { await reauthenticate(ctx, user, ctx.body?.password, false); const secret = generateTotpSecret(); // sealed: a leaked store gives nobody the secret of an authenticator about to be trusted - await store.set(pendingKey(user), seal(keys, { s: secret, u: user.id }, "totp-pending"), 600); + await store.set(pendingKey(user), seal(keys, { s: secret, u: user.id }, "totp-pending"), 600_000); const account = String(c.totp.account(user)).replace(/:/g, "") || String(user.id); return { secret, otpauthUrl: otpauthUrl({ secret, account, issuer: c.totp.issuer || undefined }) }; }, diff --git a/packages/auth/lib/store.js b/packages/auth/lib/store.js index 5616275..ad26aab 100644 --- a/packages/auth/lib/store.js +++ b/packages/auth/lib/store.js @@ -1,79 +1,11 @@ -// Where the plugin keeps what must outlive a request: sessions, refresh tokens, login -// counters, the last TOTP step. Any key-value store with a time to live will do; the one -// here lives in memory, per process. -// -// The contract: get(key), set(key, value, ttlSeconds), delete(key), and two steps that must -// be atomic in a store shared by several processes: take(key), which reads and removes in -// one step, and add(key, value, ttlSeconds), which writes only when the key is free and says -// whether it did (Redis: GETDEL, and SET … NX EX). Without them the plugin falls back to a -// read and a write, which is atomic within one process only. - -/** - * A store in memory. Values are copied in and out (structuredClone), as a store over the - * network would, so nobody changes a session by holding on to an object. At most `max` - * entries: past that the oldest go first, so a flood of made-up logins cannot fill the heap. - * @param {{ max?: number, clock?: () => number }} [options] - */ -export function memoryStore({ max = 100_000, clock = Date.now } = {}) { - if (!Number.isInteger(max) || max < 1) throw new RangeError("memoryStore: max must be a positive integer"); - /** @type {Map} */ - const entries = new Map(); - let sweepAt = clock() + 60_000; - - const live = (key) => { - const e = entries.get(key); - if (!e) return undefined; - if (e.exp <= clock()) { - entries.delete(key); - return undefined; - } - return e; - }; - const sweep = () => { - const now = clock(); - if (now < sweepAt) return; - sweepAt = now + 60_000; - for (const [k, e] of entries) if (e.exp <= now) entries.delete(k); - }; - - const set = (key, value, ttl) => { - if (!(typeof ttl === "number" && ttl > 0)) throw new RangeError("memoryStore: ttl must be a positive number of seconds"); - sweep(); - entries.delete(key); // re-inserted at the end: the oldest stay first in line - entries.set(key, { value: structuredClone(value), exp: clock() + ttl * 1000 }); - while (entries.size > max) entries.delete(entries.keys().next().value); - }; - - return { - get(key) { - const e = live(key); - return e === undefined ? undefined : structuredClone(e.value); - }, - set, - delete(key) { - entries.delete(key); - }, - /** Writes only when the key is free; true when it did. */ - add(key, value, ttl) { - if (live(key) !== undefined) return false; - set(key, value, ttl); - return true; - }, - /** Reads and removes in one step. */ - take(key) { - const e = live(key); - entries.delete(key); - return e === undefined ? undefined : structuredClone(e.value); - }, - get size() { - return entries.size; - }, - }; -} +// What the plugin keeps between requests (sessions, refresh tokens, login counters, the last +// TOTP step) lives in an inkan `Store`: `memoryStore()` from @vxnsin/inkan by default, or any +// adapter (Redis, a database) whose add, take and increment are atomic across processes. +// What is left here is the lock for the few steps that read and then write within one process. /** * Runs `fn` for one key at a time in this process, so a read and the write after it are not - * interleaved with another request's. Across processes the store has to do that itself. + * interleaved with another request's. Across processes the store's atomic steps do that. */ export function keyLock() { /** @type {Map>} */ diff --git a/packages/auth/package.json b/packages/auth/package.json index 3f00b0d..58aa551 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -1,6 +1,6 @@ { "name": "@inkanjs/auth", - "version": "0.1.0", + "version": "0.2.0", "description": "Authentication for inkan without dependencies: sessions, JWTs, API keys, passwords, TOTP and OAuth on node:crypto alone.", "type": "module", "license": "MIT", @@ -41,13 +41,13 @@ "node": ">=22.18" }, "peerDependencies": { - "@vxnsin/inkan": ">=0.7.0 <0.8.0" + "@vxnsin/inkan": ">=0.8.0 <1.0.0" }, "devDependencies": { - "@vxnsin/inkan": "^0.7.0" + "@vxnsin/inkan": "^0.8.0" }, "scripts": { - "test": "node --test \"test/*.test.ts\"" + "test": "node --import ../../scripts/no-deprecations.js --test \"test/*.test.ts\"" }, "publishConfig": { "access": "public" diff --git a/packages/auth/test/plugin-docs.test.ts b/packages/auth/test/plugin-docs.test.ts index 004f213..9e970bf 100644 --- a/packages/auth/test/plugin-docs.test.ts +++ b/packages/auth/test/plugin-docs.test.ts @@ -79,7 +79,7 @@ test("inkan check passes with failureRedirect too, and under a prefix", async () password: FAST, oauth: { baseUrl: "https://app.example", github: { clientId: "c" }, onLogin: () => undefined, failureRedirect: "/login" }, }); - const app = inkan({ log: false }).register(plugin, { prefix: "/api" }); + const app = inkan({ log: false }).register(plugin, {}, { prefix: "/api" }); const report = await app.check(); assert.ok(report.ok, JSON.stringify(report.results.filter((r) => !r.ok))); assert.ok(report.results.some((r) => r.path === "/api/auth/oauth/:provider/callback")); @@ -91,7 +91,7 @@ test("inkan check with an example account: logs in, then /me and /logout with th for (const prefix of ["", "/api"]) { const users = makeUsers(); const plugin = auth({ secret: SECRET, users: users.source, password: FAST, example: { login: "bob@example.com", password: "correct horse" } }); - const app = inkan({ log: false }).register(plugin, { prefix }); + const app = inkan({ log: false }).register(plugin, {}, { prefix }); const report = await app.check(); assert.ok(report.ok, JSON.stringify(report.results.filter((r) => !r.ok))); const names = report.results.map((r) => `${r.method} ${r.path} > ${r.example}`); diff --git a/packages/auth/test/plugin-session.test.ts b/packages/auth/test/plugin-session.test.ts index a6674ef..5b38b88 100644 --- a/packages/auth/test/plugin-session.test.ts +++ b/packages/auth/test/plugin-session.test.ts @@ -1,7 +1,9 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { auth, memoryStore } from "../index.js"; +import { memoryStore } from "@vxnsin/inkan"; +import { auth } from "../index.js"; import { OTHER_SECRET, SECRET, clock, cookieOf, login, makeApp, makeUsers, setCookieOf } from "./app.ts"; +import { countingStore } from "./store.ts"; const withSid = (sid: string | undefined, headers: Record = {}) => ({ cookie: `sid=${sid}`, ...headers }); @@ -342,9 +344,105 @@ test("brute force: many failures from many addresses lock the login itself", asy assert.equal((await tryLogin(app, "ada@example.com", "correct horse", "203.0.113.9")).status, 429); }); -test("brute force: successful logins do not count against the address", async () => { - const { app } = makeApp({ bruteForce: { ipFreeAttempts: 3 } }, makeUsers(), { trustProxy: true }); - for (let i = 0; i < 8; i++) assert.equal((await tryLogin(app, "bob@example.com", "correct horse", "198.51.100.1")).status, 200, `login ${i + 1}`); +test("brute force: a successful login does not give the address its attempts back", async () => { + // else a guesser would log into an account of their own between guesses and never be locked + const start = clock.t; + try { + const { app } = makeApp({ bruteForce: { ipFreeAttempts: 3 } }, makeUsers(), { trustProxy: true }); + const ip = "198.51.100.1"; + assert.equal((await tryLogin(app, "bob@example.com", "correct horse", ip)).status, 200); + assert.equal((await tryLogin(app, "ada@example.com", "guess", ip)).status, 401); + assert.equal((await tryLogin(app, "bob@example.com", "correct horse", ip)).status, 200); + assert.equal((await tryLogin(app, "ada@example.com", "guess", ip)).status, 401, "the attempt that claims the lock still runs"); + assert.equal((await tryLogin(app, "bob@example.com", "correct horse", ip)).status, 429, "the address is locked, own account or not"); + } finally { + clock.t = start; + } +}); + +/** One attempt every `every` seconds for `seconds`; how many were let through (not 429). */ +async function slowly(attempt: () => Promise<{ status: number }>, every: number, seconds: number) { + let through = 0; + for (let t = 0; t < seconds; t += every) { + if ((await attempt()).status !== 429) through++; + clock.t += every * 1000; + } + return through; +} + +/** Fails until a lock of `maxDelay` seconds is claimed, waiting out each lock; the clock ends where that lock ends. */ +async function escalate(attempt: () => Promise<{ status: number; headers: Record }>, maxDelay: number) { + for (let i = 0; i < 100; i++) { + const res = await attempt(); + if (res.status === 429) { + const wait = Number(res.headers["retry-after"]); + clock.t += wait * 1000; + if (wait === maxDelay) return; + } + } + assert.fail("never reached maxDelay"); +} + +test("brute force: a slow guesser stays at one attempt per maxDelay once escalated, for hours", async () => { + const start = clock.t; + try { + const { app } = makeApp({ bruteForce: { freeAttempts: 5, maxDelay: 900, window: 900 } }); + const guess = () => app.inject({ method: "POST", url: "/auth/login", body: { login: "ada@example.com", password: "guess" } }); + await escalate(guess, 900); + // one guess every maxDelay/2 for four hours: a fixed window would forget the failures after + // each window and give the free attempts back; the escalation does not + const through = await slowly(guess, 450, 4 * 3600); + assert.ok(through <= 4 * 3600 / 900 + 1, `${through} guesses in four hours`); + assert.ok(through >= 4 * 3600 / 900 - 1, `${through} guesses in four hours: one per maxDelay still gets through`); + } finally { + clock.t = start; + } +}); + +test("brute force: quiet for `window` after the last lock ends forgets the escalation", async () => { + const start = clock.t; + try { + const { app } = makeApp({ bruteForce: { freeAttempts: 3, maxDelay: 60, window: 600 } }); + const guess = () => app.inject({ method: "POST", url: "/auth/login", body: { login: "ada@example.com", password: "guess" } }); + await escalate(guess, 60); + assert.equal((await guess()).status, 401, "the lock is over: this one claims the next"); + let res = await guess(); + assert.equal(res.headers["retry-after"], "60", "still at maxDelay"); + clock.t += 59_000; + res = await guess(); + assert.equal(res.status, 429, "a guess within the window keeps the level"); + clock.t += 1000 + 599_000; + assert.equal((await guess()).status, 401, "window minus a second of quiet: this one claims the next lock"); + res = await guess(); + assert.equal(res.headers["retry-after"], "60", "at the same level"); + clock.t += 60_000 + 600_000; // the lock, then the whole window + for (let i = 0; i < 4; i++) assert.equal((await guess()).status, 401, `free again: ${i + 1}`); + res = await guess(); + assert.equal(res.status, 429); + assert.equal(res.headers["retry-after"], "1", "and the escalation starts over"); + } finally { + clock.t = start; + } +}); + +test("brute force: the owner's login clears the escalation of the login, not of the address", async () => { + const start = clock.t; + try { + const { app } = makeApp({ bruteForce: { freeAttempts: 3, ipFreeAttempts: 6, maxDelay: 60, window: 600 } }, makeUsers(), { trustProxy: true }); + const ip = "198.51.100.9"; + await escalate(() => tryLogin(app, "ada@example.com", "guess", ip), 60); + assert.equal((await tryLogin(app, "ada@example.com", "correct horse", "203.0.113.1")).status, 200, "the owner, elsewhere"); + assert.equal((await tryLogin(app, "ada@example.com", "guess", "203.0.113.1")).status, 401, "and the login's own counter starts over"); + await tryLogin(app, "bob@example.com", "correct horse", ip); // claims the address's next lock + const res = await tryLogin(app, "bob@example.com", "correct horse", ip); + assert.equal(res.status, 429, "the address stays escalated: a success does not reset it"); + } finally { + clock.t = start; + } +}); + +test("brute force: window shorter than maxDelay is refused", () => { + assert.throws(() => makeApp({ bruteForce: { maxDelay: 900, window: 600 } }), /window/); }); test("brute force: an IPv6 address counts by its /64", async () => { @@ -355,17 +453,17 @@ test("brute force: an IPv6 address counts by its /64", async () => { }); test("brute force: counters live in a store of their own; a locked address writes no more of them", async () => { - const store = memoryStore({ max: 20 }); - const counters = memoryStore(); - const { app } = makeApp({ store, bruteForce: { ipFreeAttempts: 2, store: counters } }, makeUsers(), { trustProxy: true }); + const store = countingStore(memoryStore({ max: 20 })); + const counters = countingStore(); + const { app } = makeApp({ store: store.store, bruteForce: { ipFreeAttempts: 2, store: counters.store } }, makeUsers(), { trustProxy: true }); const { sid } = await login(app); - const before = store.size; + const before = store.keys.size; for (let i = 0; i < 3; i++) await tryLogin(app, `u${i}@example.com`, "x", "198.51.100.1"); - const size = counters.size; + const size = counters.keys.size; for (let i = 0; i < 50; i++) assert.equal((await tryLogin(app, `flood${i}@example.com`, "x", "198.51.100.1")).status, 429); - assert.equal(counters.size, size, "no per-login record once the address is locked"); + assert.equal(counters.keys.size, size, "no per-login record once the address is locked"); for (let i = 0; i < 50; i++) await tryLogin(app, `flood${i}@example.com`, "x", `203.0.113.${i}`); - assert.equal(store.size, before, "the session store holds no counters"); + assert.equal(store.keys.size, before, "the session store holds no counters"); assert.equal((await app.inject({ url: "/account", headers: withSid(sid) })).status, 200, "the session survived the flood"); }); diff --git a/packages/auth/test/plugin-tokens.test.ts b/packages/auth/test/plugin-tokens.test.ts index 9203cea..cf1eb53 100644 --- a/packages/auth/test/plugin-tokens.test.ts +++ b/packages/auth/test/plugin-tokens.test.ts @@ -1,7 +1,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { inkan } from "@vxnsin/inkan"; -import { auth, memoryStore, verifyJwt, type AuthOptions, type AuthStore } from "../index.js"; +import { inkan, memoryStore, type Store } from "@vxnsin/inkan"; +import { auth, verifyJwt, type AuthOptions } from "../index.js"; import { totp } from "../lib/totp.js"; import { FAST, SECRET, clock, cookieOf, login, makeApp, makeUsers, type User } from "./app.ts"; @@ -134,10 +134,9 @@ test("refresh: two processes sharing a store, the same token at once: only one g test("refresh: the family record is written once, at login, not on every rotation", async () => { const base = memoryStore(); const writes: string[] = []; - const store: AuthStore = { - get: (k) => base.get(k), + const store: Store = { + ...base, set: (k, v, ttl) => (writes.push(k), base.set(k, v, ttl)), - delete: (k) => base.delete(k), add: (k, v, ttl) => (writes.push(k), base.add(k, v, ttl)), }; const { app } = tokenApp(makeUsers(), { store }); @@ -306,7 +305,7 @@ test("TOTP: the secret waiting for its confirmation is sealed in the store", asy let seen = 0; for (const user of [1, 2, 3]) { const { createHash } = await import("node:crypto"); - const v = store.get("p:" + createHash("sha256").update(String(user)).digest("base64url")); + const v = await store.get("p:" + createHash("sha256").update(String(user)).digest("base64url")); if (v === undefined) continue; seen++; assert.equal(typeof v, "string"); diff --git a/packages/auth/test/plugin-types.test.ts b/packages/auth/test/plugin-types.test.ts index cc425b1..a5c636b 100644 --- a/packages/auth/test/plugin-types.test.ts +++ b/packages/auth/test/plugin-types.test.ts @@ -71,7 +71,7 @@ test("the plugin's routes reach the typed client: POST /auth/login, under a pref const none = () => api.post("/auth/nope", {}); const me = () => api.get("/auth/me"); - const prefixed = inkan({ log: false }).register(auth({ secret: SECRET, users: { byId: async (id: string) => db.get(id) }, routes: "/session" }), { prefix: "/api" }); + const prefixed = inkan({ log: false }).register(auth({ secret: SECRET, users: { byId: async (id: string) => db.get(id) }, routes: "/session" }), {}, { prefix: "/api" }); const p = client("http://localhost:1"); const login2 = () => p.post("/api/session/login", { body: { login: "ada", password: "x" } }); // @ts-expect-error routes: false defines none diff --git a/packages/auth/test/store.test.ts b/packages/auth/test/store.test.ts new file mode 100644 index 0000000..39824dc --- /dev/null +++ b/packages/auth/test/store.test.ts @@ -0,0 +1,104 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { inkan, memoryStore, type Store } from "@vxnsin/inkan"; +import { auth, memoryStore as deprecatedMemoryStore, type AuthOptions } from "../index.js"; +import { totp } from "../lib/totp.js"; +import { FAST, SECRET, clock, cookieOf, makeUsers, type User } from "./app.ts"; +import { METHODS, countingStore } from "./store.ts"; + +const same = { origin: "http://localhost" }; + +function app(extra: Partial>, users = makeUsers()) { + const plugin = auth({ secret: SECRET, users: users.source, password: FAST, clock: () => clock.t, totp: true, jwt: { issue: {} }, ...extra }); + const a = inkan({ log: false }).register(plugin).get("/account", { security: plugin.session }, (ctx) => ({ id: ctx.user!.id })); + return { app: a, users }; +} + +/** Every feature that keeps state: sessions, refresh tokens and their reuse, TOTP, brute force. */ +async function everything(a: ReturnType["app"]) { + const post = (url: string, body: unknown, headers: Record = {}) => a.inject({ method: "POST", url, body, headers }); + const login = await post("/auth/login", { login: "ada@example.com", password: "correct horse" }); + assert.equal(login.status, 200); + const sid = cookieOf(login, "sid")!; + const cookie = { cookie: `sid=${sid}`, ...same }; + assert.equal((await a.inject({ url: "/account", headers: { cookie: `sid=${sid}` } })).status, 200); + + const tokens = (await post("/auth/token", { login: "bob@example.com", password: "correct horse" })).body; + const next = await post("/auth/token/refresh", { refreshToken: tokens.refreshToken }); + assert.equal(next.status, 200); + const reused = await post("/auth/token/refresh", { refreshToken: tokens.refreshToken }); + assert.equal(reused.body.type, "token-reused"); + assert.equal((await post("/auth/token/refresh", { refreshToken: next.body.refreshToken })).body.type, "token-revoked", "the family is revoked"); + + const setup = await post("/auth/totp/setup", {}, cookie); + assert.equal(setup.status, 200); + const code = totp(setup.body.secret, { now: clock.t }); + const [one, two] = await Promise.all([post("/auth/totp/confirm", { code }, cookie), post("/auth/totp/confirm", { code }, cookie)]); + assert.equal([one, two].filter((r) => r.status === 200).length, 1, "the code and the pending secret count once"); + + for (let i = 0; i < 5; i++) assert.equal((await post("/auth/login", { login: "bob@example.com", password: "wrong" })).status, 401); + assert.equal((await post("/auth/login", { login: "bob@example.com", password: "wrong" })).status, 401, "the attempt that claims the lock still runs"); + const locked = await post("/auth/login", { login: "bob@example.com", password: "correct horse" }); + assert.equal(locked.status, 429); + + assert.equal((await post("/auth/logout", {}, cookie)).status < 300, true); + assert.equal((await a.inject({ url: "/account", headers: { cookie: `sid=${sid}` } })).status, 401); +} + +test("store: every feature runs on the core's memoryStore(), TTLs in ms", async () => { + const start = clock.t; + try { + const store = memoryStore({ clock: () => clock.t }); + const counters = memoryStore({ clock: () => clock.t }); + await everything(app({ store, bruteForce: { store: counters } }).app); + + // a session lasts its maxAge in the store too: an hour is 3600000 ms there + const counting = countingStore(memoryStore({ clock: () => clock.t })); + const { app: a } = app({ store: counting.store, session: { maxAge: "1h" }, bruteForce: false }); + const res = await a.inject({ method: "POST", url: "/auth/login", body: { login: "ada@example.com", password: "correct horse" } }); + const sid = cookieOf(res, "sid")!; + assert.deepEqual(counting.ttls, [3_600_000]); + clock.t += 3_599_000; + assert.equal((await a.inject({ url: "/account", headers: { cookie: `sid=${sid}` } })).status, 200); + clock.t += 2_000; + const [key] = counting.keys; + assert.equal(await counting.inner.get(key!), undefined, "gone from the store after its TTL"); + } finally { + clock.t = start; + } +}); + +test("store: the plugin takes any Store, and uses nothing but the interface", async () => { + const start = clock.t; + try { + const main = countingStore(); + const counters = countingStore(); + const { app: a } = app({ store: main.store, bruteForce: { store: counters.store } }); + await everything(a); + const allowed = new Set(METHODS); + for (const s of [main, counters]) { + assert.deepEqual([...s.touched].filter((p) => !allowed.has(p)), [], "no property but the six methods"); + for (const ttl of s.ttls) assert.ok(typeof ttl === "number" && ttl >= 1000, `a TTL in ms: ${ttl}`); + } + for (const m of ["get", "set", "delete", "add", "take"] as const) assert.ok(main.calls[m] > 0, `store.${m} is used`); + assert.equal(main.calls.increment, 0, "no counters in the main store"); + assert.ok(counters.calls.increment > 0, "brute-force counters use increment"); + assert.ok(counters.calls.set > 0, "a lock's level is set"); + assert.ok([...counters.setKeys].every((k) => k.endsWith(":s")), "and only levels: counts are never read and written back"); + assert.ok([...counters.keys].every((k) => k.startsWith("b:"))); + assert.ok(![...main.keys].some((k) => k.startsWith("b:")), "no counters in the main store"); + } finally { + clock.t = start; + } +}); + +test("store: one without add, take or increment is refused when the plugin is made", () => { + const { get, set, delete: del } = memoryStore(); + const partial = { get, set, delete: del } as unknown as Store; + assert.throws(() => auth({ secret: SECRET, store: partial }), /store must be an inkan Store .*missing: add, take, increment/); + assert.throws(() => auth({ secret: SECRET, session: false, bruteForce: { store: partial } }), /bruteForce\.store must be an inkan Store/); +}); + +test("store: memoryStore from @inkanjs/auth is the core's, kept for 0.1 code", () => { + assert.equal(deprecatedMemoryStore, memoryStore); +}); diff --git a/packages/auth/test/store.ts b/packages/auth/test/store.ts new file mode 100644 index 0000000..42fe945 --- /dev/null +++ b/packages/auth/test/store.ts @@ -0,0 +1,36 @@ +// A Store that counts what is asked of it: only the six methods of the interface are there, +// and every property the plugin reads is noted, so a test can show it uses nothing else. +import { memoryStore, type Store } from "@vxnsin/inkan"; + +export const METHODS = ["get", "set", "delete", "add", "take", "increment"] as const; + +export function countingStore(inner: Store = memoryStore()) { + const calls = Object.fromEntries(METHODS.map((m) => [m, 0])) as Record<(typeof METHODS)[number], number>; + /** Every key written (set, add, increment). */ + const keys = new Set(); + /** Every key written with set. */ + const setKeys = new Set(); + /** Every TTL given with a set or an add, and with an increment that gives one. */ + const ttls: (number | undefined)[] = []; + /** Every property read on the store. */ + const touched = new Set(); + const wrote = (key: string, ttl: number | undefined) => { + keys.add(key); + ttls.push(ttl); + }; + const plain: Store = { + get: (key) => (calls.get++, inner.get(key)), + set: (key, value, ttl) => (calls.set++, setKeys.add(key), wrote(key, ttl), inner.set(key, value, ttl)), + delete: (key) => (calls.delete++, inner.delete(key)), + add: (key, value, ttl) => (calls.add++, wrote(key, ttl), inner.add(key, value, ttl)), + take: (key) => (calls.take++, inner.take(key)), + increment: (key, by, ttl) => (calls.increment++, keys.add(key), ttl !== undefined && ttls.push(ttl), inner.increment(key, by, ttl)), + }; + const store = new Proxy(plain, { + get(target, prop, receiver) { + touched.add(prop); + return Reflect.get(target, prop, receiver); + }, + }); + return { store, inner, calls, keys, setKeys, ttls, touched }; +} diff --git a/packages/circuit-breaker/CHANGELOG.md b/packages/circuit-breaker/CHANGELOG.md index 8e95ec7..95ba2d5 100644 --- a/packages/circuit-breaker/CHANGELOG.md +++ b/packages/circuit-breaker/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.2.0 + +- Works with inkan 0.8 (`inkan: ">=0.8.0 <1.0.0"`). +- The 503 of an open circuit is made with `problem(status, type, { detail, extra, headers })`, + and the README shows that form too. + ## 0.1.0 - `circuit(fn, options)`: wraps an outgoing call; while open it rejects at once with diff --git a/packages/circuit-breaker/README.md b/packages/circuit-breaker/README.md index 7e74175..4c39088 100644 --- a/packages/circuit-breaker/README.md +++ b/packages/circuit-breaker/README.md @@ -21,7 +21,7 @@ The common case: a service you call that sometimes falls over. Wrap the call onc use the wrapped function as before. ```ts -import { HttpProblem, inkan, problem } from "@vxnsin/inkan"; +import { inkan, problem } from "@vxnsin/inkan"; import { circuit, CircuitOpenError } from "@inkanjs/circuit-breaker"; const rates = circuit((currency: string) => fetch(`https://rates.example/${currency}`, { signal: AbortSignal.timeout(2000) }), { @@ -36,7 +36,7 @@ const app = inkan().get("/prices/:currency", async (ctx) => { if (!res.ok) throw problem(502, "rates-failed"); return await res.json(); } catch (err) { - if (err instanceof CircuitOpenError) throw new HttpProblem(503, "rates-unavailable", undefined, {}, { "retry-after": String(err.retryAfter) }); + if (err instanceof CircuitOpenError) throw problem(503, "rates-unavailable", { headers: { "retry-after": String(err.retryAfter) } }); throw err; } }); diff --git a/packages/circuit-breaker/index.js b/packages/circuit-breaker/index.js index a0d2d73..dd349c0 100644 --- a/packages/circuit-breaker/index.js +++ b/packages/circuit-breaker/index.js @@ -8,7 +8,7 @@ // open calls fail at once (CircuitOpenError, or a 503 problem) for `cooldown` ms // half-open one call goes through as a probe: success closes it, failure opens it again -import { HttpProblem, plugin } from "@vxnsin/inkan"; +import { plugin, problem } from "@vxnsin/inkan"; /** 5xx answers and thrown errors. */ const defaultIsFailure = (status, err) => err !== undefined || (typeof status === "number" && status >= 500); @@ -200,11 +200,15 @@ export function breaker(options = {}) { calls.set(ctx, { circuit: c, token: c.enter() }); } catch (err) { if (!(err instanceof CircuitOpenError)) throw err; - throw new HttpProblem(503, "circuit-open", "This service failed too often and is resting; try again later", { retryAfter: err.retryAfter }, { "retry-after": String(err.retryAfter) }); + throw problem(503, "circuit-open", { + detail: "This service failed too often and is resting; try again later", + extra: { retryAfter: err.retryAfter }, + headers: { "retry-after": String(err.retryAfter) }, + }); } }); - app.onProblem((ctx, problem) => void (calls.has(ctx) && problems.set(ctx, problem))); + app.onProblem((ctx, failed) => void (calls.has(ctx) && problems.set(ctx, failed))); app.onSend((ctx, out) => { const call = calls.get(ctx); @@ -214,7 +218,7 @@ export function breaker(options = {}) { call.circuit.settle(call.token, Boolean(s.isFailure(out.status, p && p.status >= 500 ? p : undefined))); }); }, - { name: "circuit-breaker", shared: true, inkan: ">=0.7.0 <0.8.0" }, + { name: "circuit-breaker", shared: true, inkan: ">=0.8.0 <1.0.0" }, ); } diff --git a/packages/circuit-breaker/package.json b/packages/circuit-breaker/package.json index 6e81af1..c4f2d08 100644 --- a/packages/circuit-breaker/package.json +++ b/packages/circuit-breaker/package.json @@ -1,6 +1,6 @@ { "name": "@inkanjs/circuit-breaker", - "version": "0.1.0", + "version": "0.2.0", "description": "Circuit breakers for inkan: stop calling what keeps failing, for incoming routes and outgoing calls.", "type": "module", "license": "MIT", @@ -34,13 +34,13 @@ "node": ">=22.18" }, "peerDependencies": { - "@vxnsin/inkan": ">=0.7.0 <0.8.0" + "@vxnsin/inkan": ">=0.8.0 <1.0.0" }, "devDependencies": { - "@vxnsin/inkan": "^0.7.0" + "@vxnsin/inkan": "^0.8.0" }, "scripts": { - "test": "node --test test.ts" + "test": "node --import ../../scripts/no-deprecations.js --test test.ts" }, "publishConfig": { "access": "public" diff --git a/packages/etag/CHANGELOG.md b/packages/etag/CHANGELOG.md index e67cb69..9bd849e 100644 --- a/packages/etag/CHANGELOG.md +++ b/packages/etag/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.2.0 + +- Works with inkan 0.8 (`inkan: ">=0.8.0 <1.0.0"`). + ## 0.1.0 - `etag()`: strong or weak (`weak: true`) ETags on 2xx answers to GET and HEAD, hashed diff --git a/packages/etag/index.js b/packages/etag/index.js index a6ba14f..2d4ec0d 100644 --- a/packages/etag/index.js +++ b/packages/etag/index.js @@ -63,7 +63,7 @@ export function etag(options = {}) { return { status: 304, headers: h, body: undefined }; }); }, - { name: "etag", shared: true, inkan: ">=0.7.0 <0.8.0" }, + { name: "etag", shared: true, inkan: ">=0.8.0 <1.0.0" }, ); } diff --git a/packages/etag/package.json b/packages/etag/package.json index 5f9432e..a2de487 100644 --- a/packages/etag/package.json +++ b/packages/etag/package.json @@ -1,6 +1,6 @@ { "name": "@inkanjs/etag", - "version": "0.1.0", + "version": "0.2.0", "description": "ETags and 304 answers for inkan: hashed from exactly the bytes that go out, before compression.", "type": "module", "license": "MIT", @@ -34,13 +34,13 @@ "node": ">=22.18" }, "peerDependencies": { - "@vxnsin/inkan": ">=0.7.0 <0.8.0" + "@vxnsin/inkan": ">=0.8.0 <1.0.0" }, "devDependencies": { - "@vxnsin/inkan": "^0.7.0" + "@vxnsin/inkan": "^0.8.0" }, "scripts": { - "test": "node --test test.ts" + "test": "node --import ../../scripts/no-deprecations.js --test test.ts" }, "publishConfig": { "access": "public" diff --git a/packages/idempotency/CHANGELOG.md b/packages/idempotency/CHANGELOG.md index f65e453..edc64e4 100644 --- a/packages/idempotency/CHANGELOG.md +++ b/packages/idempotency/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## 0.2.0 + +- **Any inkan `Store`** (inkan 0.8) for `store`: `memoryStore()` from @vxnsin/inkan, or an + adapter such as @inkanjs/redis. The plugin's own `memoryStore` is gone; `memoryStore` and + `IdempotencyStore` are re-exported from the core, deprecated. +- `add` is required, so two processes never both run a key; a store without it is refused + when the plugin is made. A key the store does not have is `undefined`; anything else under + it (`null` too) is somebody's, never a free key. TTLs stay milliseconds. +- Needs inkan `>=0.8.0 <1.0.0`. +- On the inkan 0.8.0 names, so the plugin prints no deprecation warning: `onOperation`, and + `problem(status, type, { detail, headers })` for the 409 with `retry-after`. What a request + carries from hook to hook is kept in `stateKey` slots instead of side tables. + ## 0.1.0 - `idempotency()`: the same Idempotency-Key replays the first answer diff --git a/packages/idempotency/README.md b/packages/idempotency/README.md index 704864b..6d1322d 100644 --- a/packages/idempotency/README.md +++ b/packages/idempotency/README.md @@ -28,7 +28,7 @@ POST /payments idempotency-key: 5f0c… {"amount":99} → 422 idempotency-ke | option | default | | | --- | --- | --- | -| `store` | `memoryStore()` | where answers are kept (below) | +| `store` | `memoryStore()` | any inkan `Store`: where answers are kept (below) | | `methods` | `["POST", "PATCH"]` | | | `header` | `"Idempotency-Key"` | | | `required` | `false` | a request without the key is a 400 `idempotency-key-missing` | @@ -55,8 +55,8 @@ POST /payments idempotency-key: 5f0c… {"amount":99} → 422 idempotency-ke input breaks the contract, or that an onRequest hook (auth, rate limit) turned away, is not kept, and its key stays free. - **At the same time.** Two requests with the same key in the same process never both run: - the second one is a 409. Across processes that holds when the store has `add` (keep only - when the key is free, as Redis `SET … NX`); `memoryStore` has it. + the second one is a 409. Across processes too: the key is claimed with the store's atomic + `add`. - **Whose key.** Keys are the client's, so they are kept per caller, and the caller is part of the fingerprint too. Without `scope` the caller is the first of: `ctx.user.id` (what @inkanjs/auth sets, so a user is the same caller by cookie or by token), the Authorization @@ -69,16 +69,17 @@ POST /payments idempotency-key: 5f0c… {"amount":99} → 422 idempotency-ke (required with `required`) and the 409 and 422 answers, as problem documents. - Shared plugin: register it on the app for every route, or inside a plugin for its routes. -## A store of your own +## Store + +Answers are kept in an inkan `Store`, by default `memoryStore()` from @vxnsin/inkan: one +process, at most `max` keys (default 10000), the least recently used dropped first. For +several processes any `Store` works, such as @inkanjs/redis, @inkanjs/mongo, @inkanjs/mysql and @inkanjs/postgres (coming with inkan 0.8): ```ts -const store: IdempotencyStore = { - get: async (key) => JSON.parse((await redis.get(`idem:${key}`)) ?? "null") ?? undefined, - set: async (key, record, ttl) => void (await redis.set(`idem:${key}`, JSON.stringify(record), "PX", ttl)), - add: async (key, record, ttl) => (await redis.set(`idem:${key}`, JSON.stringify(record), "PX", ttl, "NX")) === "OK", - delete: async (key) => void (await redis.del(`idem:${key}`)), -}; +import { redisStore } from "@inkanjs/redis"; + +app.register(idempotency({ store: redisStore(redis) })); ``` -Records are plain JSON (a binary body as base64). `memoryStore({ max })` keeps at most -`max` keys (default 10000), dropping the oldest, in one process. +The plugin uses `get`, `set`, `delete` and the atomic `add`, with TTLs in milliseconds +(`ttl`, `lockTtl`). Records are plain JSON (a binary body as base64). diff --git a/packages/idempotency/index.d.ts b/packages/idempotency/index.d.ts index afc1991..2467c2a 100644 --- a/packages/idempotency/index.d.ts +++ b/packages/idempotency/index.d.ts @@ -1,4 +1,4 @@ -import type { Context, Plugin } from "@vxnsin/inkan"; +import type { Context, memoryStore as coreMemoryStore, Plugin, Store } from "@vxnsin/inkan"; /** What is kept under a key: a marker while the first request runs, then its answer. Plain JSON. */ export type IdempotencyRecord = @@ -14,22 +14,16 @@ export type IdempotencyRecord = binary: boolean; }; -/** Where answers are kept. Every method may return a promise; `ttl` is in milliseconds. */ -export interface IdempotencyStore { - get(key: string): IdempotencyRecord | undefined | null | Promise; - set(key: string, record: IdempotencyRecord, ttl: number): void | Promise; - delete(key: string): void | Promise; - /** - * Keeps the record only when the key is free, and says whether it did: Redis - * `SET key value NX PX ttl`. With it, two processes never both run the same key; - * without it, only two requests in the same process are kept apart. - */ - add?(key: string, record: IdempotencyRecord, ttl: number): boolean | Promise; -} +/** @deprecated Use `Store` from "@vxnsin/inkan". */ +export type IdempotencyStore = Store; export type IdempotencyOptions = { - /** Default `memoryStore()`: one process only. */ - store?: IdempotencyStore; + /** + * Where answers are kept, an `IdempotencyRecord` per key: any inkan `Store` (Redis, a + * database, …); its atomic `add` keeps two processes from both running a key. Default + * `memoryStore()` from @vxnsin/inkan: one process only. + */ + store?: Store; /** Default `["POST", "PATCH"]`. */ methods?: string[]; /** Default `"Idempotency-Key"`. */ @@ -62,5 +56,5 @@ export type IdempotencyOptions = { export declare function idempotency(options?: IdempotencyOptions): Plugin<{}, {}, {}>; export default idempotency; -/** A store in this process's memory, at most `max` keys (default 10000), the oldest dropped first. */ -export declare function memoryStore(options?: { max?: number }): Required & { readonly size: number }; +/** @deprecated Use `memoryStore` from "@vxnsin/inkan": this is the same function, re-exported. */ +export declare const memoryStore: typeof coreMemoryStore; diff --git a/packages/idempotency/index.js b/packages/idempotency/index.js index 6eb9d73..5fa6c71 100644 --- a/packages/idempotency/index.js +++ b/packages/idempotency/index.js @@ -9,38 +9,12 @@ // the same key for another request (another method, path, query or body) a 422. import { createHash } from "node:crypto"; -import { HttpProblem, plugin, reply, t } from "@vxnsin/inkan"; - -/** A store in this process's memory: for development and single-process apps. */ -export function memoryStore({ max = 10_000 } = {}) { - const map = new Map(); // key → { value, until }; oldest first - const live = (key) => { - const e = map.get(key); - if (e && e.until <= Date.now()) map.delete(key); - return e && e.until > Date.now() ? e : undefined; - }; - const put = (key, value, ttl) => { - map.delete(key); - map.set(key, { value, until: Date.now() + ttl }); - while (map.size > max) map.delete(map.keys().next().value); - }; - return { - get: (key) => live(key)?.value, - set: put, - /** Keeps it only when the key is free; true when it did. */ - add(key, value, ttl) { - if (live(key)) return false; - put(key, value, ttl); - return true; - }, - delete(key) { - map.delete(key); - }, - get size() { - return map.size; - }, - }; -} +import { HttpProblem, memoryStore, plugin, problem, reply, stateKey, t } from "@vxnsin/inkan"; + +/** @deprecated the core's `memoryStore` from @vxnsin/inkan, re-exported for 0.1 code. */ +export { memoryStore } from "@vxnsin/inkan"; + +const STORE_METHODS = ["get", "set", "delete", "add"]; /** JSON with the keys sorted, so `{a,b}` and `{b,a}` are the same request. */ function stable(v) { @@ -67,6 +41,8 @@ const defaultKeep = (status) => (status >= 200 && status < 300) || (status >= 40 */ export function idempotency(options = {}) { const store = options.store ?? memoryStore(); + const missing = store !== null && typeof store === "object" ? STORE_METHODS.filter((m) => typeof store[m] !== "function") : STORE_METHODS; + if (missing.length) throw new TypeError(`idempotency: store must be an inkan Store (get, set, delete and an atomic add; TTLs in ms), missing: ${missing.join(", ")}`); const methods = new Set((options.methods ?? ["POST", "PATCH"]).map((m) => m.toUpperCase())); const header = (options.header ?? "idempotency-key").toLowerCase(); const required = options.required ?? false; @@ -114,8 +90,8 @@ export function idempotency(options = {}) { return plugin( (app) => { - const asked = new WeakMap(); // ctx → { key, auth }, read before a header schema strips them - const claimed = new WeakMap(); // ctx → { id, fingerprint } for the requests this process runs + const asked = stateKey("idempotency.asked"); // { key, auth }, read before a header schema strips them + const claimed = stateKey("idempotency.claimed"); // { id, fingerprint } for the requests this process runs const running = new Map(); // id → fingerprint: two requests at once in this process never both run app.onRequest((ctx) => { @@ -139,28 +115,27 @@ export function idempotency(options = {}) { const fingerprint = sha256(`${caller}\n${ctx.method}\n${ctx.path}\n${stable(ctx.query)}\n${stable(ctx.body)}`); const reused = () => new HttpProblem(422, "idempotency-key-reused", `This ${header} was used for another request`); - const inFlight = () => new HttpProblem(409, "idempotency-in-flight", `A request with this ${header} is still running; try again shortly`, {}, { "retry-after": "1" }); + const inFlight = () => problem(409, "idempotency-in-flight", { detail: `A request with this ${header} is still running; try again shortly`, headers: { "retry-after": "1" } }); const mine = running.get(id); if (mine !== undefined) throw mine === fingerprint ? inFlight() : reused(); running.set(id, fingerprint); // taken at once, before anything waits let held = false; try { - const before = await store.get(id); - if (before) { - if (before.fingerprint !== fingerprint) throw reused(); + const before = await store.get(id); // undefined: nobody has used the key + if (before !== undefined) { + if (before?.fingerprint !== fingerprint) throw reused(); if (before.pending) throw inFlight(); // another process runs it const headers = { ...before.headers, "idempotent-replayed": "true" }; const body = before.json ? JSON.parse(before.body) : before.binary ? Buffer.from(before.body, "base64") : before.body; return reply(before.status, body, headers); } const pending = { pending: true, fingerprint }; - if (store.add) { - if (!(await store.add(id, pending, lock))) { - const now = await store.get(id); - throw now && now.fingerprint !== fingerprint ? reused() : inFlight(); - } - } else await store.set(id, pending, lock); + // claimed in one atomic step of the store: two processes never both run the key + if (!(await store.add(id, pending, lock))) { + const now = await store.get(id); + throw now?.fingerprint !== undefined && now.fingerprint !== fingerprint ? reused() : inFlight(); + } held = true; claimed.set(ctx, { id, fingerprint }); } finally { @@ -171,7 +146,7 @@ export function idempotency(options = {}) { app.onSend(async (ctx, out) => { const c = claimed.get(ctx); if (!c) return; - claimed.delete(ctx); + claimed.set(ctx, undefined); try { if (out.stream || !keep(out.status)) return void (await store.delete(c.id)); // not kept: the client may try again const headers = {}; @@ -188,7 +163,7 @@ export function idempotency(options = {}) { }); // the header and the two answers it can bring, on every route it acts on - app.describe((op, route, components) => { + app.onOperation((op, route, components) => { if (!methods.has(route.method)) return; const params = (op.parameters ??= []); if (!params.some((p) => p.in === "header" && String(p.name).toLowerCase() === header)) params.push({ @@ -200,13 +175,13 @@ export function idempotency(options = {}) { }); const schemas = (components.schemas ??= {}); schemas.Problem ??= t.problem().toJSONSchema().$defs.Problem; - const problem = (description) => ({ description, content: { "application/problem+json": { schema: { $ref: "#/components/schemas/Problem" } } } }); - op.responses["409"] ??= problem("A request with this Idempotency-Key is still running (`idempotency-in-flight`)"); - op.responses["422"] ??= problem("This Idempotency-Key was used for another request (`idempotency-key-reused`)"); - if (required) op.responses["400"] ??= problem("The Idempotency-Key header is missing (`idempotency-key-missing`)"); + const answer = (description) => ({ description, content: { "application/problem+json": { schema: { $ref: "#/components/schemas/Problem" } } } }); + op.responses["409"] ??= answer("A request with this Idempotency-Key is still running (`idempotency-in-flight`)"); + op.responses["422"] ??= answer("This Idempotency-Key was used for another request (`idempotency-key-reused`)"); + if (required) op.responses["400"] ??= answer("The Idempotency-Key header is missing (`idempotency-key-missing`)"); }); }, - { name: "idempotency", shared: true, inkan: ">=0.7.0 <0.8.0" }, + { name: "idempotency", shared: true, inkan: ">=0.8.0 <1.0.0" }, ); } diff --git a/packages/idempotency/package.json b/packages/idempotency/package.json index 5ae1460..8b2fb87 100644 --- a/packages/idempotency/package.json +++ b/packages/idempotency/package.json @@ -1,6 +1,6 @@ { "name": "@inkanjs/idempotency", - "version": "0.1.0", + "version": "0.2.0", "description": "Idempotency-Key for inkan: a retried POST gets the first answer again instead of running twice.", "type": "module", "license": "MIT", @@ -34,13 +34,13 @@ "node": ">=22.18" }, "peerDependencies": { - "@vxnsin/inkan": ">=0.7.0 <0.8.0" + "@vxnsin/inkan": ">=0.8.0 <1.0.0" }, "devDependencies": { - "@vxnsin/inkan": "^0.7.0" + "@vxnsin/inkan": "^0.8.0" }, "scripts": { - "test": "node --test test.ts" + "test": "node --import ../../scripts/no-deprecations.js --test test.ts" }, "publishConfig": { "access": "public" diff --git a/packages/idempotency/test.ts b/packages/idempotency/test.ts index 3ac06e8..d076bf5 100644 --- a/packages/idempotency/test.ts +++ b/packages/idempotency/test.ts @@ -1,7 +1,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { inkan, plugin, problem, reply, t } from "@vxnsin/inkan"; -import { idempotency, memoryStore, type IdempotencyOptions } from "./index.js"; +import { inkan, memoryStore, plugin, problem, reply, t, type Store } from "@vxnsin/inkan"; +import { idempotency, memoryStore as deprecatedMemoryStore, type IdempotencyOptions } from "./index.js"; const quiet = { log: false as const }; const Order = t.object({ id: t.number(), tea: t.string() }); @@ -249,12 +249,65 @@ test("methods: PUT too when asked; other methods are left alone", async () => { assert.equal(n, 3); }); -test("memoryStore keeps at most max keys", async () => { +test("on the core's memoryStore(): kept at most max keys, for ttl ms", async (t) => { + t.mock.timers.enable({ apis: ["Date"], now: 1_000_000 }); const store = memoryStore({ max: 2 }); - const s = shop({ store }); + const s = shop({ store, ttl: 5000 }); for (const k of ["a", "b", "c"]) await order(s.app, k); - assert.equal(store.size, 2); assert.equal((await order(s.app, "a")).headers["idempotent-replayed"], undefined, "the oldest went first"); + assert.equal((await order(s.app, "c")).headers["idempotent-replayed"], "true"); + t.mock.timers.tick(5001); + assert.equal((await order(s.app, "c")).headers["idempotent-replayed"], undefined, "gone after ttl ms"); +}); + +const METHODS = ["get", "set", "delete", "add", "take", "increment"]; + +/** Any Store, wrapped: only the six methods are there, and every call and property read is noted. */ +function spyStore(inner: Store = memoryStore()) { + const calls: string[] = []; + const ttls: (number | undefined)[] = []; + const touched = new Set(); + const plain: Store = { + get: (k) => (calls.push("get"), inner.get(k)), + set: (k, v, ttl) => (calls.push("set"), ttls.push(ttl), inner.set(k, v, ttl)), + delete: (k) => (calls.push("delete"), inner.delete(k)), + add: (k, v, ttl) => (calls.push("add"), ttls.push(ttl), inner.add(k, v, ttl)), + take: (k) => (calls.push("take"), inner.take(k)), + increment: (k, by, ttl) => (calls.push("increment"), inner.increment(k, by, ttl)), + }; + const store = new Proxy(plain, { + get(target, prop, receiver) { + touched.add(prop); + return Reflect.get(target, prop, receiver); + }, + }); + return { store, calls, ttls, touched }; +} + +test("any Store will do: only get, add, set and delete of it are used, TTLs in ms", async () => { + const { store, calls, ttls, touched } = spyStore(); + const s = shop({ store, ttl: 120_000, lockTtl: 7_000 }); + assert.equal((await order(s.app, "k1")).status, 201); + assert.equal((await order(s.app, "k1")).headers["idempotent-replayed"], "true"); + await s.app.inject({ method: "POST", url: "/fail", headers: { "idempotency-key": "k2" } }); + assert.deepEqual([...new Set(calls)].sort(), ["add", "delete", "get", "set"]); + assert.deepEqual([...touched].filter((p) => !METHODS.includes(String(p))), [], "nothing but the Store interface"); + assert.deepEqual(ttls, [7_000, 120_000, 7_000]); +}); + +test("a key the store does not have is undefined; null or a stranger's record is not an answer", async () => { + const store = memoryStore(); + const s = shop({ store }); + assert.equal((await order(s.app, "fresh")).status, 201); + const other = shop({ store: { ...store, get: async () => null } as Store }); + assert.equal((await order(other.app, "fresh")).status, 422, "null is a value of somebody else's, never a free key"); +}); + +test("a store without add, or not a store at all, is refused when the plugin is made; memoryStore is the core's", () => { + const { get, set, delete: del } = memoryStore(); + assert.throws(() => idempotency({ store: { get, set, delete: del } as unknown as Store }), /store must be an inkan Store .*missing: add/); + assert.throws(() => idempotency({ store: 42 as unknown as Store }), /store must be an inkan Store/); + assert.equal(deprecatedMemoryStore, memoryStore); }); test("OpenAPI: the header and the 409 and 422 answers, only on the routes it acts on", async () => { diff --git a/packages/mongo/CHANGELOG.md b/packages/mongo/CHANGELOG.md new file mode 100644 index 0000000..06ae4db --- /dev/null +++ b/packages/mongo/CHANGELOG.md @@ -0,0 +1,27 @@ +# Changelog + +## 0.1.0 + +- `mongoStore(dbOrCollection, { collection, events, prefix, create, changeStreams, poll })`: + an inkan `Store` on your own `mongodb` client, passing `storeConformance`. +- Keys and channels under `prefix`, default `"inkan:"`, which ends in `:` (added when + missing) so two prefixes cannot run into each other's keys. A key or channel with NUL or a + lone surrogate is refused with a `TypeError`. +- One document a key with a TTL index on `e`; reads, `add` and `increment` filter on the + server's `$$NOW`, so nothing is seen after its TTL while the TTL monitor catches up. +- `add`, `take` and `increment` are single atomic commands. +- Errors from the server come without keys and values: a new `Error` with `code` and + `codeName`, its message without the "dup key" part and with every quoted part replaced by + `"…"`; the driver's `keyValue`, `keyPattern` and `errorResponse` are not passed on. +- A keys collection that already holds documents but has no `inkan_ttl` index (one the + store did not make) is refused with `create: true`: the TTL index on `e` could delete the + user's own documents. `create: false` takes it, the index left to the user. +- A TTL must be a finite number of ms, more than 0 and at most ten years (TypeError or + RangeError otherwise); `increment` takes only a finite number as its step, so no NaN or + null is ever counted. +- `increment` on a live key whose value is not a JSON number rejects with a `TypeError` and + leaves the value alone (it counted such a value as 0). +- publish/subscribe through a change stream on a replica set or mongos, otherwise by + reading the messages collection every `poll` ms. The poller reads only messages dated up + to five seconds past the server's clock: one written with a date in the future neither + arrives nor moves the cursor past later messages. diff --git a/packages/mongo/README.md b/packages/mongo/README.md new file mode 100644 index 0000000..4a7ad75 --- /dev/null +++ b/packages/mongo/README.md @@ -0,0 +1,116 @@ +# @inkanjs/mongo + +A `Store` for [inkan](https://github.com/inkanjs/inkan) on MongoDB: background jobs, rate +limits, sessions, logins and idempotency keys kept where every process of your app sees +them. You pass your own `mongodb` client; the package has no dependencies. + +```sh +npm install @vxnsin/inkan @inkanjs/mongo mongodb +``` + +```ts +import { MongoClient } from "mongodb"; +import { inkan, rateLimit } from "@vxnsin/inkan"; +import { mongoStore } from "@inkanjs/mongo"; + +const client = await new MongoClient(process.env.MONGO_URL!).connect(); +const store = mongoStore(client.db("app")); + +const app = inkan({ workers: 4 }) + .register(rateLimit({ max: 100, window: 60_000, store })) // counted across all workers + .job("/exports", { store }, async (job) => buildExport(job)); // progress and cancels reach every worker +``` + +The same store works for the middleware packages that take a `Store`: + +```ts +app.register(session({ secret, store })); // @inkanjs/session +app.register(auth({ secret, users, session: { store } })); // @inkanjs/auth +app.register(idempotency({ store })); // @inkanjs/idempotency +``` + +Give two uses that might pick the same keys a `prefix` of their own: +`mongoStore(db, { prefix: "sessions:" })`. + +## Options + +`mongoStore(dbOrCollection, options)` takes a `Db` (the keys go into `options.collection`) or +a `Collection` (the keys go there, messages into `options.events` of its database). + +| option | default | | +| --- | --- | --- | +| `collection` | `"inkan_store"` | the collection for the keys, when a `Db` is given | +| `events` | `"inkan_events"` | the collection for publish/subscribe messages | +| `prefix` | `"inkan:"` | put in front of every key and channel; ends in `:` (added when missing) | +| `create` | `true` | create the TTL indexes on first use; `false` when your migrations do it | +| `changeStreams` | `"auto"` | `true`, `false`, or `"auto"`: change streams when the server is a replica set or mongos | +| `poll` | `1000` | ms between two reads of the messages without change streams | + +`store.close()` stops listening for messages. The client is yours: close it yourself. + +**Give the keys a collection of their own.** The TTL index on `e` deletes every document +whose `e` is a past date, the store's or not. So with `create` (the default) the store +refuses a collection that already holds documents but has no `inkan_ttl` index (one it did +not make before): the first call rejects with an error and nothing is changed. A collection +the store made itself is taken again after a restart. To use a collection of yours anyway, +pass `create: false` and make the index yourself, knowing what it deletes. + +## How it keeps things + +One document a key: `{ _id: key, v: "", e: Date | null }`, or `{ _id, n, e }` for +a counter. Values are kept as JSON text, so objects whose fields start with `$` or hold a +dot come back as they were. A TTL index on `e` (`expireAfterSeconds: 0`) deletes what is +over. + +- **Atomic.** `add` is one `findOneAndUpdate` with an upsert whose pipeline keeps the + document when it is live and replaces it when it is missing or expired; `take` is one + `findOneAndDelete`; `increment` one `findOneAndUpdate` that adds to a live count, or + starts over at `by` with a new TTL when the key is missing or expired. Fifty at once on + one key, from any number of processes: one `add` wins, one `take` gets the value, the + counts go 1 to 50. +- **One clock.** Expiry is computed and checked with `$$NOW`, the server's clock, so app + servers whose clocks differ agree on what is over. +- **Messages.** `publish` inserts into the `events` collection. On a replica set or mongos, + a change stream delivers them within milliseconds. On a standalone server, change streams + do not exist: the collection is read every `poll` ms while someone listens, so a message + arrives **up to about a second** later. Messages are kept a minute (a TTL index on `at`). + +## Security + +- **Publish/subscribe has no authentication.** A message is whatever arrives on the channel: + anyone who can insert into the `events` collection can send one, and inkan's jobs act on + them (a forged message can cancel a running job or report false progress). Use a MongoDB + user of its own for the app with a role limited to the two collections, and no other user + with write access to them. Do not share the database with tenants you do not trust. +- **Writing the store is trusting it.** Whoever can write the keys can forge sessions, + idempotent replies, job records and rate-limit counts, and whoever can read them reads + every session. Give the app's database user access to the store's two collections (`find, + insert, update, remove`, `createIndex` with `create: true`, and `changeStream` on the + events) and nothing else, and nobody else write access to them. +- **Values are stored as JSON text**, as they are given: not encrypted, not signed. Keep + secrets out of them or encrypt them yourself; keys are stored as given too (under the + prefix), so do not put secrets in keys. +- **Errors carry no keys or values.** A failing command rejects with a new `Error` that + keeps `code` and `codeName`, its message without the "dup key" part and with every quoted + part replaced by `"…"`; the driver's `keyValue`, `keyPattern` and `errorResponse` are not + passed on. Log them freely. + +## Limits + +- **Keys and the prefix.** The prefix (default `"inkan:"`) ends in `:`; one without gets + it (`"app"` is `"app:"`), so prefix `"app"` with key `"1x"` and prefix `"app1"` with key + `"x"` stay apart. `""` puts nothing in front. Do not give one use a prefix that starts + another's (`"app:"` and `"app:sessions:"` can meet). A key or channel holding NUL (`\0`) or a + lone UTF-16 surrogate is refused with a `TypeError`, in all four inkan stores alike. +- **TTL monitor.** MongoDB deletes expired documents about once a minute, not at `e`. + Nothing returns them in the meantime (every read filters on `e > $$NOW`, and `add` and + `increment` treat them as absent), but they take space until then. +- **Polling.** Without a replica set, messages arrive up to `poll` ms late, and a message + whose write takes more than five seconds to land after a newer one can be missed. +- **Keys.** Any string; MongoDB 4.2 and later have no index-key size limit, so long keys + are kept as they are. A value must fit a 16 MB document. +- **Versions.** MongoDB 4.4 or later (`$isNumber`, pipeline updates); tested on MongoDB 7 + and 8 with the `mongodb` driver 7. Change streams need a replica set: a single-node one + (`mongod --replSet rs0` and `rs.initiate()`) is enough. +- `publish` reaches subscribers in the order one process published, when it awaits each + `publish`. A message sent while nobody listens is lost, as the `Store` contract allows. diff --git a/packages/mongo/index.d.ts b/packages/mongo/index.d.ts new file mode 100644 index 0000000..c068658 --- /dev/null +++ b/packages/mongo/index.d.ts @@ -0,0 +1,65 @@ +import type { Store } from "@vxnsin/inkan"; + +/** The parts of a `Db` of the official `mongodb` driver this store uses. */ +export type MongoDbLike = { + collection(name: string): any; + admin(): { command(command: object): Promise }; + command(command: object): Promise; +}; + +/** The parts of a `Collection` of the official `mongodb` driver this store uses. */ +export type MongoCollectionLike = { + readonly collectionName: string; + findOne(...args: any[]): Promise; + findOneAndUpdate(...args: any[]): Promise; + findOneAndDelete(...args: any[]): Promise; + updateOne(...args: any[]): Promise; + deleteOne(...args: any[]): Promise; + createIndex(...args: any[]): Promise; + listIndexes(...args: any[]): { toArray(): Promise }; +}; + +export type MongoStoreOptions = { + /** The collection for the keys, when a `Db` is given. Default `"inkan_store"`. */ + collection?: string; + /** The collection for publish/subscribe messages. Default `"inkan_events"`. */ + events?: string; + /** + * Put in front of every key and channel, so several apps can share the collections. Ends in + * `:` (one is added when missing); `""` for none. Default `"inkan:"`. + */ + prefix?: string; + /** + * Create the TTL indexes on first use. Default true; false when your migrations do it. With + * true, a keys collection that already has documents but no `inkan_ttl` index is refused: + * the TTL index on `e` would delete documents of yours. + */ + create?: boolean; + /** + * How subscribers hear of messages: `true` change streams (a replica set or mongos), + * `false` reading the messages collection every `poll` ms, `"auto"` change streams when + * the server says it is a replica set or mongos. Default `"auto"`. + */ + changeStreams?: boolean | "auto"; + /** Milliseconds between two reads of the messages without change streams. Default 1000. */ + poll?: number; +}; + +/** A `Store` on MongoDB, with a `close()` that stops listening for messages. */ +export type MongoStore = Store & { + readonly shared: true; + publish(channel: string, message: unknown): Promise; + subscribe(channel: string, fn: (message: unknown) => void): Promise<() => void>; + /** Stops the change stream or the polling. The client stays open: it is yours. */ + close(): Promise; +}; + +/** + * A `Store` for inkan on MongoDB (4.4 or later), on a `Db` or a `Collection` of your own + * `mongodb` client. One document a key, `{ _id: key, v: "", e: Date | null }`, a TTL + * index on `e`; reads filter on the server's clock, so a key is never seen after its TTL + * even though MongoDB deletes it up to a minute later. `add`, `take` and `increment` are + * single atomic commands. Shared by every process that uses the same database. + */ +export declare function mongoStore(target: MongoDbLike | MongoCollectionLike, options?: MongoStoreOptions): MongoStore; +export default mongoStore; diff --git a/packages/mongo/index.js b/packages/mongo/index.js new file mode 100644 index 0000000..05c34b4 --- /dev/null +++ b/packages/mongo/index.js @@ -0,0 +1,435 @@ +// A `Store` for inkan on MongoDB, with the user's own client from the official `mongodb` +// driver. No dependency of its own. +// +// const store = mongoStore(client.db("app")); +// app.job("/exports", { store }, run); +// +// One document a key: { _id: key, v: "", e: Date | null }, or { _id, n: number, e } +// for a key `increment` wrote. A TTL index on `e` lets MongoDB delete what is over, but its +// monitor only runs about once a minute, so every read filters on `e > $$NOW` too, and +// `add`/`increment` treat an expired document as absent. Every time comes from the server +// ($$NOW), so app servers whose clocks differ still agree. +// +// publish/subscribe: one small collection of messages. On a replica set (or mongos) a change +// stream delivers them at once; on a standalone server it is read every `poll` ms. + +import { randomBytes } from "node:crypto"; + +const DUPLICATE = 11000; +const LOOKBACK = 5000; // ms a message may be older than the newest one seen and still arrive +const KEEP_EVENTS = 60; // s a message stays in the collection + +/** Whether the document is there and its TTL is not over, as an aggregation expression. */ +const LIVE = { + $and: [ + { $or: [{ $eq: [{ $type: "$v" }, "string"] }, { $isNumber: "$n" }] }, + { $or: [{ $eq: [{ $ifNull: ["$e", null] }, null] }, { $gt: ["$e", "$$NOW"] }] }, + ], +}; +const UNEXPIRED = { $expr: { $or: [{ $eq: [{ $ifNull: ["$e", null] }, null] }, { $gt: ["$e", "$$NOW"] }] } }; + +/** A key or channel as given, or a TypeError: a string without NUL and without a lone surrogate. */ +const checked = (key, what = "key") => { + if (typeof key !== "string") throw new TypeError(`mongoStore: a ${what} is a string, not ${typeof key}`); + if (key.includes("\0") || !key.isWellFormed()) throw new TypeError(`mongoStore: a ${what} cannot hold NUL or a lone surrogate`); + return key; +}; + +/** The prefix: "" for none, otherwise ending in ":" (added when missing), so a key never runs into another prefix. */ +const prefixOf = (prefix) => { + checked(prefix, "prefix"); + return prefix === "" || prefix.endsWith(":") ? prefix : `${prefix}:`; +}; + +/** Ten years in ms: the longest TTL taken. */ +const MAX_TTL = 315_576_000_000; + +/** When a key with this TTL ends, as an aggregation expression: null for none. */ +const until = (ttl) => { + if (ttl === undefined || ttl === null) return null; + if (typeof ttl !== "number" || !Number.isFinite(ttl)) throw new TypeError(`mongoStore: a TTL is a finite number of milliseconds, not ${String(ttl)}`); + if (ttl <= 0 || ttl > MAX_TTL) throw new RangeError(`mongoStore: a TTL is more than 0 and at most ten years (${MAX_TTL} ms), not ${ttl}`); + return { $add: ["$$NOW", Math.ceil(ttl)] }; +}; + +const json = (value) => JSON.stringify(value === undefined ? null : value); + +/** The value of a document, or undefined. */ +function decode(doc) { + if (!doc) return undefined; + if (typeof doc.v === "string") return JSON.parse(doc.v); + if (typeof doc.n === "number") return doc.n; + return undefined; +} + +/** Runs `fn` again when two upserts on one new _id meet (the server usually retries itself). */ +async function retrying(fn) { + for (let i = 0; ; i++) { + try { + return await fn(); + } catch (err) { + if (err?.code !== DUPLICATE || i >= 5) throw err; + } + } +} + +/** + * A `Store` on MongoDB. Takes a `Db` or a `Collection` of the official driver. + * @param {import("mongodb").Db | import("mongodb").Collection} target + * @param {import("./index.d.ts").MongoStoreOptions} [options] + */ +export function mongoStore(target, options = {}) { + if (!target || (typeof target.collection !== "function" && typeof target.findOneAndUpdate !== "function")) { + throw new TypeError("mongoStore: pass a Db or a Collection of the mongodb driver"); + } + const isDb = typeof target.collection === "function" && typeof target.command === "function"; + const db = isDb ? target : (target.db ?? target.s?.db); + const name = options.collection ?? (isDb ? "inkan_store" : target.collectionName); + const keys = isDb ? target.collection(name) : target; + const eventsName = options.events ?? "inkan_events"; + const prefix = prefixOf(options.prefix ?? "inkan:"); + const create = options.create ?? true; + const poll = Math.max(10, options.poll ?? 1000); + const mode = options.changeStreams ?? "auto"; + const id = (key, what) => prefix + checked(key, what); + + let ready; + const setup = () => + (ready ??= (async () => { + if (!create) return; + // The TTL index deletes every document whose `e` is a past date, ours or not. Made on a + // collection that already holds documents (and not by a store before: no inkan_ttl), + // it could delete the user's own; refused, unless the user makes the index (create: false). + const indexes = await keys + .listIndexes() + .toArray() + .catch((err) => { + if (err?.code === 26) return []; // NamespaceNotFound: no collection yet, so empty + throw err; + }); + if (!indexes.some((i) => i.name === "inkan_ttl") && (await keys.findOne({}, { projection: { _id: 1 } }))) { + throw new Error( + `mongoStore: the collection ${JSON.stringify(keys.collectionName)} already has documents and no inkan_ttl index; ` + + "the TTL index on `e` the store makes could delete documents of yours. Use a new or empty collection, " + + "or create: false and make the index yourself", + ); + } + await keys.createIndex({ e: 1 }, { name: "inkan_ttl", expireAfterSeconds: 0 }); + })().catch((err) => { + ready = undefined; + throw err; + })); + + // publish/subscribe + const channels = new Map(); // full channel -> Set of fns + let events; + let eventsReady; + let streams; // whether change streams are used, once known + let listener; // { stop(): Promise } while anybody listens + let starting; + + const eventsCollection = () => { + if (!db) throw new TypeError("mongoStore: publish/subscribe needs the collection's Db"); + events ??= db.collection(eventsName); + return (eventsReady ??= (async () => { + if (create) await events.createIndex({ at: 1 }, { name: "inkan_events_ttl", expireAfterSeconds: KEEP_EVENTS }); + return events; + })().catch((err) => { + eventsReady = undefined; + throw err; + })); + }; + + const useStreams = async () => { + if (streams !== undefined) return streams; + if (mode === true || mode === false) return (streams = mode); + const hello = await db.admin().command({ hello: 1 }).catch(() => db.admin().command({ isMaster: 1 })); + return (streams = Boolean(hello.setName) || hello.msg === "isdbgrid"); + }; + + const dispatch = (doc) => { + const fns = channels.get(doc.c); + if (!fns) return; + let message; + try { + message = JSON.parse(doc.m); + } catch { + return; + } + for (const fn of [...fns]) { + try { + fn(message); + } catch {} + } + }; + + /** A change stream on the messages, open from before this resolves until stop(). */ + async function openStream(col) { + let stopped = false; + let token; + let stream; + const open = async () => { + const pipeline = [{ $match: { operationType: "insert" } }]; + stream = col.watch(pipeline, token ? { startAfter: token } : {}); + const first = await stream.tryNext(); // the stream listens once this is back + if (first) { + token = first._id; + dispatch(first.fullDocument); + } + }; + const loop = async () => { + while (!stopped) { + try { + const change = await stream.next(); + if (stopped) break; + token = change._id; + if (change.fullDocument) dispatch(change.fullDocument); + } catch { + if (stopped) break; + await stream?.close().catch(() => {}); + await new Promise((r) => setTimeout(r, 1000).unref?.()); + if (stopped) break; + await open().catch(() => {}); + } + } + }; + await open(); + loop(); + return { + async stop() { + stopped = true; + await stream?.close().catch(() => {}); + }, + }; + } + + /** Reads the messages every `poll` ms: those since the newest seen, minus a lookback for late writes. */ + async function openPoller(col) { + const seen = new Map(); // id -> at (ms) + const hello = await db.admin().command({ hello: 1 }).catch(() => ({})); + let cursor = hello.localTime instanceof Date ? hello.localTime.getTime() : Date.now(); + let stopped = false; + let timer; + let running; + const read = async (deliver) => { + // a message dated past the server's clock is not one of ours (publish dates by $$NOW): + // left out, so it can neither arrive nor move the cursor past the messages to come + const docs = await col + .find({ c: { $in: [...channels.keys()] }, at: { $gte: new Date(cursor - LOOKBACK) }, $expr: { $lte: ["$at", { $add: ["$$NOW", LOOKBACK] }] } }) + .sort({ at: 1, _id: 1 }) + .toArray(); + for (const doc of docs) { + const key = String(doc._id); + if (seen.has(key)) continue; + const at = doc.at.getTime(); + seen.set(key, at); + if (at > cursor) cursor = at; + if (deliver && !stopped) dispatch(doc); + } + for (const [key, at] of seen) if (at < cursor - LOOKBACK) seen.delete(key); + }; + await read(false); // what is there already is not for these subscribers + const tick = () => { + if (stopped) return; + running = read(true) + .catch(() => {}) + .finally(() => { + if (!stopped) timer = setTimeout(tick, poll).unref?.() ?? timer; + }); + }; + timer = setTimeout(tick, poll); + timer.unref?.(); + return { + async stop() { + stopped = true; + clearTimeout(timer); + await running; + }, + }; + } + + const startListening = () => + (starting ??= (async () => { + const col = await eventsCollection(); + listener = (await useStreams()) ? await openStream(col) : await openPoller(col); + })().catch((err) => { + starting = undefined; + throw err; + })); + + const stopListening = async () => { + if (channels.size || !starting) return; + const was = starting; + starting = undefined; + await was.catch(() => {}); + const l = listener; + listener = undefined; + await l?.stop(); + }; + + return sanitized({ + shared: true, + + async get(key) { + await setup(); + return decode(await keys.findOne({ _id: id(key), ...UNEXPIRED }, { projection: { v: 1, n: 1 } })); + }, + + async set(key, value, ttl) { + await setup(); + await retrying(() => + keys.updateOne({ _id: id(key) }, [{ $replaceWith: { _id: "$_id", v: { $literal: json(value) }, e: until(ttl) } }], { upsert: true }), + ); + }, + + async delete(key) { + await setup(); + await keys.deleteOne({ _id: id(key) }); + }, + + async add(key, value, ttl) { + await setup(); + const token = randomBytes(12).toString("base64url"); + const doc = await retrying(() => + keys.findOneAndUpdate( + { _id: id(key) }, + [{ $replaceWith: { $cond: [LIVE, "$$ROOT", { _id: "$_id", v: { $literal: json(value) }, e: until(ttl), t: token }] } }], + { upsert: true, returnDocument: "after", projection: { t: 1 } }, + ), + ); + return doc?.t === token; + }, + + async take(key) { + await setup(); + return decode(await keys.findOneAndDelete({ _id: id(key), ...UNEXPIRED }, { projection: { v: 1, n: 1 } })); + }, + + async increment(key, by = 1, ttl) { + await setup(); + const step = by; + if (typeof step !== "number" || !Number.isFinite(step)) throw new TypeError(`mongoStore: increment takes a finite number, not ${String(by)}`); + const live = "$$live"; + const doc = await retrying(() => + keys.findOneAndUpdate( + { _id: id(key) }, + [ + { + $replaceWith: { + $let: { + vars: { live: LIVE }, + in: { + $let: { + // the count so far: 0 for a missing or expired key, null for a live + // value that is not a number (JSON text that does not convert) + vars: { + count: { + $cond: [ + live, + { + $cond: [ + { $isNumber: "$n" }, + "$n", + { $convert: { input: { $ifNull: ["$v", null] }, to: "double", onError: null, onNull: null } }, + ], + }, + 0, + ], + }, + }, + in: { + $cond: [ + { $eq: ["$$count", null] }, + "$$ROOT", // left as it was, and refused below + { _id: "$_id", n: { $add: ["$$count", { $literal: step }] }, e: { $cond: [live, "$e", until(ttl)] } }, + ], + }, + }, + }, + }, + }, + }, + ], + { upsert: true, returnDocument: "after", projection: { n: 1 } }, + ), + ); + if (typeof doc?.n !== "number") throw new TypeError("mongoStore: increment on a key whose value is not a number"); + return doc.n; + }, + + async publish(channel, message) { + const col = await eventsCollection(); + const doc = { c: id(channel, "channel"), m: json(message) }; + if (await useStreams()) await col.insertOne({ ...doc, at: new Date() }); + // the server's time, so that pollers on other machines read it in order + else await col.updateOne({ _id: messageId() }, [{ $set: { c: { $literal: doc.c }, m: { $literal: doc.m }, at: "$$NOW" } }], { upsert: true }); + }, + + async subscribe(channel, fn) { + const full = id(channel, "channel"); + let fns = channels.get(full); + if (!fns) channels.set(full, (fns = new Set())); + const one = (m) => fn(m); + fns.add(one); + try { + await startListening(); + } catch (err) { + fns.delete(one); + if (!fns.size) channels.delete(full); + throw err; + } + let off = false; + return () => { + if (off) return; + off = true; + fns.delete(one); + if (!fns.size && channels.get(full) === fns) channels.delete(full); + stopListening().catch(() => {}); + }; + }, + + /** Stops listening for messages. The client is yours to close. */ + async close() { + channels.clear(); + await stopListening(); + }, + }); +} + +/** + * The store with every method's errors cleaned: the driver's server errors name the key or + * value they are about ("dup key: { _id: … }", `keyValue`, `errorResponse`), and an app logs + * errors. The error thrown instead keeps `code` and `codeName`, its message without the + * "dup key" part and with every quoted part replaced by "…". The store's own errors pass as + * they are. + */ +function sanitized(store) { + for (const name of ["get", "set", "delete", "add", "take", "increment", "publish", "subscribe"]) { + const method = store[name]; + store[name] = async (...args) => { + try { + return await method(...args); + } catch (err) { + throw clean(err); + } + }; + } + return store; +} + +const clean = (err) => { + if (err instanceof Error && err.message.startsWith("mongoStore:")) return err; + const message = String(err?.message ?? err) + .replace(/dup key:[\s\S]*$/, "dup key: { … }") + .replace(/"(?:[^"\\]|\\.)*"/g, '"…"') + .replace(/'(?:[^'\\]|\\.)*'/g, "'…'"); + const out = new Error(`mongoStore: ${message}`); + for (const k of ["code", "codeName"]) if (err?.[k] !== undefined) out[k] = err[k]; + return out; +}; + +/** An _id for a message that sorts after the ones this process sent before. */ +const self = randomBytes(6).toString("hex"); +const messageId = () => `${process.hrtime.bigint().toString().padStart(20, "0")}:${self}`; + +export default mongoStore; diff --git a/packages/mongo/package.json b/packages/mongo/package.json new file mode 100644 index 0000000..dd878fe --- /dev/null +++ b/packages/mongo/package.json @@ -0,0 +1,51 @@ +{ + "name": "@inkanjs/mongo", + "version": "0.1.0", + "description": "A MongoDB Store for inkan: jobs, rate limits, sessions and idempotency keys shared by every process, on your own mongodb client.", + "type": "module", + "license": "MIT", + "author": "Vensin (https://github.com/vxnsin)", + "repository": { + "type": "git", + "url": "git+https://github.com/inkanjs/middleware.git", + "directory": "packages/mongo" + }, + "keywords": [ + "inkan", + "store", + "mongodb", + "mongo", + "rate-limit", + "session", + "jobs" + ], + "exports": { + ".": { + "types": "./index.d.ts", + "default": "./index.js" + } + }, + "files": [ + "index.js", + "index.d.ts", + "README.md", + "CHANGELOG.md" + ], + "engines": { + "node": ">=22.18" + }, + "peerDependencies": { + "@vxnsin/inkan": ">=0.8.0 <1.0.0" + }, + "devDependencies": { + "@vxnsin/inkan": "^0.8.0", + "mongodb": "^7.7.0", + "mongodb-memory-server-core": "^11.3.0" + }, + "scripts": { + "test": "node --import ../../scripts/no-deprecations.js --test test.ts" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/mongo/test.ts b/packages/mongo/test.ts new file mode 100644 index 0000000..7b53905 --- /dev/null +++ b/packages/mongo/test.ts @@ -0,0 +1,274 @@ +// Runs against a real MongoDB: MONGO_URL when it is set (CI), otherwise servers that +// mongodb-memory-server-core starts (it downloads a mongod binary once). Without either, the +// tests are skipped with a message, as they are in CI without MONGO_URL. + +import { after, describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { randomBytes } from "node:crypto"; +import { inspect } from "node:util"; +import { MongoClient } from "mongodb"; +import { storeConformance } from "@vxnsin/inkan/testing"; +import { mongoStore, type MongoStore, type MongoStoreOptions } from "./index.js"; + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); +const until = async (ok: () => boolean, ms: number) => { + const end = Date.now() + ms; + while (!ok() && Date.now() < end) await sleep(20); +}; + +type Server = { name: string; uri: string; options: MongoStoreOptions; streams: boolean }; +const servers: Server[] = []; +const stops: (() => Promise)[] = []; +let skip: string | undefined; + +if (process.env.MONGO_URL) { + servers.push({ name: "MONGO_URL, polling", uri: process.env.MONGO_URL, options: { changeStreams: false }, streams: false }); + servers.push({ name: "MONGO_URL, auto", uri: process.env.MONGO_URL, options: {}, streams: process.env.MONGO_REPLSET === "1" }); +} else if (process.env.CI) { + // the "stores" CI job brings a server; the others do not download one + skip = "no MONGO_URL in CI: the stores job runs these"; +} else { + try { + const { MongoMemoryServer, MongoMemoryReplSet } = await import("mongodb-memory-server-core"); + const single = await MongoMemoryServer.create(); + stops.push(() => single.stop()); + servers.push({ name: "standalone, polling", uri: single.getUri(), options: {}, streams: false }); + const set = await MongoMemoryReplSet.create({ replSet: { count: 1 } }); + await set.waitUntilRunning(); + stops.push(() => set.stop()); + servers.push({ name: "replica set, change streams", uri: set.getUri(), options: {}, streams: true }); + } catch (err) { + skip = `no MongoDB: set MONGO_URL or let mongodb-memory-server-core download mongod (${(err instanceof Error ? err.message : String(err))})`; + } +} + +const clients: MongoClient[] = []; +const connect = async (uri: string) => { + const client = await new MongoClient(uri).connect(); + clients.push(client); + return client; +}; +after(async () => { + await Promise.all(clients.map((c) => c.close().catch(() => {}))); + await Promise.all(stops.map((s) => s().catch(() => {}))); +}); + +if (skip) test("MongoDB store", { skip }, () => {}); + +for (const server of servers) { + const client = await connect(server.uri); + const db = client.db("inkan_test"); + const make = () => mongoStore(db, { ...server.options, prefix: `t:${randomBytes(4).toString("hex")}:` }); + + storeConformance(make, { name: `mongoStore conformance (${server.name})`, close: (s) => (s as MongoStore).close() }); + + describe(`mongoStore (${server.name})`, () => { + test("two clients on one server see each other's keys and messages", async () => { + const other = await connect(server.uri); + const prefix = `x:${randomBytes(4).toString("hex")}:`; + const a = mongoStore(db, { ...server.options, prefix }); + const b = mongoStore(other.db("inkan_test"), { ...server.options, prefix }); + try { + await a.set("k", { from: "a" }); + assert.deepEqual(await b.get("k"), { from: "a" }); + assert.equal(await b.add("k", "b"), false); + const counts = await Promise.all(Array.from({ length: 20 }, (_, i) => (i % 2 ? a : b).increment("n"))); + assert.deepEqual(counts.sort((x, y) => x - y), Array.from({ length: 20 }, (_, i) => i + 1)); + const got: unknown[] = []; + const off = await b.subscribe("ch", (m) => got.push(m)); + for (const m of [1, { two: 2 }, "three"]) await a.publish("ch", m); + await until(() => got.length >= 3, 5000); + assert.deepEqual(got, [1, { two: 2 }, "three"]); + off(); + } finally { + await a.close(); + await b.close(); + } + }); + + test("an expired document the TTL monitor has not deleted yet counts as absent", async () => { + const prefix = `e:${randomBytes(4).toString("hex")}:`; + const store = mongoStore(db, { ...server.options, prefix }); + const col = db.collection("inkan_store"); + // as MongoDB may keep it for a minute after `e` + await col.insertOne({ _id: `${prefix}old` as never, v: JSON.stringify("stale"), e: new Date(Date.now() - 10_000) }); + assert.equal(await store.get("old"), undefined); + assert.equal(await store.take("old"), undefined); + assert.equal(await store.add("old", "fresh", 60_000), true); + assert.equal(await store.get("old"), "fresh"); + await col.updateOne({ _id: `${prefix}old` as never }, { $set: { e: new Date(Date.now() - 10_000) } }); + assert.equal(await store.increment("old", 3), 3, "starts over"); + const doc = await col.findOne({ _id: `${prefix}old` as never }); + assert.equal(doc?.e, null, "without a ttl, the new count stays"); + await store.close(); + }); + + test("the TTL index is there, and values with $ and dots in their keys survive", async () => { + const store = mongoStore(db, server.options); + const value = { $where: "x", "a.b": { $gt: 1 }, list: [{ "$ref": 1 }] }; + await store.set(`v:${randomBytes(4).toString("hex")}`, value); + const indexes = await db.collection("inkan_store").indexes(); + assert.ok(indexes.some((i) => i.key.e === 1 && i.expireAfterSeconds === 0)); + const k = `v:${randomBytes(4).toString("hex")}`; + await store.set(k, value); + assert.deepEqual(await store.get(k), value); + await store.close(); + }); + + test("polling: a message dated in the future neither arrives nor holds the later ones back", async () => { + const prefix = `f:${randomBytes(4).toString("hex")}:`; + const store = mongoStore(db, { prefix, changeStreams: false, poll: 20 }); + const got: unknown[] = []; + const off = await store.subscribe("ch", (m) => got.push(m)); + try { + await db.collection("inkan_events").insertOne({ c: `${prefix}ch`, m: JSON.stringify("forged"), at: new Date(Date.now() + 3_600_000) }); + await sleep(200); + await store.publish("ch", "real"); + await until(() => got.length > 0, 5000); + await sleep(100); + assert.deepEqual(got, ["real"]); + } finally { + off(); + await store.close(); + } + }); + + test("refuses a step or a TTL that is not a sane number", async () => { + const store = mongoStore(db, { ...server.options, prefix: `n:${randomBytes(4).toString("hex")}:` }); + try { + for (const by of [NaN, Infinity, -Infinity, "1"]) await assert.rejects(store.increment("n", by as never), TypeError, String(by)); + assert.equal(await store.get("n"), undefined, "no NaN or null was counted"); + for (const ttl of [NaN, Infinity, "10"]) { + await assert.rejects(store.set("t", 1, ttl as never), TypeError, String(ttl)); + await assert.rejects(store.add("t", 1, ttl as never), TypeError, String(ttl)); + await assert.rejects(store.increment("t", 1, ttl as never), TypeError, String(ttl)); + } + for (const ttl of [0, -5, 315_576_000_001]) await assert.rejects(store.set("t", 1, ttl), RangeError, String(ttl)); + assert.equal(await store.get("t"), undefined, "nothing was written"); + await store.set("t", 1, 315_576_000_000); + assert.equal(await store.get("t"), 1); + } finally { + await store.close(); + } + }); + + test("increment on a value that is not a number throws a TypeError and leaves it", async () => { + const store = mongoStore(db, { ...server.options, prefix: `s:${randomBytes(4).toString("hex")}:` }); + try { + for (const value of ["not a number", "5", true, null, { n: 1 }, [1]]) { + await store.set("s", value, 60_000); + await assert.rejects(store.increment("s", 2), TypeError, `${JSON.stringify(value)} is not a number`); + assert.deepEqual(await store.get("s"), value, "and it stays as it was"); + } + await store.set("s", 5); + assert.equal(await store.increment("s", 2), 7, "a number counts on"); + } finally { + await store.close(); + } + }); + + test("refuses a collection with documents of yours: its TTL index would delete them", async () => { + const name = `users_${randomBytes(4).toString("hex")}`; + const mine = db.collection(name); + await mine.insertOne({ _id: "ada" as never, e: new Date(0), email: "ada@example.com" }); + for (const store of [mongoStore(mine, server.options), mongoStore(db, { ...server.options, collection: name })]) { + await assert.rejects(store.set("k", 1), /already has documents/); + await store.close(); + } + assert.ok(!(await mine.indexes()).some((i) => i.key.e === 1), "no TTL index was made"); + assert.equal((await mine.findOne({ _id: "ada" as never }))?.email, "ada@example.com"); + const own = mongoStore(mine, { ...server.options, create: false }); + await own.set("k", 1); + assert.equal(await own.get("k"), 1, "create: false leaves the index to you"); + await own.close(); + await mine.drop(); + + // a collection the store made before is its own: a restart takes it + const fresh = db.collection(`inkan_${randomBytes(4).toString("hex")}`); + const first = mongoStore(fresh, server.options); + await first.set("k", 1); + await first.close(); + const again = mongoStore(fresh, server.options); + assert.equal(await again.get("k"), 1); + await again.close(); + await fresh.drop(); + }); + + test("a prefix ends in ':' (one is added); keys with NUL or a lone surrogate are refused", async () => { + const base = `p${randomBytes(4).toString("hex")}`; + const a = mongoStore(db, { ...server.options, prefix: base }); + const b = mongoStore(db, { ...server.options, prefix: `${base}1` }); + const plain = mongoStore(db, server.options); + try { + await a.set("1x", "a"); + await b.set("x", "b"); + assert.equal(await a.get("1x"), "a", "prefix p + key 1x is not prefix p1 + key x"); + assert.ok(await db.collection("inkan_store").findOne({ _id: `${base}:1x` as never })); + const k = randomBytes(4).toString("hex"); + await plain.set(k, 1); + assert.ok(await db.collection("inkan_store").findOne({ _id: `inkan:${k}` as never }), "the default prefix is inkan:"); + assert.throws(() => mongoStore(db, { prefix: "a\0" }), TypeError); + for (const bad of ["a\0b", "\uD800", "x\uDC00y"]) { + await assert.rejects(a.set(bad, 1), TypeError, JSON.stringify(bad)); + await assert.rejects(a.get(bad), TypeError); + await assert.rejects(a.increment(bad), TypeError); + await assert.rejects(a.publish(bad, 1), TypeError); + await assert.rejects(a.subscribe(bad, () => {}), TypeError); + } + await assert.rejects(a.get(1 as never), TypeError); + } finally { + await a.close(); + await b.close(); + await plain.close(); + } + }); + + test("a database error carries no key and no value: no dup key, no keyValue", async () => { + const name = `inkan_err_${randomBytes(4).toString("hex")}`; + // a unique index of the user's on the value: a second key with the same value fails, and + // MongoDB names the value ("dup key: { v: ... }", keyValue) + await db.collection(name).createIndex({ v: 1 }, { unique: true }); + const store = mongoStore(db, { ...server.options, collection: name, prefix: "secret-key-1:" }); + try { + await store.set("a", "secret-value-1"); + const err = await store.set("b", "secret-value-1").then( + () => assert.fail("the unique index refuses it"), + (e: unknown) => e as Error & Record, + ); + assert.equal(err.code, 11000); + assert.match(err.message, /duplicate key/); + for (const k of ["keyValue", "keyPattern", "errorResponse", "errmsg"]) assert.equal(err[k], undefined, k); + const all = inspect(err, { depth: 10, showHidden: true }); + assert.ok(!all.includes("secret-value-1"), all); + assert.ok(!all.includes("secret-key-1"), all); + } finally { + await store.close(); + await db.collection(name).drop(); + } + }); + + test("takes a Collection too", async () => { + const store = mongoStore(db.collection("inkan_other"), { ...server.options, prefix: randomBytes(4).toString("hex") }); + await store.set("k", 1, 60_000); + assert.equal(await store.increment("k", 2), 3); + assert.equal(await store.take("k"), 3); + await store.close(); + }); + + test(`listens with ${server.streams ? "a change stream" : "polling"}`, async () => { + const store = mongoStore(db, server.options); + const off = await store.subscribe(`c:${randomBytes(4).toString("hex")}`, () => {}); + const ops = await client.db("admin").command({ currentOp: 1, $all: true }); + const watching = (ops.inprog as { cursor?: { originatingCommand?: { pipeline?: { $changeStream?: unknown }[] } } }[]).some((op) => + op.cursor?.originatingCommand?.pipeline?.some((stage) => stage.$changeStream), + ); + assert.equal(watching, server.streams); + off(); + await store.close(); + }); + }); +} + +test("refuses what is not a Db or a Collection", () => { + assert.throws(() => mongoStore({} as never), TypeError); +}); diff --git a/packages/mysql/CHANGELOG.md b/packages/mysql/CHANGELOG.md new file mode 100644 index 0000000..c2ab013 --- /dev/null +++ b/packages/mysql/CHANGELOG.md @@ -0,0 +1,28 @@ +# Changelog + +## 0.1.0 + +- `mysqlStore(pool, { table, events, prefix, create, sweep, poll })`: an inkan `Store` on + your own `mysql2/promise` pool, passing `storeConformance`. +- Keys and channels under `prefix`, default `"inkan:"`, which ends in `:` (added when + missing) so two prefixes cannot run into each other's keys. A key or channel with NUL or a + lone surrogate is refused with a `TypeError`. +- Binary keys (exact bytes), JSON values, TTLs on the database's UTC clock; keys over 255 + bytes kept under their SHA-256. +- `add`, `take` and `increment` atomic across processes; deadlocks retried. +- A TTL must be a finite number of ms, more than 0 and at most ten years (TypeError or + RangeError otherwise), so no TTL overflows `DATETIME` into a NULL (no expiry); `increment` + takes only a finite number as its step. +- `increment` on a live key whose value is not a JSON number rejects with a `TypeError` and + leaves the value alone (it counted such a value as 0). +- publish/subscribe by polling a messages table every `poll` ms; a periodic, unref'd sweep + of expired keys and old messages, once at startup and then every `sweep` ms, in batches of + 10000 rows until none are left (at most a million a sweep), so the table cannot outgrow it. +- The poller reads only messages dated up to five seconds past the server's clock: a row + written with a date in the future neither arrives nor moves the cursor past later messages. +- Table names checked against `/^[A-Za-z_][A-Za-z0-9_]*$/`; every value a parameter of a + server-side prepared statement (`execute`), so `NO_BACKSLASH_ESCAPES` and multibyte client + charsets (gbk, big5, sjis) cannot turn a key into SQL. The pool needs `execute`. +- Errors from the database come without keys and values: a new `Error` with `code`, `errno` + and `sqlState`, its message with every quoted part replaced by `'…'`; mysql2's `sql` (the + statement, with the values in it for `query`) is not passed on. diff --git a/packages/mysql/README.md b/packages/mysql/README.md new file mode 100644 index 0000000..d40fc09 --- /dev/null +++ b/packages/mysql/README.md @@ -0,0 +1,122 @@ +# @inkanjs/mysql + +A `Store` for [inkan](https://github.com/inkanjs/inkan) on MySQL: background jobs, rate +limits, sessions, logins and idempotency keys kept where every process of your app sees +them. You pass your own `mysql2/promise` pool; the package has no dependencies. + +```sh +npm install @vxnsin/inkan @inkanjs/mysql mysql2 +``` + +```ts +import mysql from "mysql2/promise"; +import { inkan, rateLimit } from "@vxnsin/inkan"; +import { mysqlStore } from "@inkanjs/mysql"; + +const pool = mysql.createPool(process.env.MYSQL_URL!); +const store = mysqlStore(pool); + +const app = inkan({ workers: 4 }) + .register(rateLimit({ max: 100, window: 60_000, store })) // counted across all workers + .job("/exports", { store }, async (job) => buildExport(job)); // progress and cancels reach every worker +``` + +The same store works for the middleware packages that take a `Store`: + +```ts +app.register(session({ secret, store })); // @inkanjs/session +app.register(auth({ secret, users, session: { store } })); // @inkanjs/auth +app.register(idempotency({ store })); // @inkanjs/idempotency +``` + +Give two uses that might pick the same keys a `prefix` of their own: +`mysqlStore(pool, { prefix: "sessions:" })`. + +## Options + +| option | default | | +| --- | --- | --- | +| `table` | `"inkan_store"` | the table for the keys | +| `events` | `"inkan_events"` | the table for publish/subscribe messages | +| `prefix` | `"inkan:"` | put in front of every key and channel; ends in `:` (added when missing) | +| `create` | `true` | `CREATE TABLE IF NOT EXISTS` both tables on first use | +| `sweep` | `60000` | ms between two deletes of expired keys and old messages; `0` for none | +| `poll` | `1000` | ms between two reads of the messages while someone listens | + +Table names must match `/^[A-Za-z_][A-Za-z0-9_]*$/`; anything else throws. `store.sweep()` +deletes what is over now; `store.close()` stops the sweep and the polling. The pool is +yours: end it yourself. It has to be a pool (`createPool`), not one connection, since +`take` and `increment` run in transactions of their own. + +## How it keeps things + +```sql +CREATE TABLE inkan_store ( + k VARBINARY(255) NOT NULL PRIMARY KEY, -- the key, byte for byte + v JSON NOT NULL, + e DATETIME(3) NULL, -- UTC; NULL for no TTL + KEY inkan_e (e) +); +CREATE TABLE inkan_events ( + id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, + c VARBINARY(255) NOT NULL, m LONGTEXT NOT NULL, at DATETIME(3) NOT NULL, KEY inkan_at (at) +); +``` + +- **Exact keys.** `k` is binary, so `a`, `A`, `a ` and `ä` are four keys. (A `utf8mb4_bin` + column would pad: MySQL compares `"a"` and `"a "` as equal there.) +- **Atomic.** `add` deletes the key if it is expired, then `INSERT IGNORE`s: the insert + decides, and of many at once exactly one sees a row inserted. `take` is `SELECT … FOR + UPDATE` and `DELETE` in one transaction. `increment` is `INSERT … ON DUPLICATE KEY UPDATE` + (add to a live count, or start over at `by` with a new TTL) and a read of the new number + in one transaction. Fifty at once on one key, from any number of processes: one `add` + wins, one `take` gets the value, the counts go 1 to 50. Deadlocks are retried. +- **One clock.** Expiry is computed and checked with `UTC_TIMESTAMP(3)`, the database's + clock, whatever the session time zone, so app servers whose clocks differ agree. +- **SQL.** Every key, value and TTL is a parameter of a server-side prepared statement + (`pool.execute`), never escaped into the SQL text, so `NO_BACKSLASH_ESCAPES` or a multibyte + client charset (gbk, big5, sjis) cannot turn a key into SQL. The only names in the SQL text + are the two checked, backtick-quoted table names. + +## Security + +- **Publish/subscribe has no authentication.** A message is whatever arrives on the channel: + anyone who can `INSERT` into the `events` table can send one, and inkan's jobs act on them + (a forged message can cancel a running job or report false progress). Use a MySQL user of + its own for the app with grants on the two tables only, and no other user with `INSERT` on + them. Do not share the database with tenants you do not trust. +- **Writing the store is trusting it.** Whoever can write the keys can forge sessions, + idempotent replies, job records and rate-limit counts, and whoever can read them reads + every session. Give the app's database user access to the store's two tables (`SELECT, + INSERT, UPDATE, DELETE` on them, plus `CREATE` with `create: true`) and nothing else, and + nobody else write access to them. +- **Values are stored as JSON text**, as they are given: not encrypted, not signed. Keep + secrets out of them or encrypt them yourself; keys are stored as given too (under the + prefix), so do not put secrets in keys. +- **Errors carry no keys or values.** A failing statement rejects with a new `Error` that + keeps `code`, `errno` and `sqlState`, its message with every quoted part replaced by `'…'` + (MySQL quotes values: "Duplicate entry '…'"); mysql2's `sql` is not passed on. Log them + freely. + +## Limits + +- **Keys and the prefix.** The prefix (default `"inkan:"`) ends in `:`; one without gets + it (`"app"` is `"app:"`), so prefix `"app"` with key `"1x"` and prefix `"app1"` with key + `"x"` stay apart. `""` puts nothing in front. Do not give one use a prefix that starts + another's (`"app:"` and `"app:sessions:"` can meet). A key or channel holding NUL (`\0`) or a + lone UTF-16 surrogate is refused with a `TypeError`, in all four inkan stores alike. +- **Long keys.** A key (with its prefix) longer than 255 bytes of UTF-8 is kept under + `\x01sha256:` of it. Two different keys share a row only if SHA-256 collides. Keys + starting with `\x01` are hashed too, so no key can pose as a hash. +- **Expired rows stay until the sweep** (once when the store starts, then every `sweep` ms, + unref'd). A sweep deletes 10000 rows a statement until a statement comes back short, at + most a million rows a sweep; the next one goes on. Nothing returns expired rows in the + meantime: every read filters on `e > UTC_TIMESTAMP(3)`. +- **Messages are polled.** MySQL has no notifications, so subscribers read the `events` + table every `poll` ms: a message arrives **up to about a second** later, one query a + second per store that has subscribers. Messages are kept a minute. One whose insert + commits more than five seconds after a newer one can be missed. +- **Versions.** MySQL 8.0 or later (`CAST(… AS JSON)`, `JSON_TYPE`); tested on 8.4 with + `mysql2` 3. MariaDB is not supported (its JSON is text). +- `publish` reaches subscribers in the order one process published, when it awaits each + `publish`. A message sent while nobody listens is lost, as the `Store` contract allows. diff --git a/packages/mysql/index.d.ts b/packages/mysql/index.d.ts new file mode 100644 index 0000000..0be4dcb --- /dev/null +++ b/packages/mysql/index.d.ts @@ -0,0 +1,59 @@ +import type { Store } from "@vxnsin/inkan"; + +/** A connection from the pool, as `mysql2/promise` gives it. */ +export type MysqlConnectionLike = { + query(sql: string, values?: unknown[]): Promise<[any, any]>; + execute(sql: string, values?: any[]): Promise<[any, any]>; + beginTransaction(): Promise; + commit(): Promise; + rollback(): Promise; + release(): void; +}; + +/** The parts of a `mysql2/promise` pool this store uses. */ +export type MysqlPoolLike = { + query(sql: string, values?: unknown[]): Promise<[any, any]>; + /** Every statement with values runs as a server-side prepared statement. */ + execute(sql: string, values?: any[]): Promise<[any, any]>; + getConnection(): Promise; +}; + +export type MysqlStoreOptions = { + /** The table for the keys. Letters, digits and `_`, not starting with a digit. Default `"inkan_store"`. */ + table?: string; + /** The table for publish/subscribe messages, named by the same rule. Default `"inkan_events"`. */ + events?: string; + /** + * Put in front of every key and channel, so several apps can share the tables. Ends in `:` + * (one is added when missing); `""` for none. Default `"inkan:"`. + */ + prefix?: string; + /** Create both tables on first use (`CREATE TABLE IF NOT EXISTS`). Default true. */ + create?: boolean; + /** Milliseconds between two sweeps of expired keys and old messages; 0 for none. Default 60000. */ + sweep?: number; + /** Milliseconds between two reads of the messages table while somebody listens. Default 1000. */ + poll?: number; +}; + +/** A `Store` on MySQL, with `sweep()` and a `close()` that stops its timers. */ +export type MysqlStore = Store & { + readonly shared: true; + publish(channel: string, message: unknown): Promise; + subscribe(channel: string, fn: (message: unknown) => void): Promise<() => void>; + /** Deletes expired keys and old messages now, as the periodic sweep does. */ + sweep(): Promise; + /** Stops the sweep and the polling. The pool stays open: it is yours. */ + close(): Promise; +}; + +/** + * A `Store` for inkan on MySQL 8 or later, on your own `mysql2/promise` pool. One row a + * key in a JSON column, TTLs on the server's UTC clock, `add`, `take` and `increment` atomic + * across processes, publish/subscribe by polling a messages table every `poll` ms. + */ +export declare function mysqlStore(pool: MysqlPoolLike, options?: MysqlStoreOptions): MysqlStore; +export default mysqlStore; + +/** What a key is kept under: itself, or `\x01sha256:` when it is longer than 255 bytes (or starts with `\x01`). */ +export declare function storedKey(key: string): string; diff --git a/packages/mysql/index.js b/packages/mysql/index.js new file mode 100644 index 0000000..2182c6f --- /dev/null +++ b/packages/mysql/index.js @@ -0,0 +1,398 @@ +// A `Store` for inkan on MySQL 8, with the user's own `mysql2/promise` pool. No dependency +// of its own. +// +// const store = mysqlStore(mysql.createPool(process.env.MYSQL_URL)); +// app.job("/exports", { store }, run); +// +// One row a key: k VARBINARY(255) (exact bytes: case, trailing spaces and accents all count), +// v JSON, e DATETIME(3) in UTC or NULL. Every read filters on `e > UTC_TIMESTAMP(3)`, the +// server's clock, so app servers whose clocks differ still agree; a sweep deletes what is +// over every `sweep` ms. Keys longer than 255 bytes are kept under their SHA-256. +// +// SQL: table names are checked against /^[A-Za-z_][A-Za-z0-9_]*$/ and quoted; every value is +// a parameter of a server-side prepared statement (`execute`), never escaped into the SQL +// text, so neither `NO_BACKSLASH_ESCAPES` nor a multibyte client charset (gbk, big5, sjis) +// can turn a key into SQL. `query` runs only statements without parameters. + +import { createHash } from "node:crypto"; + +const NAME = /^[A-Za-z_][A-Za-z0-9_]*$/; +const RETRY = new Set([1213, 1205]); // deadlock, lock wait timeout +const LOOKBACK = 5000; // ms a message may commit after a newer one and still arrive +const KEEP_EVENTS = 60; // s a message stays in the table +const SWEEP_BATCH = 10_000; // rows one DELETE of the sweep takes, so no lock is held long +const SWEEP_BATCHES = 100; // DELETEs one sweep runs at most (a million rows); the next goes on +const NOW = "UTC_TIMESTAMP(3)"; +const LIVE = `(e IS NULL OR e > ${NOW})`; +const NUMBER = "('INTEGER','UNSIGNED INTEGER','DOUBLE','DECIMAL')"; + +const ident = (name, what) => { + if (typeof name !== "string" || !NAME.test(name)) throw new TypeError(`mysqlStore: ${what} must match ${NAME}, got ${JSON.stringify(name)}`); + return `\`${name}\``; +}; + +/** What a key or channel is kept under: itself, or its SHA-256 when it is longer than 255 bytes. */ +export function storedKey(key) { + // \x01 starts only hashed keys, so a key that looks like a hash is hashed as well + if (Buffer.byteLength(key, "utf8") <= 255 && !key.startsWith("\x01")) return key; + return `\x01sha256:${createHash("sha256").update(key, "utf8").digest("hex")}`; +} + +/** A key or channel as given, or a TypeError: a string without NUL and without a lone surrogate. */ +const checked = (key, what = "key") => { + if (typeof key !== "string") throw new TypeError(`mysqlStore: a ${what} is a string, not ${typeof key}`); + if (key.includes("\0") || !key.isWellFormed()) throw new TypeError(`mysqlStore: a ${what} cannot hold NUL or a lone surrogate`); + return key; +}; + +/** The prefix: "" for none, otherwise ending in ":" (added when missing), so a key never runs into another prefix. */ +const prefixOf = (prefix) => { + checked(prefix, "prefix"); + return prefix === "" || prefix.endsWith(":") ? prefix : `${prefix}:`; +}; + +/** Ten years in ms: the longest TTL taken, far from DATETIME's end (an overflow there is NULL: no TTL). */ +const MAX_TTL = 315_576_000_000; + +/** The end of a TTL as SQL and its parameters: NULL for none. */ +const until = (ttl) => { + if (ttl === undefined || ttl === null) return ["NULL", []]; + if (typeof ttl !== "number" || !Number.isFinite(ttl)) throw new TypeError(`mysqlStore: a TTL is a finite number of milliseconds, not ${String(ttl)}`); + if (ttl <= 0 || ttl > MAX_TTL) throw new RangeError(`mysqlStore: a TTL is more than 0 and at most ten years (${MAX_TTL} ms), not ${ttl}`); + return [`${NOW} + INTERVAL ? MICROSECOND`, [Math.ceil(ttl) * 1000]]; +}; + +const json = (value) => JSON.stringify(value === undefined ? null : value); +const parse = (text) => (text === null || text === undefined ? undefined : JSON.parse(Buffer.isBuffer(text) ? text.toString("utf8") : text)); + +const utc = (text) => Date.parse(`${text.replace(" ", "T")}Z`); +const sqlTime = (ms) => new Date(ms).toISOString().replace("T", " ").replace("Z", ""); + +/** + * A `Store` on MySQL 8 (or later), on a `mysql2/promise` pool. + * @param {import("./index.d.ts").MysqlPoolLike} pool + * @param {import("./index.d.ts").MysqlStoreOptions} [options] + */ +export function mysqlStore(pool, options = {}) { + if (!pool || typeof pool.query !== "function" || typeof pool.execute !== "function" || typeof pool.getConnection !== "function") { + throw new TypeError("mysqlStore: pass a mysql2/promise pool (it needs query, execute and getConnection)"); + } + const table = ident(options.table ?? "inkan_store", "table"); + const events = ident(options.events ?? "inkan_events", "events"); + const prefix = prefixOf(options.prefix ?? "inkan:"); + const create = options.create ?? true; + const sweepEvery = options.sweep ?? 60_000; + const poll = Math.max(10, options.poll ?? 1000); + const id = (key, what) => storedKey(prefix + checked(key, what)); + const v = `CAST(v AS CHAR CHARACTER SET utf8mb4) AS v`; + + let ready; + const setup = () => + (ready ??= (async () => { + if (create) { + await pool.query( + `CREATE TABLE IF NOT EXISTS ${table} (k VARBINARY(255) NOT NULL PRIMARY KEY, v JSON NOT NULL, e DATETIME(3) NULL, KEY inkan_e (e)) ENGINE=InnoDB`, + ); + await pool.query( + `CREATE TABLE IF NOT EXISTS ${events} (id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, c VARBINARY(255) NOT NULL, m LONGTEXT CHARACTER SET utf8mb4 NOT NULL, at DATETIME(3) NOT NULL, KEY inkan_at (at)) ENGINE=InnoDB`, + ); + } + if (sweepEvery > 0 && !closed) { + // once now, so a process that never lives a whole `sweep` still deletes; then on a timer + background(); + sweeper = setInterval(background, sweepEvery); + sweeper.unref?.(); + } + })().catch((err) => { + ready = undefined; + throw err; + })); + let sweeper; + let closed = false; + + /** Deletes in batches of SWEEP_BATCH until one comes back short, at most SWEEP_BATCHES of them. */ + const drain = async (sql) => { + for (let i = 0; i < SWEEP_BATCHES && !closed; i++) { + const [result] = await pool.query(sql); + if (result.affectedRows < SWEEP_BATCH) return; + } + }; + const sweep = async () => { + await drain(`DELETE FROM ${table} WHERE e <= ${NOW} LIMIT ${SWEEP_BATCH}`); + await drain(`DELETE FROM ${events} WHERE at < ${NOW} - INTERVAL ${KEEP_EVENTS} SECOND LIMIT ${SWEEP_BATCH}`); + }; + let sweeping; + /** The timer's sweep: one at a time, errors left for the next one. */ + const background = () => { + sweeping ??= sweep() + .catch(() => {}) + .finally(() => { + sweeping = undefined; + }); + }; + + /** Runs `fn` in a transaction on a connection of its own, again after a deadlock. */ + const transaction = async (fn) => { + for (let i = 0; ; i++) { + const conn = await pool.getConnection(); + try { + await conn.beginTransaction(); + const result = await fn(conn); + await conn.commit(); + return result; + } catch (err) { + await conn.rollback().catch(() => {}); + if (!RETRY.has(err?.errno) || i >= 20) throw err; + } finally { + conn.release(); + } + await new Promise((r) => setTimeout(r, Math.random() * 5 * (i + 1))); + } + }; + const retrying = async (fn) => { + for (let i = 0; ; i++) { + try { + return await fn(); + } catch (err) { + if (!RETRY.has(err?.errno) || i >= 20) throw err; + } + await new Promise((r) => setTimeout(r, Math.random() * 5 * (i + 1))); + } + }; + + // publish/subscribe: a poller that runs while anybody listens + const channels = new Map(); // stored channel -> Set of fns + let starting; + let poller; + + const dispatch = (row) => { + const fns = channels.get(Buffer.isBuffer(row.c) ? row.c.toString("utf8") : String(row.c)); + if (!fns) return; + let message; + try { + message = parse(row.m); + } catch { + return; + } + for (const fn of [...fns]) { + try { + fn(message); + } catch {} + } + }; + + async function openPoller() { + const seen = new Map(); // id -> at (ms) + const [[{ now }]] = await pool.query(`SELECT CAST(${NOW} AS CHAR) AS now`); + let cursor = utc(now); + let stopped = false; + let timer; + let running; + const read = async (deliver) => { + const names = [...channels.keys()]; + if (!names.length) return; + // a row dated past the server's clock is not one of ours (publish dates by that clock): + // left out, so it can neither arrive nor move the cursor past the messages to come + const [rows] = await pool.execute( + `SELECT id, c, m, CAST(at AS CHAR) AS at FROM ${events} ` + + `WHERE at >= ? AND at <= ${NOW} + INTERVAL ${LOOKBACK / 1000} SECOND AND c IN (${names.map(() => "?").join(", ")}) ORDER BY id`, + [sqlTime(cursor - LOOKBACK), ...names], + ); + for (const row of rows) { + const key = String(row.id); + if (seen.has(key)) continue; + const at = utc(row.at); + seen.set(key, at); + if (at > cursor) cursor = at; + if (deliver && !stopped) dispatch(row); + } + for (const [key, at] of seen) if (at < cursor - LOOKBACK) seen.delete(key); + }; + await read(false); // what is there already is not for these subscribers + const tick = () => { + if (stopped) return; + running = read(true) + .catch(() => {}) + .finally(() => { + if (stopped) return; + timer = setTimeout(tick, poll); + timer.unref?.(); + }); + }; + timer = setTimeout(tick, poll); + timer.unref?.(); + return { + async stop() { + stopped = true; + clearTimeout(timer); + await running; + }, + }; + } + + const startListening = () => + (starting ??= (async () => { + await setup(); + poller = await openPoller(); + })().catch((err) => { + starting = undefined; + throw err; + })); + + const stopListening = async () => { + if (channels.size || !starting) return; + const was = starting; + starting = undefined; + await was.catch(() => {}); + const p = poller; + poller = undefined; + await p?.stop(); + }; + + return sanitized({ + shared: true, + + async get(key) { + await setup(); + const [rows] = await pool.execute(`SELECT ${v} FROM ${table} WHERE k = ? AND ${LIVE}`, [id(key)]); + return parse(rows[0]?.v); + }, + + async set(key, value, ttl) { + await setup(); + const [e, p] = until(ttl); + const text = json(value); + await retrying(() => + pool.execute(`INSERT INTO ${table} (k, v, e) VALUES (?, CAST(? AS JSON), ${e}) ON DUPLICATE KEY UPDATE v = CAST(? AS JSON), e = ${e}`, [ + id(key), + text, + ...p, + text, + ...p, + ]), + ); + }, + + async delete(key) { + await setup(); + await pool.execute(`DELETE FROM ${table} WHERE k = ?`, [id(key)]); + }, + + async add(key, value, ttl) { + await setup(); + const k = id(key); + const [e, p] = until(ttl); + // An expired row is absent: remove it, then insert only where nothing is. The INSERT is + // what decides; of many at once exactly one gets affectedRows 1. + return retrying(async () => { + await pool.execute(`DELETE FROM ${table} WHERE k = ? AND e <= ${NOW}`, [k]); + const [result] = await pool.execute(`INSERT IGNORE INTO ${table} (k, v, e) VALUES (?, CAST(? AS JSON), ${e})`, [k, json(value), ...p]); + return result.affectedRows === 1; + }); + }, + + async take(key) { + await setup(); + const k = id(key); + return transaction(async (conn) => { + const [rows] = await conn.execute(`SELECT ${v}, ${LIVE} AS live FROM ${table} WHERE k = ? FOR UPDATE`, [k]); + if (!rows.length) return undefined; + await conn.execute(`DELETE FROM ${table} WHERE k = ?`, [k]); + return Number(rows[0].live) ? parse(rows[0].v) : undefined; + }); + }, + + async increment(key, by = 1, ttl) { + await setup(); + const k = id(key); + const step = by; + if (typeof step !== "number" || !Number.isFinite(step)) throw new TypeError(`mysqlStore: increment takes a finite number, not ${String(by)}`); + const [e, p] = until(ttl); + return transaction(async (conn) => { + // `v` first, while `e` is still the old one; then `e` + await conn.execute( + `INSERT INTO ${table} (k, v, e) VALUES (?, CAST(? AS JSON), ${e}) ON DUPLICATE KEY UPDATE ` + + // a live value that is not a number stays as it is, and is refused below + `v = IF(${LIVE}, IF(JSON_TYPE(v) IN ${NUMBER}, CAST(JSON_EXTRACT(v, '$') + ? AS JSON), v), CAST(? AS JSON)), ` + + `e = IF(${LIVE}, e, ${e})`, + [k, json(step), ...p, step, json(step), ...p], + ); + const [rows] = await conn.execute(`SELECT ${v}, JSON_TYPE(v) IN ${NUMBER} AS num FROM ${table} WHERE k = ?`, [k]); + if (!Number(rows[0].num)) throw new TypeError("mysqlStore: increment on a key whose value is not a number"); + return Number(parse(rows[0].v)); + }); + }, + + async publish(channel, message) { + await setup(); + await pool.execute(`INSERT INTO ${events} (c, m, at) VALUES (?, ?, ${NOW})`, [id(channel, "channel"), json(message)]); + }, + + async subscribe(channel, fn) { + const full = id(channel, "channel"); + let fns = channels.get(full); + if (!fns) channels.set(full, (fns = new Set())); + const one = (m) => fn(m); + fns.add(one); + try { + await startListening(); + } catch (err) { + fns.delete(one); + if (!fns.size) channels.delete(full); + throw err; + } + let off = false; + return () => { + if (off) return; + off = true; + fns.delete(one); + if (!fns.size && channels.get(full) === fns) channels.delete(full); + stopListening().catch(() => {}); + }; + }, + + /** Deletes what is over now, as the periodic sweep does. */ + sweep: async () => { + await setup(); + await sweep(); + }, + + /** Stops the sweep and the polling. The pool is yours to end. */ + async close() { + closed = true; + clearInterval(sweeper); + channels.clear(); + await stopListening(); + }, + }); +} + +/** + * The store with every method's errors cleaned: what mysql2 rejects with carries the SQL text + * (`sql`, with the values in it for `query`) and messages that quote values ("Duplicate entry + * ''"), and an app logs errors. The error thrown instead keeps `code`, `errno`, `sqlState` + * and the message with every quoted part replaced by '…'. The store's own errors pass as they are. + */ +function sanitized(store) { + for (const name of ["get", "set", "delete", "add", "take", "increment", "publish", "subscribe", "sweep"]) { + const method = store[name]; + store[name] = async (...args) => { + try { + return await method(...args); + } catch (err) { + throw clean(err); + } + }; + } + return store; +} + +const clean = (err) => { + if (err instanceof Error && err.message.startsWith("mysqlStore:")) return err; + const message = String(err?.message ?? err).replace(/'(?:[^'\\]|\\.|'')*'/g, "'…'").replace(/"(?:[^"\\]|\\.)*"/g, '"…"'); + const out = new Error(`mysqlStore: ${message}`); + for (const k of ["code", "errno", "sqlState", "fatal"]) if (err?.[k] !== undefined) out[k] = err[k]; + return out; +}; + +export default mysqlStore; diff --git a/packages/mysql/package.json b/packages/mysql/package.json new file mode 100644 index 0000000..0746354 --- /dev/null +++ b/packages/mysql/package.json @@ -0,0 +1,51 @@ +{ + "name": "@inkanjs/mysql", + "version": "0.1.0", + "description": "A MySQL Store for inkan: jobs, rate limits, sessions and idempotency keys shared by every process, on your own mysql2 pool.", + "type": "module", + "license": "MIT", + "author": "Vensin (https://github.com/vxnsin)", + "repository": { + "type": "git", + "url": "git+https://github.com/inkanjs/middleware.git", + "directory": "packages/mysql" + }, + "keywords": [ + "inkan", + "store", + "mysql", + "database", + "rate-limit", + "session", + "jobs" + ], + "exports": { + ".": { + "types": "./index.d.ts", + "default": "./index.js" + } + }, + "files": [ + "index.js", + "index.d.ts", + "README.md", + "CHANGELOG.md" + ], + "engines": { + "node": ">=22.18" + }, + "peerDependencies": { + "@vxnsin/inkan": ">=0.8.0 <1.0.0" + }, + "devDependencies": { + "@vxnsin/inkan": "^0.8.0", + "mysql-memory-server": "^1.17.0", + "mysql2": "^3.24.6" + }, + "scripts": { + "test": "node --import ../../scripts/no-deprecations.js --test test.ts" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/mysql/test.ts b/packages/mysql/test.ts new file mode 100644 index 0000000..12e0f23 --- /dev/null +++ b/packages/mysql/test.ts @@ -0,0 +1,312 @@ +// Runs against a real MySQL: MYSQL_URL when it is set (CI), otherwise one that +// mysql-memory-server starts (it downloads the official MySQL 8.4 binary once). Without +// either, the tests are skipped with a message, as they are in CI without MYSQL_URL. + +import { after, describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { randomBytes } from "node:crypto"; +import { inspect } from "node:util"; +import mysql from "mysql2/promise"; +import { storeConformance } from "@vxnsin/inkan/testing"; +import { mysqlStore, storedKey, type MysqlStore } from "./index.js"; + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); +const until = async (ok: () => boolean, ms: number) => { + const end = Date.now() + ms; + while (!ok() && Date.now() < end) await sleep(20); +}; + +let url = process.env.MYSQL_URL; +let stop: (() => Promise) | undefined; +let skip: string | undefined; +if (!url && process.env.CI) { + // the "stores" CI job brings a server; the others do not download one + skip = "no MYSQL_URL in CI: the stores job runs these"; +} else if (!url) { + try { + const { createDB } = await import("mysql-memory-server"); + const db = await createDB({ version: "8.4.x", dbName: "inkan_test", logLevel: "ERROR" }); + stop = () => db.stop(); + url = `mysql://${db.username}@127.0.0.1:${db.port}/${db.dbName}`; + } catch (err) { + skip = `no MySQL: set MYSQL_URL or let mysql-memory-server download MySQL (${(err instanceof Error ? err.message : String(err))})`; + } +} + +const pools: mysql.Pool[] = []; +const pool = () => { + const p = mysql.createPool({ uri: url!, connectionLimit: 20 }); + pools.push(p); + return p; +}; +after(async () => { + await Promise.all(pools.map((p) => p.end().catch(() => {}))); + await stop?.(); +}); + +if (skip) { + test("MySQL store", { skip }, () => {}); +} else { + const main = pool(); + const fresh = () => `t:${randomBytes(4).toString("hex")}:`; + + storeConformance(() => mysqlStore(main, { prefix: fresh() }), { + name: "mysqlStore conformance", + close: (s) => (s as MysqlStore).close(), + }); + + describe("mysqlStore", () => { + test("two pools on one server see each other's keys and messages", async () => { + const prefix = fresh(); + const a = mysqlStore(main, { prefix }); + const b = mysqlStore(pool(), { prefix }); + try { + await a.set("k", { from: "a" }); + assert.deepEqual(await b.get("k"), { from: "a" }); + assert.equal(await b.add("k", "b"), false); + const counts = await Promise.all(Array.from({ length: 50 }, (_, i) => (i % 2 ? a : b).increment("n", 1, 60_000))); + assert.deepEqual(counts.sort((x, y) => x - y), Array.from({ length: 50 }, (_, i) => i + 1)); + const wins = await Promise.all(Array.from({ length: 50 }, (_, i) => (i % 2 ? a : b).add("once", i))); + assert.equal(wins.filter(Boolean).length, 1); + const got: unknown[] = []; + const off = await b.subscribe("ch", (m) => got.push(m)); + for (const m of [1, { two: 2 }, "three"]) await a.publish("ch", m); + await until(() => got.length >= 3, 5000); + assert.deepEqual(got, [1, { two: 2 }, "three"]); + off(); + } finally { + await a.close(); + await b.close(); + } + }); + + test("an expired row the sweep has not deleted yet counts as absent", async () => { + const prefix = fresh(); + const store = mysqlStore(main, { prefix, sweep: 0 }); + await store.set("old", "stale", 60_000); + await main.query("UPDATE inkan_store SET e = UTC_TIMESTAMP(3) - INTERVAL 10 SECOND WHERE k = ?", [`${prefix}old`]); + assert.equal(await store.get("old"), undefined); + assert.equal(await store.take("old"), undefined); + await store.set("old", "stale", 60_000); + await main.query("UPDATE inkan_store SET e = UTC_TIMESTAMP(3) - INTERVAL 10 SECOND WHERE k = ?", [`${prefix}old`]); + assert.equal(await store.add("old", "fresh"), true); + assert.equal(await store.get("old"), "fresh"); + await main.query("UPDATE inkan_store SET e = UTC_TIMESTAMP(3) - INTERVAL 10 SECOND WHERE k = ?", [`${prefix}old`]); + assert.equal(await store.increment("old", 3), 3, "starts over"); + const [rows] = await main.query("SELECT e FROM inkan_store WHERE k = ?", [`${prefix}old`]); + assert.equal(rows[0]!.e, null, "without a ttl, the new count stays"); + await main.query("UPDATE inkan_store SET e = UTC_TIMESTAMP(3) - INTERVAL 10 SECOND WHERE k = ?", [`${prefix}old`]); + await store.sweep(); + const [none] = await main.query("SELECT 1 FROM inkan_store WHERE k = ?", [`${prefix}old`]); + assert.equal(none.length, 0, "the sweep deletes it"); + await store.close(); + }); + + test("the sweep deletes in batches until nothing expired is left", async () => { + const table = `inkan_sw_${randomBytes(4).toString("hex")}`; + const store = mysqlStore(main, { table, events: `${table}_ev`, sweep: 0 }); + try { + await store.set("live", 1, 600_000); + const conn = await main.getConnection(); + try { + await conn.query("SET SESSION cte_max_recursion_depth = 30000"); + await conn.query( + `INSERT INTO ${table} (k, v, e) WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 25000) ` + + "SELECT CONCAT('old:', i), CAST('1' AS JSON), UTC_TIMESTAMP(3) - INTERVAL 1 SECOND FROM n", + ); + } finally { + conn.release(); + } + await store.sweep(); + const [[left]] = await main.query(`SELECT COUNT(*) AS n FROM ${table}`); + assert.equal(Number(left!.n), 1, "25000 expired rows gone in one sweep, the live one kept"); + } finally { + await store.close(); + await main.query(`DROP TABLE IF EXISTS ${table}, ${table}_ev`); + } + }); + + test("keys are exact bytes: case, trailing spaces, accents", async () => { + const store = mysqlStore(main, { prefix: fresh() }); + const names = ["a", "A", "a ", "a ", "ä", "ä", "ß", "ss"]; + await Promise.all(names.map((k, i) => store.set(k, i))); + for (const [i, k] of names.entries()) assert.equal(await store.get(k), i, JSON.stringify(k)); + await store.close(); + }); + + test("keys longer than 255 bytes are kept under their SHA-256", async () => { + const store = mysqlStore(main, { prefix: fresh() }); + const long = "k".repeat(300); + const other = "k".repeat(299) + "j"; + await store.set(long, 1); + await store.set(other, 2); + assert.equal(await store.get(long), 1); + assert.equal(await store.get(other), 2); + assert.equal(await store.increment(long, 1), 2); + assert.equal(await store.take(long), 2); + const hashed = storedKey(long); + assert.match(hashed, /^\x01sha256:[0-9a-f]{64}$/); + assert.equal(await store.get(hashed), undefined, "a key that looks like a hash is not the hash"); + assert.equal(storedKey("ü".repeat(127)), "ü".repeat(127), "254 bytes stay"); + assert.notEqual(storedKey("ü".repeat(128)), "ü".repeat(128), "256 bytes are hashed"); + await store.close(); + }); + + test("table names are checked; values never reach the SQL text", async () => { + for (const table of ["x; DROP TABLE users", "a`b", "1abc", "", "a-b", "inkan store"]) { + assert.throws(() => mysqlStore(main, { table }), TypeError, table); + assert.throws(() => mysqlStore(main, { events: table }), TypeError, table); + } + const store = mysqlStore(main, { prefix: fresh() }); + const evil = "'); DROP TABLE inkan_store; --"; + await store.set(evil, evil); + assert.equal(await store.get(evil), evil); + assert.equal(await store.add(`${evil}\``, `\\'`), true); + assert.equal(await store.get(`${evil}\``), `\\'`); + await store.close(); + }); + + test("under NO_BACKSLASH_ESCAPES a key with a quote is still just a key", async () => { + const p = pool(); + p.on("connection", (conn) => { + (conn as unknown as { query(sql: string): void }).query("SET SESSION sql_mode = CONCAT(@@SESSION.sql_mode, ',NO_BACKSLASH_ESCAPES')"); + }); + const [[mode]] = await p.query("SELECT @@SESSION.sql_mode AS m"); + assert.match(mode!.m, /NO_BACKSLASH_ESCAPES/); + const store = mysqlStore(p, { prefix: fresh() }); + try { + const evil = "x' OR 1=1 -- "; + await store.set("other", "theirs"); + assert.equal(await store.get(evil), undefined, "no other row comes back"); + assert.equal(await store.take(evil), undefined); + await store.set(evil, "mine"); + assert.equal(await store.get(evil), "mine"); + assert.equal(await store.increment(`${evil}n`, 2), 2); + await store.delete(evil); + assert.equal(await store.get("other"), "theirs", "the delete hit one key"); + assert.equal(await store.add(evil, 1), true); + } finally { + await store.close(); + } + }); + + test("a message dated in the future neither arrives nor holds the later ones back", async () => { + const prefix = fresh(); + const store = mysqlStore(main, { prefix, poll: 20 }); + const got: unknown[] = []; + const off = await store.subscribe("ch", (m) => got.push(m)); + try { + await main.query("INSERT INTO inkan_events (c, m, at) VALUES (?, ?, UTC_TIMESTAMP(3) + INTERVAL 1 HOUR)", [`${prefix}ch`, JSON.stringify("forged")]); + await sleep(200); + await store.publish("ch", "real"); + await until(() => got.length > 0, 5000); + await sleep(100); + assert.deepEqual(got, ["real"]); + } finally { + off(); + await store.close(); + } + }); + + test("a table of its own", async () => { + const store = mysqlStore(main, { table: "inkan_other", events: "inkan_other_events" }); + const k = randomBytes(4).toString("hex"); + await store.set(k, [1], 60_000); + const [rows] = await main.query("SELECT 1 FROM inkan_other WHERE k = ?", [`inkan:${k}`]); + assert.equal(rows.length, 1, "under the default prefix inkan:"); + await store.close(); + }); + + test("a prefix ends in ':' (one is added); keys with NUL or a lone surrogate are refused", async () => { + const base = `p${randomBytes(4).toString("hex")}`; + const a = mysqlStore(main, { prefix: base }); + const b = mysqlStore(main, { prefix: `${base}1` }); + try { + await a.set("1x", "a"); + await b.set("x", "b"); + assert.equal(await a.get("1x"), "a", "prefix p + key 1x is not prefix p1 + key x"); + const [rows] = await main.query("SELECT k FROM inkan_store WHERE k = ?", [`${base}:1x`]); + assert.equal(rows.length, 1); + assert.throws(() => mysqlStore(main, { prefix: "a\0" }), TypeError); + for (const bad of ["a\0b", "\uD800", "x\uDC00y"]) { + await assert.rejects(a.set(bad, 1), TypeError, JSON.stringify(bad)); + await assert.rejects(a.get(bad), TypeError); + await assert.rejects(a.increment(bad), TypeError); + await assert.rejects(a.publish(bad, 1), TypeError); + await assert.rejects(a.subscribe(bad, () => {}), TypeError); + } + await assert.rejects(a.get(1 as never), TypeError); + assert.equal(await a.get("😀"), undefined, "a whole pair is fine"); + } finally { + await a.close(); + await b.close(); + } + }); + + test("refuses a step or a TTL that is not a sane number; ten years still expire", async () => { + const prefix = fresh(); + const store = mysqlStore(main, { prefix }); + try { + for (const by of [NaN, Infinity, -Infinity, "1"]) await assert.rejects(store.increment("n", by as never), TypeError, String(by)); + for (const ttl of [NaN, Infinity, "10"]) { + await assert.rejects(store.set("t", 1, ttl as never), TypeError, String(ttl)); + await assert.rejects(store.add("t", 1, ttl as never), TypeError, String(ttl)); + await assert.rejects(store.increment("t", 1, ttl as never), TypeError, String(ttl)); + } + for (const ttl of [0, -5, 315_576_000_001, 1e15]) await assert.rejects(store.set("t", 1, ttl), RangeError, String(ttl)); + assert.equal(await store.get("t"), undefined, "nothing was written"); + await store.set("t", 1, 315_576_000_000); + const [[row]] = await main.query("SELECT e IS NOT NULL AS has FROM inkan_store WHERE k = ?", [`${prefix}t`]); + assert.equal(Number(row!.has), 1, "ten years is a TTL, not none"); + } finally { + await store.close(); + } + }); + + test("increment on a value that is not a number throws a TypeError and leaves it", async () => { + const store = mysqlStore(main, { prefix: fresh() }); + try { + for (const value of ["not a number", "5", true, null, { n: 1 }, [1]]) { + await store.set("s", value, 60_000); + await assert.rejects(store.increment("s", 2), TypeError, `${JSON.stringify(value)} is not a number`); + assert.deepEqual(await store.get("s"), value, "and it stays as it was"); + } + await store.set("s", 5); + assert.equal(await store.increment("s", 2), 7, "a number counts on"); + } finally { + await store.close(); + } + }); + + test("a database error carries no key and no value: no SQL, no parameters, none in the message", async () => { + const table = `inkan_err_${randomBytes(4).toString("hex")}`; + // a column of the user's that reads the value as a number, so a set fails with "Truncated + // incorrect INTEGER value: ''" + await main.query( + `CREATE TABLE ${table} (k VARBINARY(255) NOT NULL PRIMARY KEY, v JSON NOT NULL, e DATETIME(3) NULL, ` + + "u BIGINT AS (CAST(JSON_UNQUOTE(v) AS SIGNED)) STORED, KEY inkan_e (e)) ENGINE=InnoDB", + ); + const store = mysqlStore(main, { table, events: `${table}_ev`, prefix: "secret-key-1:", sweep: 0 }); + try { + const err = await store.set("a", "secret-value-1").then( + () => assert.fail("the column refuses it"), + (e: unknown) => e as Error & Record, + ); + assert.equal(typeof err.code, "string"); + assert.equal(typeof err.errno, "number"); + assert.match(err.message, /incorrect INTEGER value/i); + assert.equal(err.sql, undefined); + const all = inspect(err, { depth: 10, showHidden: true }); + assert.ok(!all.includes("secret-value-1"), all); + assert.ok(!all.includes("secret-key-1"), all); + } finally { + await store.close(); + await main.query(`DROP TABLE IF EXISTS ${table}, ${table}_ev`); + } + }); + + test("a single connection is refused: transactions need a pool", async () => { + assert.throws(() => mysqlStore({ query: async () => [[], []] } as never), TypeError); + }); + }); +} diff --git a/packages/otel/CHANGELOG.md b/packages/otel/CHANGELOG.md index 9fe5006..8dbfa12 100644 --- a/packages/otel/CHANGELOG.md +++ b/packages/otel/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 0.2.0 + +- Works with inkan 0.8 (`inkan: ">=0.8.0 <1.0.0"`). +- **The span is the active one while the handler runs**: an `around` hook (inkan 0.8) runs the + route middleware and the handler in `context.with`, so `trace.getActiveSpan()` and + instrumented libraries find it. `ctx.traceContext` stays, for hooks and for code that + passes the context itself. +- `network.protocol.version` comes from `ctx.httpVersion`, so `inject()` requests have it too + (`1.1`). A request's span is kept in a `stateKey` slot instead of a side table. + ## 0.1.0 - `otel()`: a SERVER span per request, named `METHOD /route/:pattern`, with the HTTP diff --git a/packages/otel/README.md b/packages/otel/README.md index 23faba5..3fd85a0 100644 --- a/packages/otel/README.md +++ b/packages/otel/README.md @@ -63,13 +63,15 @@ GET /teas/:id SERVER | `ctx.span` | this request's span: `setAttribute`, `addEvent`, `recordException`. Non-recording (calls do nothing) for an ignored request | | `ctx.traceContext` | the OpenTelemetry context with the span in it, for children | -The span is not made the active one for the handler's async work: a hook cannot wrap it. -Start children under it with the context: +While the route middleware and the handler run, the span is the active one (an `around` +hook runs them in `context.with`), so `trace.getActiveSpan()`, `startActiveSpan` and +instrumented libraries (a database driver, `fetch`) put their spans under it, across +`await`s too, given the context manager every Node SDK sets up: ```ts const tracer = trace.getTracer("tea-shop"); app.get("/teas/:id", (ctx) => - tracer.startActiveSpan("load tea", {}, ctx.traceContext, async (span) => { + tracer.startActiveSpan("load tea", async (span) => { try { return await db.tea(ctx.params.id); } finally { @@ -79,6 +81,9 @@ app.get("/teas/:id", (ctx) => ); ``` +Hooks run outside it (`onRequest`, `preHandler`, `onSend`): there, start children under +`ctx.traceContext`, as in `tracer.startActiveSpan(name, {}, ctx.traceContext, fn)`. + A thrown error reaches the span as a problem only (`internal`): inkan keeps the error to `onError`. Record it there with `spanOf`: diff --git a/packages/otel/index.js b/packages/otel/index.js index badd366..7fb6b7d 100644 --- a/packages/otel/index.js +++ b/packages/otel/index.js @@ -8,7 +8,7 @@ // is up to the SDK the app sets up. Without one every call here is a no-op. import { context, propagation, ROOT_CONTEXT, SpanKind, SpanStatusCode, trace, INVALID_SPAN_CONTEXT } from "@opentelemetry/api"; -import { plugin } from "@vxnsin/inkan"; +import { plugin, stateKey } from "@vxnsin/inkan"; const NAME = "@inkanjs/otel"; const VERSION = "0.1.0"; @@ -22,11 +22,11 @@ const getter = { get: (carrier, key) => carrier[key.toLowerCase()], }; -const traced = new WeakMap(); // ctx → { span, context } +const traced = stateKey("otel"); // { span, context, problem } of a traced request /** The span of a request, or undefined when it has none (not traced, or before the hook ran). */ export function spanOf(ctx) { - return ctx ? traced.get(ctx)?.span : undefined; + return ctx?.state ? traced.get(ctx)?.span : undefined; } /** Host and port of a Host header; undefined for a broken one. */ @@ -76,7 +76,7 @@ export function otel(options = {}) { attributes["server.address"] = host.address; if (host.port) attributes["server.port"] = host.port; } - const version = ctx.req?.httpVersion; + const version = ctx.httpVersion; if (version) attributes["network.protocol.version"] = version; if (extra) Object.assign(attributes, extra(ctx)); const name = nameOf ? nameOf(ctx) : route ? `${method} ${route}` : method; @@ -88,6 +88,13 @@ export function otel(options = {}) { ctx.traceContext = active; }); + // the span is the active one while the route middleware and the handler run, so + // trace.getActiveSpan() and instrumented libraries find it (given a context manager) + app.around((ctx, next) => { + const t = traced.get(ctx); + return t ? context.with(t.context, next) : next(); + }); + app.onProblem((ctx, problem) => { const t = traced.get(ctx); if (!t) return; @@ -99,7 +106,7 @@ export function otel(options = {}) { app.onResponse((ctx, done) => { const t = traced.get(ctx); if (!t) return; - traced.delete(ctx); + traced.set(ctx, undefined); const { span, problem } = t; span.setAttribute("http.response.status_code", done.status); if (done.route && !ctx.route) span.setAttribute("http.route", done.route); @@ -114,7 +121,7 @@ export function otel(options = {}) { .decorateRequest("span", () => trace.wrapSpanContext(INVALID_SPAN_CONTEXT)) .decorateRequest("traceContext", () => context.active()); }, - { name: "otel", shared: true, inkan: ">=0.7.0 <0.8.0" }, + { name: "otel", shared: true, inkan: ">=0.8.0 <1.0.0" }, ); } diff --git a/packages/otel/package.json b/packages/otel/package.json index 112e5ba..33d19ad 100644 --- a/packages/otel/package.json +++ b/packages/otel/package.json @@ -1,6 +1,6 @@ { "name": "@inkanjs/otel", - "version": "0.1.0", + "version": "0.2.0", "description": "OpenTelemetry tracing for inkan: a server span per request, with the semantic conventions for HTTP.", "type": "module", "license": "MIT", @@ -35,14 +35,14 @@ }, "peerDependencies": { "@opentelemetry/api": "^1.4.0", - "@vxnsin/inkan": ">=0.7.0 <0.8.0" + "@vxnsin/inkan": ">=0.8.0 <1.0.0" }, "devDependencies": { "@opentelemetry/api": "^1.9.1", - "@vxnsin/inkan": "^0.7.0" + "@vxnsin/inkan": "^0.8.0" }, "scripts": { - "test": "node --test test.ts" + "test": "node --import ../../scripts/no-deprecations.js --test test.ts" }, "publishConfig": { "access": "public" diff --git a/packages/otel/test.ts b/packages/otel/test.ts index 8861122..f2ace76 100644 --- a/packages/otel/test.ts +++ b/packages/otel/test.ts @@ -1,11 +1,14 @@ import { test } from "node:test"; import assert from "node:assert/strict"; +import { AsyncLocalStorage } from "node:async_hooks"; import { randomBytes } from "node:crypto"; import http from "node:http"; import type { AddressInfo } from "node:net"; import { once } from "node:events"; import { + context, propagation, + ROOT_CONTEXT, trace, SpanKind, SpanStatusCode, @@ -13,6 +16,7 @@ import { type Attributes, type AttributeValue, type Context, + type ContextManager, type Exception, type Span, type SpanContext, @@ -154,6 +158,7 @@ test("a SERVER span per request, named by the route's pattern, with the HTTP sem "user_agent.original": "curl/8", "server.address": "api.example.com", "server.port": 8443, + "network.protocol.version": "1.1", "tea.id": "7", "http.response.status_code": 200, }); @@ -286,3 +291,49 @@ test("only the scope it is registered in", async () => { ["GET /in"], ); }); + +/** What an SDK sets up: the active context follows the async work, on AsyncLocalStorage. */ +class StorageContextManager implements ContextManager { + private readonly storage = new AsyncLocalStorage(); + active() { + return this.storage.getStore() ?? ROOT_CONTEXT; + } + with ReturnType>(ctx: Context, fn: F, thisArg?: ThisParameterType, ...args: A): ReturnType { + return this.storage.run(ctx, () => fn.call(thisArg, ...args)); + } + bind(_ctx: Context, target: T): T { + return target; + } + enable() { + return this; + } + disable() { + this.storage.disable(); + return this; + } +} + +test("the span is the active one while the handler runs, across awaits, and only for traced requests", async () => { + const manager = new StorageContextManager(); + context.setGlobalContextManager(manager); + try { + const tracer = new MemoryTracer(); + const app = inkan(quiet) + .register((scope) => { + scope + .register(otel({ tracer, ignore: (ctx) => ctx.path === "/in/health" })) + .get("/in/tea", async (ctx) => { + await new Promise((resolve) => setTimeout(resolve, 1)); + return { active: trace.getActiveSpan() === ctx.span, stored: trace.getSpan(ctx.traceContext) === ctx.span }; + }) + .get("/in/health", () => ({ none: trace.getActiveSpan() === undefined })); + }) + .get("/out", () => ({ none: trace.getActiveSpan() === undefined })); + assert.deepEqual((await app.inject({ url: "/in/tea" })).body, { active: true, stored: true }); + assert.deepEqual((await app.inject({ url: "/in/health" })).body, { none: true }); + assert.deepEqual((await app.inject({ url: "/out" })).body, { none: true }); + assert.equal(trace.getActiveSpan(), undefined, "not active after the request"); + } finally { + context.disable(); + } +}); diff --git a/packages/postgres/CHANGELOG.md b/packages/postgres/CHANGELOG.md new file mode 100644 index 0000000..a1033b3 --- /dev/null +++ b/packages/postgres/CHANGELOG.md @@ -0,0 +1,29 @@ +# Changelog + +## 0.1.0 + +- `postgresStore(db, { table, schema, prefix, create, sweep, listener })`: an inkan `Store` + (0.8) in one PostgreSQL table, on the app's own `pg` Pool or Client, or postgres.js. No + dependencies of its own. +- `add`, `take` and `increment` are one statement each (`insert … on conflict`, `delete … + returning`); an expired row counts as gone. Expiry by the database's clock. +- A TTL must be a finite number of ms, more than 0 and at most ten years (TypeError or + RangeError otherwise), as in the other stores. +- `increment` on a live key whose value is not a JSON number rejects with a `TypeError` and + leaves the value alone (it counted such a value as 0). +- Keys and channels under `prefix`, default `"inkan:"`, which ends in `:` (added when + missing) so two prefixes cannot run into each other's keys. A key or channel with NUL or a + lone surrogate is refused with a `TypeError`. Keys over 2048 bytes are kept under their + SHA-256 (`storedKey`), as in @inkanjs/mysql, instead of failing on the btree's size limit. +- Errors from the database come without keys and values: a new `Error` with `code`, + `severity`, `where` and the schema, table, column and constraint names, its message with + every quoted part replaced by `"…"`; pg's `detail` and postgres.js's `query` and + `parameters` are not passed on. +- Values as JSON text, so they come back exactly as `JSON.stringify` made them. +- `publish`/`subscribe` on LISTEN/NOTIFY (messages under 8000 bytes), on one connection + kept out of the pool; a dropped one is replaced. A connection whose LISTEN fails goes back + to the pool closed, without the store's handlers, so the retries leak nothing. +- Expired rows swept once at startup and then every minute, in batches of 10000 rows until + none are left (at most a million a sweep); `store.sweep()` runs one now, `close()` stops + the timer. Table and schema names checked and quoted, every value a parameter. +- Passes `storeConformance` from `@vxnsin/inkan/testing` on pg and postgres.js. diff --git a/packages/postgres/README.md b/packages/postgres/README.md new file mode 100644 index 0000000..56d2e24 --- /dev/null +++ b/packages/postgres/README.md @@ -0,0 +1,156 @@ +# @inkanjs/postgres + +A PostgreSQL `Store` for [inkan](https://github.com/inkanjs/inkan): background jobs, +sessions, logins, idempotency keys and rate limits kept in one table, so every process and +every server of an app sees the same state. It runs on the database client you already +have, `pg` (a Pool or a Client) or `postgres` (postgres.js), and has no dependencies of its +own. + +```sh +npm install @vxnsin/inkan @inkanjs/postgres pg # or postgres +``` + +```ts +import { inkan, rateLimit } from "@vxnsin/inkan"; +import pg from "pg"; +import { postgresStore } from "@inkanjs/postgres"; + +const pool = new pg.Pool({ connectionString: process.env.DATABASE_URL }); +const store = postgresStore(pool); // or postgresStore(postgres(url)) with postgres.js + +const app = inkan({ workers: "auto" }) + .register(rateLimit({ max: 100, window: 60_000, store })) + .job("/exports", { store }, async (job) => buildExport(job)); +``` + +One store serves everything that keeps state; each part keeps its keys apart by a prefix of +its own: + +```ts +import { session } from "@inkanjs/session"; +import { auth } from "@inkanjs/auth"; +import { idempotency } from "@inkanjs/idempotency"; + +app + .register(session({ secret, store })) + .register(auth({ /* … */ store })) + .register(idempotency({ store })); +``` + +With `workers`, inkan refuses jobs kept in memory; this store is `shared: true`, so jobs run +in any worker, and progress and cancels reach the others through LISTEN/NOTIFY. + +## Options + +`postgresStore(db, options?)` + +| option | default | | +| --- | --- | --- | +| `table` | `"inkan_store"` | the table; letters, digits and `_`, at most 63 | +| `schema` | the `search_path` | the schema the table is in; it must exist | +| `prefix` | `"inkan:"` | put in front of every key and channel, for two apps on one table; ends in `:` (added when missing) | +| `create` | `true` | create the table and its index when they are missing | +| `sweep` | `60000` | delete expired rows once at startup and then every this many ms; `0` turns it off | +| `listener` | from the pool | pg only: the connection LISTEN runs on (see below) | + +A sweep deletes 10000 rows a statement (`for update skip locked`, so two processes do not +wait on each other) until a statement comes back short, at most a million rows a sweep; the +next one goes on. `store.sweep()` runs one now. `store.close()` stops the sweep and the listening. The pool (or client, or `sql`) you passed +in stays open: end it yourself when the app stops. + +## The table + +```sql +create table if not exists "inkan_store" (key text primary key, value text not null, expires_at timestamptz); +create index if not exists "inkan_store_expires_at" on "inkan_store" (expires_at) where expires_at is not null; +``` + +With `create: false`, create it yourself (in a migration) with exactly these columns. +`value` is the JSON text, `text` and not `jsonb`: jsonb cannot hold `\u0000`, reorders the +keys of objects and drops duplicate ones, and a store has to give back what +`JSON.stringify` made. `value::jsonb` still works in a query of your own. + +## What it runs + +| method | SQL (one statement each) | +| --- | --- | +| `get` | `select … where key = $1 and (expires_at is null or expires_at > now())` | +| `set` | `insert … on conflict (key) do update` (value and TTL; no TTL clears it) | +| `delete` | `delete … where key = $1` | +| `add` | `insert … on conflict (key) do update … where` the row there has expired, `returning` | +| `take` | `delete … returning value`, the value only when it had not expired | +| `increment` | `insert … on conflict do update`: a new or expired row starts at `by` with the TTL, a live one adds `by` and keeps its TTL | +| `publish` | `select pg_notify($1, $2)` | +| `subscribe` | `LISTEN` | + +Every value is a parameter. `table` and `schema` are checked against +`/^[A-Za-z_][A-Za-z0-9_]*$/` before any SQL and then quoted; anything else throws. Channel +names never reach SQL as given: each channel is a database channel named by a hash of the +table, prefix and channel (`inkan_<40 hex>`), so any name fits the 63 bytes of an identifier. + +The driver is told apart by its methods: postgres.js has `unsafe` and `listen`, pg has +`query` (a Pool also `totalCount`). A single pg Client gets one query at a time. + +## Guarantees + +- `add`, `take` and `increment` are each one statement on a row lock, so they hold across + every process and server on the same database, table and prefix. +- **Expiry is by the database's clock**, `now()`, never the app's, so servers whose clocks + disagree still agree on what is over. An expired row counts as gone for every method + (`add` takes its key, `increment` starts over) before the sweep deletes it. +- A key is never gone before its TTL (microsecond timestamps) and never returned after it. +- `increment` counts in `numeric`: `0.1 + 0.2` is `0.3`. On a live key whose value is not a + JSON number (a string, even `"5"`, a boolean, `null`, an object) it rejects with a + `TypeError` and leaves the value as it was, as the other stores do. +- The suite from `@vxnsin/inkan/testing` (`storeConformance`) passes on a pg Pool, a pg + Client and postgres.js. + +## Security + +- **Publish/subscribe has no authentication.** A message is whatever arrives on the channel: + any role that can connect to the database can `NOTIFY` (or `select pg_notify(…)`) any + channel, without a grant, so anyone with a login to that database can send one, and + inkan's jobs act on them (a forged message can cancel a running job or report false + progress). The channel names are hashes of table, prefix and channel, which hides nothing + from someone who knows them. Give the app a database of its own, and `CONNECT` on it only + to the roles that should be able to talk to the app. Do not share the database with + tenants you do not trust. +- **Writing the store is trusting it.** Whoever can write the keys can forge sessions, + idempotent replies, job records and rate-limit counts, and whoever can read them reads + every session. Give the app's database user access to the store's table (`select, insert, + update, delete` on it) and nothing else, and nobody else write access to it. +- **Values are stored as JSON text**, as they are given: not encrypted, not signed. Keep + secrets out of them or encrypt them yourself; keys are stored as given too (under the + prefix), so do not put secrets in keys. +- **Errors carry no keys or values.** A failing statement rejects with a new `Error` that + keeps `code`, `severity`, `where` and the schema, table, column and constraint names, its + message with every quoted part replaced by `"…"`; pg's `detail` ("Key (value)=(…)") and + postgres.js's `query` and `parameters` are not passed on. Log them freely. + +## Limits + +- **NOTIFY payloads are under 8000 bytes.** `publish` rejects a larger message (as JSON) with + a `RangeError`. inkan's jobs send small messages; keep your own small too and put large + data in the store. +- **Publish/subscribe is at most once.** A message published while nobody listens, or while + the listening connection reconnects, is lost. A dropped pg listening connection is + replaced (tried every second); postgres.js reconnects on its own. +- With a pg Pool, the first `subscribe` takes one connection out of the pool for good (until + `close()`): size the pool for it, or pass a connected `pg.Client` as `listener`. Behind + PgBouncer in transaction mode LISTEN does not work; give it a direct connection. +- `now()` is the start of the transaction, so the TTL counts from when the statement began. +- The sweep runs in every process that has the store; the deletes overlap harmlessly. Rows + without a TTL stay until deleted. +- The database should be UTF-8, or values with characters its encoding lacks fail. +- **Keys and the prefix.** The prefix (default `"inkan:"`) ends in `:`; one without gets + it (`"app"` is `"app:"`), so prefix `"app"` with key `"1x"` and prefix `"app1"` with key + `"x"` stay apart. `""` puts nothing in front. Do not give one use a prefix that starts + another's (`"app:"` and `"app:sessions:"` can meet). A key or channel holding NUL (`\0`, + which PostgreSQL text cannot hold) or a lone UTF-16 surrogate is refused with a `TypeError`, + in all four inkan stores alike; in a value either is escaped by JSON and fine. +- **Long keys.** A key (with its prefix) longer than 2048 bytes of UTF-8 is kept under + `\x01sha256:` of it (`storedKey`), as in @inkanjs/mysql, so it fits the primary key's + btree. Two different keys share a row only if SHA-256 collides; keys starting with `\x01` + are hashed too, so no key can pose as a hash. +- Every call is a round trip. For thousands of increments a second, Redis is the better + store. diff --git a/packages/postgres/index.d.ts b/packages/postgres/index.d.ts new file mode 100644 index 0000000..4bfff9a --- /dev/null +++ b/packages/postgres/index.d.ts @@ -0,0 +1,57 @@ +import type { Store } from "@vxnsin/inkan"; + +/** A `pg` Pool or Client (or anything with the same `query`). A Pool is told apart by `totalCount`. */ +export interface PgQueryable { + query(text: string, values?: any[]): Promise<{ rows: any[] }>; +} + +/** porsager's postgres.js: the `sql` that `postgres(url)` returns. */ +export interface PostgresJsSql { + unsafe(query: string, parameters?: any[], ...rest: any[]): PromiseLike; + listen(channel: string, onnotify: (payload: string) => void, ...rest: any[]): Promise<{ unlisten(): Promise }>; +} + +export type PostgresStoreOptions = { + /** The table. Letters, digits and `_`, quoted. Default `"inkan_store"`. */ + table?: string; + /** The schema the table is in, which must exist. Default: the connection's `search_path`. */ + schema?: string; + /** + * Put in front of every key and channel, for two apps on one table. Ends in `:` (one is + * added when missing); `""` for none. Default `"inkan:"`. + */ + prefix?: string; + /** Create the table and its index when they are missing. Default true. */ + create?: boolean; + /** Delete expired rows every this many ms (an unref'd timer). 0 turns it off. Default 60000. */ + sweep?: number; + /** + * pg only: the connection LISTEN runs on. Default: one taken from the pool at the first + * `subscribe` and kept (a Client passed as `db` listens itself). One given here is the + * app's to connect and end. + */ + listener?: any; +}; + +/** A `Store` in PostgreSQL. `publish`/`subscribe` are always there, `shared` is true. */ +export interface PostgresStore extends Store { + readonly shared: true; + /** NOTIFY: the message as JSON must be under 8000 bytes, or this rejects with a RangeError. */ + publish(channel: string, message: unknown): Promise; + subscribe(channel: string, fn: (message: unknown) => void): Promise<() => void>; + /** Deletes expired rows now, as the periodic sweep does: 10000 a statement until none are left. */ + sweep(): Promise; + /** Stops the sweep and the listening; the connection taken from the pool goes back closed. The pool stays open. */ + close(): Promise; +} + +/** + * A `Store` in one PostgreSQL table on the app's own `pg` Pool/Client or postgres.js `sql`, + * seen by every process on the same database, table and prefix. Expiry is by the database's + * clock (`now()`); `add`, `take` and `increment` are each one statement. + */ +export declare function postgresStore(db: PgQueryable | PostgresJsSql, options?: PostgresStoreOptions): PostgresStore; +export default postgresStore; + +/** What a key is kept under: itself, or `\x01sha256:` when it is longer than 2048 bytes (or starts with `\x01`). */ +export declare function storedKey(key: string): string; diff --git a/packages/postgres/index.js b/packages/postgres/index.js new file mode 100644 index 0000000..1787907 --- /dev/null +++ b/packages/postgres/index.js @@ -0,0 +1,404 @@ +// A `Store` (from @vxnsin/inkan) in one PostgreSQL table, on a pool the app already has: +// `pg` (a Pool or a Client) or `postgres` (porsager's postgres.js). No dependencies. +// +// create table "inkan_store" (key text primary key, value text not null, expires_at timestamptz) +// +// Every expiry is worked out by the database (`now()`), never by the app's clock, so app +// servers whose clocks disagree still agree on what is over. An expired row counts as gone +// before the sweep deletes it. Identifiers are checked and quoted; every value is a parameter. + +import { createHash } from "node:crypto"; + +const IDENT = /^[A-Za-z_][A-Za-z0-9_]*$/; +/** pg_notify refuses a payload of 8000 bytes or more. */ +const NOTIFY_MAX = 8000; +const SWEEP_BATCH = 10_000; // rows one DELETE of the sweep takes, so no lock is held long +const SWEEP_BATCHES = 100; // DELETEs one sweep runs at most (a million rows); the next goes on + +const ident = (name, what) => { + if (typeof name !== "string" || !IDENT.test(name) || name.length > 63) { + throw new TypeError(`postgresStore: ${what} must be letters, digits and _ (not first), at most 63, not ${JSON.stringify(name)}`); + } + return `"${name}"`; +}; +const json = (value) => { + const s = JSON.stringify(value); + if (s === undefined) throw new TypeError("postgresStore: undefined is not a value a store keeps"); + return s; +}; +/** A key or channel as given, or a TypeError: a string without NUL and without a lone surrogate. */ +const checked = (key, what = "key") => { + if (typeof key !== "string") throw new TypeError(`postgresStore: a ${what} is a string, not ${typeof key}`); + if (key.includes("\0") || !key.isWellFormed()) throw new TypeError(`postgresStore: a ${what} cannot hold NUL or a lone surrogate`); + return key; +}; + +/** The prefix: "" for none, otherwise ending in ":" (added when missing), so a key never runs into another prefix. */ +const prefixOf = (prefix) => { + checked(prefix, "prefix"); + return prefix === "" || prefix.endsWith(":") ? prefix : `${prefix}:`; +}; + +/** Keys up to this many bytes of UTF-8 are kept as they are; a btree entry must stay under about 2.7 KB. */ +const KEY_MAX = 2048; + +/** What a key is kept under: itself, or `\x01sha256:` when it is longer than KEY_MAX bytes (as @inkanjs/mysql does). */ +export function storedKey(key) { + // \x01 starts only hashed keys, so a key that looks like a hash is hashed as well + if (Buffer.byteLength(key, "utf8") <= KEY_MAX && !key.startsWith("\x01")) return key; + return `\x01sha256:${createHash("sha256").update(key, "utf8").digest("hex")}`; +} + +/** A JSON number, as the text of a value. */ +const IS_NUMBER = /^-?(0|[1-9][0-9]*)([.][0-9]+)?([eE][-+]?[0-9]+)?$/; +/** Ten years in ms: the longest TTL taken. */ +const MAX_TTL = 315_576_000_000; +/** The TTL as a parameter: milliseconds as text, or null for none. */ +const ms = (ttl) => { + if (ttl === undefined) return null; + if (typeof ttl !== "number" || !Number.isFinite(ttl)) throw new TypeError(`postgresStore: a TTL is a finite number of milliseconds, not ${String(ttl)}`); + if (ttl <= 0 || ttl > MAX_TTL) throw new RangeError(`postgresStore: a TTL is more than 0 and at most ten years (${MAX_TTL} ms), not ${ttl}`); + return String(ttl); +}; +const parse = (row) => (row === undefined ? undefined : JSON.parse(row.value)); + +/** `query(text, params) → rows` and how to listen, for either driver. */ +function driver(db) { + if (db && typeof db.unsafe === "function" && typeof db.listen === "function") { + return { kind: "postgres.js", query: async (text, params) => [...(await db.unsafe(text, params))] }; + } + if (db && typeof db.query === "function") { + if (typeof db.totalCount === "number") return { kind: "pg", pool: true, query: async (text, params) => (await db.query(text, params)).rows }; + const send = queued(db); + return { kind: "pg", pool: false, query: async (text, params) => (await send(text, params)).rows }; + } + throw new TypeError("postgresStore: pass a pg Pool or Client, or a postgres.js sql"); +} + +/** + * `query` on one pg Client, one at a time: a Client runs one query at a time, and pg 9 drops + * the queue it keeps for callers that do not wait. + */ +function queued(conn) { + let tail = Promise.resolve(); + return (text, params) => { + const result = tail.then(() => conn.query(text, params)); + tail = result.then( + () => {}, + () => {}, + ); + return result; + }; +} + +/** The queue of each listening connection that is not the `db` passed in. */ +const queues = new WeakMap(); + +/** One CREATE per database handle and table, however many stores are made on it. */ +const created = new WeakMap(); +function ensureTable(db, query, table, tableName) { + let tables = created.get(db); + if (!tables) created.set(db, (tables = new Map())); + let ready = tables.get(table); + if (!ready) { + ready = (async () => { + try { + await query(`create table if not exists ${table} (key text primary key, value text not null, expires_at timestamptz)`, []); + await query(`create index if not exists ${ident(`${tableName}_expires_at`.slice(0, 63), "index")} on ${table} (expires_at) where expires_at is not null`, []); + } catch (err) { + // two processes creating it at once: one of them is told it exists, which is fine + if (err?.code !== "23505" && err?.code !== "42P07") throw err; + } + })(); + tables.set(table, ready); + ready.catch(() => tables.delete(table)); // a failed CREATE is tried again by the next store + } + return ready; +} + +/** + * A `Store` in one PostgreSQL table, on `db`: a `pg` Pool (or Client) or a postgres.js + * `sql`. Shared by every process that uses the same database, table and prefix. + */ +export function postgresStore(db, options = {}) { + const { kind, pool, query } = driver(db); + const tableName = options.table ?? "inkan_store"; + const table = (options.schema === undefined ? "" : `${ident(options.schema, "schema")}.`) + ident(tableName, "table"); + const prefix = prefixOf(options.prefix ?? "inkan:"); + const sweepEvery = options.sweep ?? 60_000; + const k = (key) => storedKey(prefix + checked(key)); + + const ready = options.create === false ? Promise.resolve() : ensureTable(db, query, table, tableName); + ready.catch(() => {}); // reported by the first call that waits for it + const run = async (text, params) => { + await ready; + return query(text, params); + }; + + const EXPIRES = `now() + $3::text::float8 * interval '1 millisecond'`; + const LIVE = `(expires_at is null or expires_at > now())`; + const OVER = `(s.expires_at is not null and s.expires_at <= now())`; + const NUMBER = `'^-?(0|[1-9][0-9]*)([.][0-9]+)?([eE][-+]?[0-9]+)?$'`; + const sql = { + get: `select value from ${table} where key = $1 and ${LIVE}`, + set: `insert into ${table} (key, value, expires_at) values ($1, $2, ${EXPIRES}) + on conflict (key) do update set value = excluded.value, expires_at = excluded.expires_at`, + delete: `delete from ${table} where key = $1`, + add: `insert into ${table} as s (key, value, expires_at) values ($1, $2, ${EXPIRES}) + on conflict (key) do update set value = excluded.value, expires_at = excluded.expires_at + where ${OVER} + returning true as ok`, + take: `delete from ${table} where key = $1 returning value, ${LIVE} as live`, + increment: `insert into ${table} as s (key, value, expires_at) values ($1, $2::text::numeric::text, ${EXPIRES}) + on conflict (key) do update set + value = case when ${OVER} then excluded.value + when s.value ~ ${NUMBER} then (s.value::numeric + $2::text::numeric)::text + else s.value end, + expires_at = case when ${OVER} then excluded.expires_at else s.expires_at end + returning value`, + // one batch: rows another process is deleting are skipped, not waited for + sweep: `with d as (delete from ${table} where ctid in ( + select ctid from ${table} where expires_at <= now() limit ${SWEEP_BATCH} for update skip locked + ) returning 1) select count(*)::int as n from d`, + }; + + let closed = false; + /** Deletes in batches until one comes back short, at most SWEEP_BATCHES of them. */ + const sweep = async () => { + for (let i = 0; i < SWEEP_BATCHES && !closed; i++) { + const [row] = await run(sql.sweep, []); + if (Number(row?.n) < SWEEP_BATCH) return; + } + }; + let sweeping; + /** The timer's sweep: one at a time; the next tries again, and reads skip expired rows anyway. */ + const background = () => { + sweeping ??= sweep() + .catch(() => {}) + .finally(() => { + sweeping = undefined; + }); + }; + let timer; + if (sweepEvery) { + // once now, so a process that never lives a whole `sweep` still deletes; then on a timer + background(); + timer = setInterval(background, sweepEvery); + timer.unref?.(); + } + + // publish/subscribe: LISTEN/NOTIFY, one database channel per store channel, named by a hash + // of the table and the channel, so any channel name fits (an identifier is at most 63 bytes) + /** @type {Map void>, ready: Promise }>} */ + const channels = new Map(); + const physical = (channel) => `inkan_${createHash("sha256").update(`${table}\0${prefix}${checked(channel, "channel")}`).digest("hex").slice(0, 40)}`; + const deliver = (name, payload) => { + const fns = channels.get(name)?.fns; + if (!fns || !fns.size) return; + let m; + try { + m = JSON.parse(payload); + } catch { + return; + } + for (const fn of [...fns]) fn(m); + }; + + // pg: one connection that listens, kept out of the pool; postgres.js listens on its own + let listening; // Promise<{ conn, owned }> + /** @type {Map Promise>} postgres.js unlisten per channel */ + const unlistens = new Map(); + /** A command on the listening connection; through the queue when that is the Client passed in. */ + const on = (conn, text) => (conn === db ? query(text, []) : queues.get(conn)(text)); + const onNotification = (msg) => deliver(msg.channel, msg.payload); + const retry = () => { + if (closed || !channels.size) return; + const t = setTimeout(() => listener().catch(retry), 1000); + t.unref?.(); + }; + const listener = () => + (listening ??= (async () => { + const owned = !options.listener && pool; + const conn = options.listener ?? (pool ? await db.connect() : db); + if (conn !== db && !queues.has(conn)) queues.set(conn, queued(conn)); + conn.on("notification", onNotification); + // a dropped connection: listen again on a new one; what was sent meanwhile is lost + const onError = (err) => { + conn.off("notification", onNotification); + conn.off("error", onError); + listening = undefined; + try { + conn.release(err); + } catch {} + retry(); + }; + if (owned) conn.on("error", onError); + try { + for (const name of channels.keys()) await on(conn, `listen "${name}"`); + } catch (err) { + // nothing of this try stays: no handler on the connection, and one from the pool goes + // back closed (it may be listening on some channels), so a retry each second leaks none + conn.off("notification", onNotification); + if (owned) { + conn.off("error", onError); + try { + conn.release(true); + } catch {} + } + throw err; + } + return { conn, owned }; + })().catch((err) => { + listening = undefined; + throw err; + })); + const unlisten = async (name) => { + if (kind === "postgres.js") { + const off = unlistens.get(name); + unlistens.delete(name); + await off?.(); + } else if (listening) { + const { conn } = await listening; + await on(conn, `unlisten "${name}"`); + } + }; + + return sanitized({ + shared: true, + + async get(key) { + return parse((await run(sql.get, [k(key)]))[0]); + }, + + async set(key, value, ttl) { + await run(sql.set, [k(key), json(value), ms(ttl)]); + }, + + async delete(key) { + await run(sql.delete, [k(key)]); + }, + + async add(key, value, ttl) { + return (await run(sql.add, [k(key), json(value), ms(ttl)])).length === 1; + }, + + async take(key) { + const row = (await run(sql.take, [k(key)]))[0]; + return row?.live ? JSON.parse(row.value) : undefined; + }, + + async increment(key, by = 1, ttl) { + if (typeof by !== "number" || !Number.isFinite(by)) throw new TypeError(`postgresStore: increment takes a finite number, not ${by}`); + const { value } = (await run(sql.increment, [k(key), String(by), ms(ttl)]))[0]; + // a live value that is not a number was left as it was + if (!IS_NUMBER.test(value)) throw new TypeError("postgresStore: increment on a key whose value is not a number"); + return Number(value); + }, + + async publish(channel, message) { + const payload = json(message); + if (Buffer.byteLength(payload) >= NOTIFY_MAX) { + throw new RangeError(`postgresStore: a message is at most ${NOTIFY_MAX - 1} bytes as JSON (NOTIFY), this one is ${Buffer.byteLength(payload)}`); + } + await query("select pg_notify($1, $2)", [physical(channel), payload]); + }, + + async subscribe(channel, fn) { + if (closed) throw new Error("postgresStore: closed"); + const name = physical(channel); + let entry = channels.get(name); + if (!entry) { + // the LISTEN once per channel; a second subscribe meanwhile waits for the same one + const ready = (async () => { + if (kind === "postgres.js") { + const { unlisten: off } = await db.listen(name, (payload) => deliver(name, payload)); + unlistens.set(name, off); + } else { + const { conn } = await listener(); + await on(conn, `listen "${name}"`); + } + })(); + channels.set(name, (entry = { fns: new Set(), ready })); + const mine = entry; + ready.catch(() => { + if (channels.get(name) === mine) channels.delete(name); + }); + } + await entry.ready; + const { fns } = entry; + const one = (m) => fn(m); + fns.add(one); + let done = false; + return () => { + if (done) return; + done = true; + fns.delete(one); + if (!fns.size && channels.get(name) === entry) { + channels.delete(name); + unlisten(name).catch(() => {}); + } + }; + }, + + /** Deletes what is over now, as the periodic sweep does. */ + sweep, + + /** + * Stops the sweep and ends listening: the connection the store took from the pool goes + * back closed. The pool (or client, or sql) passed in stays open; the app ends it. + */ + async close() { + closed = true; + clearInterval(timer); + const names = [...channels.keys()]; + channels.clear(); + if (kind === "postgres.js") { + await Promise.all([...unlistens.values()].map((off) => off().catch(() => {}))); + unlistens.clear(); + return; + } + if (!listening) return; + const l = await listening.catch(() => undefined); + listening = undefined; + if (!l) return; + l.conn.off("notification", onNotification); + if (l.owned) l.conn.release(true); + else for (const name of names) await on(l.conn, `unlisten "${name}"`).catch(() => {}); + }, + }); +} + +/** What of a driver's error is kept: none of these hold a key or a value. */ +const KEEP = ["code", "severity", "where", "schema", "table", "column", "constraint", "routine", "schema_name", "table_name", "column_name", "constraint_name"]; + +/** + * The store with every method's errors cleaned: pg puts values in `detail` ("Key (value)=(…)") + * and in messages ('invalid input syntax for type numeric: "…"'), postgres.js the whole + * statement and its parameters in `query` and `parameters`, and an app logs errors. The error + * thrown instead keeps `code`, `severity`, `where` and the names of schema, table, column and + * constraint, with the message's quoted parts replaced by "…". The store's own errors pass as + * they are. + */ +function sanitized(store) { + for (const name of ["get", "set", "delete", "add", "take", "increment", "publish", "subscribe", "sweep"]) { + const method = store[name]; + store[name] = async (...args) => { + try { + return await method(...args); + } catch (err) { + throw clean(err); + } + }; + } + return store; +} + +const clean = (err) => { + if (err instanceof Error && err.message.startsWith("postgresStore:")) return err; + const message = String(err?.message ?? err).replace(/"(?:[^"]|"")*"/g, '"…"'); + const out = new Error(`postgresStore: ${message}`); + for (const k of KEEP) if (err?.[k] !== undefined) out[k] = err[k]; + return out; +}; + +export default postgresStore; diff --git a/packages/postgres/package.json b/packages/postgres/package.json new file mode 100644 index 0000000..8ca9d74 --- /dev/null +++ b/packages/postgres/package.json @@ -0,0 +1,54 @@ +{ + "name": "@inkanjs/postgres", + "version": "0.1.0", + "description": "A PostgreSQL store for inkan: jobs, sessions, auth, idempotency keys and rate limits in one table, on the pool you already have.", + "type": "module", + "license": "MIT", + "author": "Vensin (https://github.com/vxnsin)", + "repository": { + "type": "git", + "url": "git+https://github.com/inkanjs/middleware.git", + "directory": "packages/postgres" + }, + "keywords": [ + "inkan", + "store", + "postgres", + "postgresql", + "pg", + "session", + "rate-limit", + "jobs" + ], + "exports": { + ".": { + "types": "./index.d.ts", + "default": "./index.js" + } + }, + "files": [ + "index.js", + "index.d.ts", + "README.md", + "CHANGELOG.md" + ], + "engines": { + "node": ">=22.18" + }, + "peerDependencies": { + "@vxnsin/inkan": ">=0.8.0 <1.0.0" + }, + "devDependencies": { + "@types/pg": "^8.23.1", + "@vxnsin/inkan": "^0.8.0", + "embedded-postgres": "^18.4.0-beta.17", + "pg": "^8.23.1", + "postgres": "^3.4.9" + }, + "scripts": { + "test": "node --import ../../scripts/no-deprecations.js --test test.ts" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/postgres/test.ts b/packages/postgres/test.ts new file mode 100644 index 0000000..4eead4b --- /dev/null +++ b/packages/postgres/test.ts @@ -0,0 +1,397 @@ +import { after, describe, it, test } from "node:test"; +import assert from "node:assert/strict"; +import { randomUUID } from "node:crypto"; +import { spawn } from "node:child_process"; +import { mkdtempSync } from "node:fs"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { inspect } from "node:util"; +import pg from "pg"; +import postgres from "postgres"; +import { storeConformance } from "@vxnsin/inkan/testing"; +import { postgresStore, storedKey, type PostgresStore } from "./index.js"; + +// The server: DATABASE_URL when set (CI runs a postgres:17 service), else embedded-postgres, +// whose binaries come with its npm package for this platform. Without either: skipped. +async function server(): Promise<{ url: string; stop(): Promise } | string> { + if (process.env.DATABASE_URL) return { url: process.env.DATABASE_URL, stop: async () => {} }; + try { + const { default: EmbeddedPostgres } = await import("embedded-postgres"); + const port = await freePort(); + const db = new EmbeddedPostgres({ + databaseDir: mkdtempSync(join(tmpdir(), "inkan-pg-")), + user: "postgres", + password: "postgres", + port, + persistent: false, + initdbFlags: ["--encoding=UTF8", "--locale=C"], // not the OS locale: on Windows that is WIN1252 + // PostgreSQL 18's I/O workers outlive a stop on Windows and hold the test runner open + postgresFlags: ["-c", "io_method=sync"], + onLog: () => {}, + onError: () => {}, + }); + await db.initialise(); + await db.start(); + return { url: `postgres://postgres:postgres@127.0.0.1:${port}/postgres`, stop: () => db.stop() }; + } catch (err) { + return `no PostgreSQL to test against: set DATABASE_URL, or install embedded-postgres for this platform (${(err instanceof Error ? err.message : String(err))})`; + } +} + +function freePort() { + return new Promise((resolve, reject) => { + const s = createServer().listen(0, "127.0.0.1", () => { + const { port } = s.address() as { port: number }; + s.close(() => resolve(port)); + }); + s.on("error", reject); + }); +} + +const srv = await server(); +if (typeof srv === "string") { + console.log(`# skipped: ${srv}`); + test("@inkanjs/postgres against a real server", { skip: srv }, () => {}); +} else { + const pool = new pg.Pool({ connectionString: srv.url, max: 20 }); + const client = new pg.Client({ connectionString: srv.url }); + await client.connect(); + const sql = postgres(srv.url, { onnotice: () => {}, max: 10 }); + const table = `inkan_test_${randomUUID().replaceAll("-", "")}`; + after(async () => { + await pool.query(`drop table if exists "${table}"`); + await pool.end(); + await client.end(); + await sql.end(); + await srv.stop(); + }); + + const fresh = () => `test:${randomUUID()}:`; + const close = (s: unknown) => (s as PostgresStore).close(); + + storeConformance(() => postgresStore(pool, { table, prefix: fresh() }), { name: "postgresStore on a pg Pool", close }); + storeConformance(() => postgresStore(client, { table, prefix: fresh() }), { name: "postgresStore on a pg Client", close }); + storeConformance(() => postgresStore(sql, { table, prefix: fresh() }), { name: "postgresStore on postgres.js", close }); + + describe("postgresStore", () => { + it("refuses a table or schema that is not a plain identifier, before any SQL", () => { + for (const bad of ['x"; drop table users; --', "1abc", "a-b", "a b", "", "ü", "a".repeat(64)]) { + assert.throws(() => postgresStore(pool, { table: bad }), TypeError, bad); + assert.throws(() => postgresStore(pool, { schema: bad }), TypeError, bad); + } + assert.throws(() => postgresStore({} as never), /pg Pool|postgres\.js/); + }); + + it("keeps keys and JSON text in its table; values are parameters", async () => { + const prefix = fresh(); + const store = postgresStore(pool, { table, prefix }); + const evil = `'); drop table "${table}"; --`; + await store.set(evil, { evil }, 60_000); + assert.deepEqual(await store.get(evil), { evil }); + const { rows } = await pool.query(`select value, expires_at > now() as later from "${table}" where key = $1`, [prefix + evil]); + assert.deepEqual(rows, [{ value: JSON.stringify({ evil }), later: true }]); + await store.set(evil, 1); + const again = await pool.query(`select expires_at from "${table}" where key = $1`, [prefix + evil]); + assert.equal(again.rows[0].expires_at, null, "set without a TTL clears it"); + await store.close(); + }); + + it("an expired row that is still there counts as gone for get, add, take and increment", async () => { + const prefix = fresh(); + const store = postgresStore(pool, { table, prefix, sweep: 0 }); + for (const key of ["g", "a", "t", "i"]) { + await pool.query(`insert into "${table}" values ($1, '41', now() - interval '1 second')`, [prefix + key]); + } + assert.equal(await store.get("g"), undefined); + assert.equal(await store.add("a", "new", 60_000), true); + assert.equal(await store.take("t"), undefined); + assert.equal(await store.increment("i", 1, 60_000), 1); + const { rows } = await pool.query(`select expires_at > now() as later from "${table}" where key = $1`, [prefix + "i"]); + assert.equal(rows[0].later, true, "the new count got the new TTL"); + await store.close(); + }); + + it("increment keeps the first TTL, and counts fractions exactly", async () => { + const prefix = fresh(); + const store = postgresStore(sql, { table, prefix }); + await store.increment("n", 1, 50_000); + const first = (await pool.query(`select expires_at from "${table}" where key = $1`, [prefix + "n"])).rows[0].expires_at; + await store.increment("n", 1, 900_000); + const second = (await pool.query(`select expires_at from "${table}" where key = $1`, [prefix + "n"])).rows[0].expires_at; + assert.deepEqual(second, first); + assert.equal(await store.increment("f", 0.1), 0.1); + assert.equal(await store.increment("f", 0.2), 0.3); + for (const value of ["not a number", "5", true, null, { n: 1 }]) { + await store.set("s", value, 60_000); + await assert.rejects(store.increment("s", 2), TypeError, `${JSON.stringify(value)} is not a number`); + assert.deepEqual(await store.get("s"), value, "and it stays as it was"); + } + await store.close(); + }); + + it("sweeps expired rows on its timer, and close() stops it", async () => { + const prefix = fresh(); + const store = postgresStore(pool, { table, prefix, sweep: 100 }); + await store.set("short", 1, 10); + await store.set("long", 1, 60_000); + await new Promise((r) => setTimeout(r, 400)); + const { rows } = await pool.query(`select key from "${table}" where key like $1 order by key`, [`${prefix}%`]); + assert.deepEqual(rows.map((r) => r.key), [`${prefix}long`]); + await store.close(); + }); + + it("sweep() deletes in batches until nothing expired is left", async () => { + const own = `inkan_sw_${randomUUID().replaceAll("-", "")}`; + const store = postgresStore(pool, { table: own, sweep: 0 }); + try { + await store.set("live", 1, 600_000); + await pool.query(`insert into "${own}" select 'old:' || i, '1', now() - interval '1 second' from generate_series(1, 25000) i`); + await store.sweep(); + const { rows } = await pool.query(`select count(*)::int as n from "${own}"`); + assert.equal(rows[0].n, 1, "25000 expired rows gone in one sweep, the live one kept"); + } finally { + await store.close(); + await pool.query(`drop table if exists "${own}"`); + } + }); + + it("a table in a schema of its own, and create: false leaves a missing table missing", async () => { + const schema = `inkan_schema_${randomUUID().replaceAll("-", "")}`; + await pool.query(`create schema "${schema}"`); + try { + const store = postgresStore(pool, { schema, table: "kv" }); + await store.set("k", "v"); + assert.equal((await pool.query(`select value from "${schema}"."kv"`)).rows[0].value, '"v"'); + await store.close(); + const none = postgresStore(pool, { schema, table: "missing", create: false }); + await assert.rejects(none.get("k"), /does not exist/); + await none.close(); + } finally { + await pool.query(`drop schema "${schema}" cascade`); + } + }); + + it("a LISTEN that fails gives its connection back closed and leaves no handler on it", async () => { + const p = new pg.Pool({ connectionString: srv.url, max: 2 }); + const taken: { client: pg.PoolClient; handlers: [string, (...args: unknown[]) => void][] }[] = []; + const released: (Error | boolean | undefined)[] = []; + const connect = p.connect.bind(p) as (...args: unknown[]) => Promise; + // pool.query takes its connection with a callback: only the store's own connect() is watched + (p as unknown as { connect: (...args: unknown[]) => unknown }).connect = (...args: unknown[]) => (args.length ? connect(...args) : take()); + const take = async () => { + const client = await connect(); + const handlers: [string, (...args: unknown[]) => void][] = []; + taken.push({ client, handlers }); + // what is put on it while the store has it (the pool puts its own on at release) + let out = true; + const on = client.on.bind(client) as (event: string, fn: (...args: unknown[]) => void) => unknown; + (client as unknown as { on: (event: string, fn: (...args: unknown[]) => void) => unknown }).on = (event, fn) => { + if (out) handlers.push([event, fn]); + return on(event, fn); + }; + const release = client.release.bind(client); + client.release = (err?: Error | boolean) => { + if (out) released.push(err); + out = false; + return release(err); + }; + const query = client.query.bind(client) as (...args: unknown[]) => Promise; + (client as unknown as { query: (...args: unknown[]) => Promise }).query = (text: unknown, ...rest: unknown[]) => + typeof text === "string" && text.startsWith("listen") ? Promise.reject(new Error("LISTEN refused")) : query(text, ...rest); + return client; + }; + const store = postgresStore(p, { table, prefix: fresh(), sweep: 0 }); + try { + await assert.rejects(store.subscribe("c", () => {}), /LISTEN refused/); + await assert.rejects(store.subscribe("c", () => {}), /LISTEN refused/); + assert.equal(taken.length, 2, "each try takes a connection"); + for (const { client, handlers } of taken) { + assert.ok(handlers.length >= 2, "the store put its handlers on it"); + for (const [event, fn] of handlers) assert.ok(!client.listeners(event).includes(fn), `the ${event} handler is gone`); + } + assert.equal(p.totalCount - p.idleCount, 0, "none is held out of the pool"); + assert.deepEqual(released, [true, true], "both went back closed"); + } finally { + await store.close(); + for (const { client } of taken) { + try { + client.release(true); // one that leaked would hold p.end() forever + } catch {} + } + await p.end(); + } + }); + + it("a prefix ends in ':' (one is added); keys with NUL or a lone surrogate are refused", async () => { + const base = `p${randomUUID().replaceAll("-", "")}`; + const a = postgresStore(pool, { table, prefix: base, sweep: 0 }); + const b = postgresStore(pool, { table, prefix: `${base}1`, sweep: 0 }); + const plain = postgresStore(pool, { table, sweep: 0 }); + try { + await a.set("1x", "a"); + await b.set("x", "b"); + assert.equal(await a.get("1x"), "a", "prefix p + key 1x is not prefix p1 + key x"); + assert.equal((await pool.query(`select 1 from "${table}" where key = $1`, [`${base}:1x`])).rows.length, 1); + const k = randomUUID(); + await plain.set(k, 1); + assert.equal((await pool.query(`select 1 from "${table}" where key = $1`, [`inkan:${k}`])).rows.length, 1, "the default prefix is inkan:"); + assert.throws(() => postgresStore(pool, { table, prefix: "a\0" }), TypeError); + for (const bad of ["a\0b", "\uD800", "x\uDC00y"]) { + await assert.rejects(a.set(bad, 1), TypeError, JSON.stringify(bad)); + await assert.rejects(a.get(bad), TypeError); + await assert.rejects(a.increment(bad), TypeError); + await assert.rejects(a.publish(bad, 1), TypeError); + await assert.rejects(a.subscribe(bad, () => {}), TypeError); + } + await assert.rejects(a.get(1 as never), TypeError); + } finally { + await a.close(); + await b.close(); + await plain.close(); + } + }); + + it("keys longer than 2048 bytes are kept under their SHA-256, as in the MySQL store", async () => { + const prefix = fresh(); + const store = postgresStore(pool, { table, prefix, sweep: 0 }); + try { + const long = "k".repeat(5000); + const other = "k".repeat(4999) + "j"; + await store.set(long, 1); + await store.set(other, 2); + assert.equal(await store.get(long), 1); + assert.equal(await store.get(other), 2); + assert.equal(await store.increment(long, 1), 2); + assert.equal(await store.take(long), 2); + assert.equal(await store.add(long, 3, 60_000), true); + const hashed = storedKey(prefix + other); + assert.match(hashed, /^\x01sha256:[0-9a-f]{64}$/); + assert.equal((await pool.query(`select 1 from "${table}" where key = $1`, [hashed])).rows.length, 1); + assert.equal(await store.get(storedKey(other)), undefined, "a key that looks like a hash is not the hash"); + assert.equal(storedKey("ü".repeat(1024)), "ü".repeat(1024), "2048 bytes stay"); + assert.notEqual(storedKey("ü".repeat(1025)), "ü".repeat(1025), "2050 bytes are hashed"); + } finally { + await store.close(); + } + }); + + it("a database error carries no key and no value: no query, no parameters, no detail", async () => { + const own = `inkan_err_${randomUUID().replaceAll("-", "")}`; + const setup = postgresStore(pool, { table: own, sweep: 0 }); + await setup.get("x"); // makes the table + await setup.close(); + // a unique index of the user's on the value: a second key with the same value fails, and + // PostgreSQL says which value in the detail (Key (value)=(...) already exists) + await pool.query(`create unique index on "${own}" (value)`); + try { + for (const db of [pool, sql]) { + const store = postgresStore(db, { table: own, prefix: "secret-key-1:", sweep: 0 }); + try { + await store.set("a", "secret-value-1"); + const err = await store.set("b", "secret-value-1").then( + () => assert.fail("the unique index refuses it"), + (e: unknown) => e as Error & Record, + ); + assert.equal(err.code, "23505"); + assert.match(err.message, /duplicate key/); + for (const k of ["detail", "query", "parameters", "args"]) assert.equal(err[k], undefined, k); + const all = inspect(err, { depth: 10, showHidden: true }); + assert.ok(!all.includes("secret-value-1"), all); + assert.ok(!all.includes("secret-key-1"), all); + await store.delete("a"); + } finally { + await store.close(); + } + } + } finally { + await pool.query(`drop table if exists "${own}"`); + } + }); + + it("refuses a TTL that is not a finite number of ms up to ten years", async () => { + const store = postgresStore(pool, { table, prefix: fresh(), sweep: 0 }); + try { + for (const ttl of [NaN, Infinity, "10"]) await assert.rejects(store.set("x", 1, ttl as never), TypeError, String(ttl)); + for (const ttl of [0, -5, 315_576_000_001]) { + await assert.rejects(store.set("x", 1, ttl), RangeError, String(ttl)); + await assert.rejects(store.add("x", 1, ttl), RangeError, String(ttl)); + await assert.rejects(store.increment("x", 1, ttl), RangeError, String(ttl)); + } + assert.equal(await store.get("x"), undefined, "nothing was written"); + } finally { + await store.close(); + } + }); + + it("refuses a message NOTIFY cannot carry", async () => { + const store = postgresStore(pool, { table, prefix: fresh() }); + await assert.rejects(store.publish("c", "x".repeat(8000)), RangeError); + await store.publish("c", "x".repeat(7000)); + await store.close(); + }); + + it("two stores on one database, each on another driver, see each other's keys and messages", async () => { + const prefix = fresh(); + const a = postgresStore(pool, { table, prefix }); + const b = postgresStore(sql, { table, prefix }); + try { + await a.set("shared", { from: "a" }); + assert.deepEqual(await b.get("shared"), { from: "a" }); + const won = await Promise.all(Array.from({ length: 20 }, (_, i) => (i % 2 ? a : b).add("lock", i, 60_000))); + assert.equal(won.filter(Boolean).length, 1); + const seen: unknown[] = []; + const off = await b.subscribe("news", (m) => seen.push(m)); + const back: unknown[] = []; + const offBack = await a.subscribe("news", (m) => back.push(m)); + await a.publish("news", { hello: "b" }); + await until(() => seen.length > 0 && back.length > 0); + assert.deepEqual(seen, [{ hello: "b" }]); + assert.deepEqual(back, [{ hello: "b" }]); + off(); + offBack(); + } finally { + await a.close(); + await b.close(); + } + }); + + it("another process counts, writes and publishes on the same rows", async () => { + const prefix = fresh(); + const store = postgresStore(pool, { table, prefix }); + const seen: unknown[] = []; + const off = await store.subscribe("from-child", (m) => seen.push(m)); + const child = spawn( + process.execPath, + [ + "--input-type=module", + "-e", + `import pg from "pg"; + import { postgresStore } from "./index.js"; + const pool = new pg.Pool({ connectionString: process.env.URL }); + const store = postgresStore(pool, { table: process.env.TABLE, prefix: process.env.PREFIX }); + await Promise.all(Array.from({ length: 50 }, () => store.increment("count", 1, 60000))); + await store.set("written", "by the child"); + await store.publish("from-child", { done: true }); + await store.close(); + await pool.end();`, + ], + { cwd: import.meta.dirname, env: { ...process.env, URL: srv.url, TABLE: table, PREFIX: prefix }, stdio: "inherit" }, + ); + const exited = new Promise((resolve) => child.on("exit", resolve)); + await Promise.all(Array.from({ length: 50 }, () => store.increment("count", 1, 60_000))); + assert.equal(await exited, 0); + assert.equal(await store.get("count"), 100); + assert.equal(await store.get("written"), "by the child"); + await until(() => seen.length > 0); + assert.deepEqual(seen, [{ done: true }]); + off(); + await store.close(); + }); + }); +} + +async function until(ok: () => boolean, ms = 5000) { + const end = Date.now() + ms; + while (!ok() && Date.now() < end) await new Promise((r) => setTimeout(r, 10)); +} diff --git a/packages/proxy/CHANGELOG.md b/packages/proxy/CHANGELOG.md index dc381c7..76fddec 100644 --- a/packages/proxy/CHANGELOG.md +++ b/packages/proxy/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## 0.2.0 + +- Works with inkan 0.8 (`inkan: ">=0.8.0 <1.0.0"`). +- Its routes come from `app.all()` and its mount path from inkan's `joinPath` (inkan 0.8), not + from a loop and a copy of its own. `prefix` stays the proxy's own option, never read from + `register`; `register(proxy(…), {}, { prefix })` mounts it under that prefix. + ## 0.1.0 - `proxy({ prefix, to })`: every method under a prefix to an upstream; `proxyTo(to)`: one diff --git a/packages/proxy/README.md b/packages/proxy/README.md index f8c0480..db97cb5 100644 --- a/packages/proxy/README.md +++ b/packages/proxy/README.md @@ -36,7 +36,7 @@ GET /status?verbose=1 → GET http://status:9000/health?verbose=1 | option | default | | | --- | --- | --- | | `to` | required | the upstream: `http://host:port`, or with a path every request goes under (`http://legacy:8080/api`) | -| `prefix` | `/` | for `proxy()`: the path it takes, in the scope it is registered in | +| `prefix` | `/` | for `proxy()`: the path it takes, in the scope it is registered in (and under a mount: `register(proxy(…), {}, { prefix: "/v1" })`) | | `rewrite` | none | `(path) => string`: the path to ask for, from the request's path (for `proxy()`, the part after the prefix) | | `forwardCredentials` | `false` | pass the client's `cookie` and `authorization` on (see above) | | `headers` | none | headers to set on the upstream request, `null` removes one; or `(ctx) => headers` | diff --git a/packages/proxy/index.js b/packages/proxy/index.js index 30ba7ce..082a7ed 100644 --- a/packages/proxy/index.js +++ b/packages/proxy/index.js @@ -11,7 +11,7 @@ import http from "node:http"; import https from "node:https"; import { Readable, pipeline } from "node:stream"; -import { HttpProblem, plugin, reply, t } from "@vxnsin/inkan"; +import { HttpProblem, joinPath, plugin, reply, t } from "@vxnsin/inkan"; // hop-by-hop: they describe one connection, not the message (RFC 9110 7.6.1) const HOP = new Set(["connection", "keep-alive", "transfer-encoding", "upgrade", "te", "trailer", "trailers", "proxy-connection", "proxy-authenticate", "proxy-authorization"]); @@ -326,10 +326,10 @@ export function proxyTo(to, opts = {}) { export function proxy(opts) { const o = options(opts); const target = targetOf(o.to); - const prefix = "/" + String(o.prefix ?? "").split("/").filter(Boolean).join("/"); + const prefix = joinPath("/", String(o.prefix ?? "")); return plugin( (app) => { - const mount = "/" + [app.prefix, prefix].join("/").split("/").filter(Boolean).join("/"); + const mount = joinPath(app.prefix, prefix); const handler = (ctx) => { let rest = ctx.path.startsWith(mount) ? ctx.path.slice(mount === "/" ? 0 : mount.length) : "/" + ctx.params.path; rest = rest.replace(/\/{2,}/g, "/"); @@ -340,9 +340,9 @@ export function proxy(opts) { }; const spec = { body: t.stream(), hidden: o.hidden ?? true, meta: o.meta }; const at = prefix === "/" ? "/*path" : prefix + "/*path"; - for (const m of ["get", "post", "put", "patch", "delete"]) app[m](at, spec, handler); + app.all(at, spec, handler); // GET, POST, PUT, PATCH and DELETE; HEAD comes with GET }, - { name: "proxy", inkan: ">=0.7.0 <0.8.0" }, + { name: "proxy", inkan: ">=0.8.0 <1.0.0" }, ); } diff --git a/packages/proxy/package.json b/packages/proxy/package.json index 1c754e4..d7a280a 100644 --- a/packages/proxy/package.json +++ b/packages/proxy/package.json @@ -1,6 +1,6 @@ { "name": "@inkanjs/proxy", - "version": "0.1.0", + "version": "0.2.0", "description": "A streaming reverse proxy for inkan: a prefix or one route handed on to a server behind it, on node:http alone.", "type": "module", "license": "MIT", @@ -34,13 +34,13 @@ "node": ">=22.18" }, "peerDependencies": { - "@vxnsin/inkan": ">=0.7.0 <0.8.0" + "@vxnsin/inkan": ">=0.8.0 <1.0.0" }, "devDependencies": { - "@vxnsin/inkan": "^0.7.0" + "@vxnsin/inkan": "^0.8.0" }, "scripts": { - "test": "node --test test.ts" + "test": "node --import ../../scripts/no-deprecations.js --test test.ts" }, "publishConfig": { "access": "public" diff --git a/packages/proxy/test.ts b/packages/proxy/test.ts index ab0bd9e..1eb784c 100644 --- a/packages/proxy/test.ts +++ b/packages/proxy/test.ts @@ -286,11 +286,21 @@ test("in a scope registered with a prefix, and the app's hooks still run", async const up = await serve(t, echo); const app = inkan(quiet) .onRequest((ctx) => (ctx.rawHeaders["x-block"] ? ctx.reply(403, { blocked: true }) : undefined)) - .register((scope) => void scope.register(proxy({ prefix: "/p", to: up })), { prefix: "/v1" }); + .register((scope) => void scope.register(proxy({ prefix: "/p", to: up })), {}, { prefix: "/v1" }); assert.equal((await app.inject({ url: "/v1/p/x" })).body.url, "/x"); assert.equal((await app.inject({ url: "/v1/p/x", headers: { "x-block": "1" } })).status, 403); }); +test("mounted by register(plugin, options, { prefix }): its own prefix goes under the mount", async (t) => { + const up = await serve(t, echo); + const app = inkan(quiet) + .register(proxy({ prefix: "/p", to: up }), {}, { prefix: "/v2" }) + .register(proxy({ to: up + "/api" }), {}, { prefix: "/whole" }); + assert.equal((await app.inject({ url: "/v2/p/x?q=1" })).body.url, "/x?q=1"); + assert.equal((await app.inject({ url: "/whole/y" })).body.url, "/api/y"); + assert.equal((await app.inject({ url: "/p/x" })).status, 404); +}); + test("encoded slashes, backslashes and NUL, also double-encoded, are a 400 before the upstream is asked", async (t) => { const asked: string[] = []; const up = await serve(t, (req, res) => { @@ -370,7 +380,7 @@ test("Location on the upstream's origin is turned into the public path; others a .register(proxy({ prefix: "/raw", to: up + "/api", rewriteLocation: false })) .register(proxy({ prefix: "/fn", to: up + "/api", rewriteLocation: (l) => l.replace(up, "https://pub.example") })) .get("/one", proxyTo(up + "/status")) - .register((s) => void s.register(proxy({ prefix: "/p", to: up })), { prefix: "/v1" }); + .register((s) => void s.register(proxy({ prefix: "/p", to: up })), {}, { prefix: "/v1" }); const loc = async (path: string, to: string) => (await app.inject({ url: `${path}?to=${encodeURIComponent(to)}` })).headers.location; assert.equal(await loc("/legacy/x", "UP/api/login?next=%2Fa#top"), "/legacy/login?next=%2Fa#top"); assert.equal(await loc("/legacy/x", "UP/api"), "/legacy"); diff --git a/packages/redis/CHANGELOG.md b/packages/redis/CHANGELOG.md new file mode 100644 index 0000000..2d30c00 --- /dev/null +++ b/packages/redis/CHANGELOG.md @@ -0,0 +1,19 @@ +# Changelog + +## 0.1.0 + +- `redisStore(client, { prefix, subscriber })`: an inkan `Store` (0.8) in Redis, on the + app's own node-redis (v4+) or ioredis client. No dependencies of its own. +- `add` is `SET NX`, `take` is `GETDEL` (a Lua script before Redis 6.2), `increment` one + Lua script that gives only a new key its TTL. Every key and channel under `prefix`. +- A TTL must be a finite number of ms, more than 0 and at most ten years (TypeError or + RangeError otherwise), as in the other stores. +- `increment` on a key whose value is not a JSON number rejects with a `TypeError` (it was + the Redis error), as in the other stores. +- A prefix ends in `:` (one is added when missing), so two prefixes cannot run into each + other's keys. A key or channel with NUL or a lone surrogate is refused with a `TypeError`. +- Errors from the server come without keys and values: a new `Error` with the server's + message; ioredis's `command` (the command with its key and value) is not passed on. +- `publish`/`subscribe` on Redis Pub/Sub, with a duplicated connection made at the first + `subscribe` and closed by `store.close()`. +- Passes `storeConformance` from `@vxnsin/inkan/testing` on both clients. diff --git a/packages/redis/README.md b/packages/redis/README.md new file mode 100644 index 0000000..631ad5e --- /dev/null +++ b/packages/redis/README.md @@ -0,0 +1,118 @@ +# @inkanjs/redis + +A Redis `Store` for [inkan](https://github.com/inkanjs/inkan): background jobs, sessions, +logins, idempotency keys and rate limits kept in Redis, so every process and every server +of an app sees the same state. It runs on the Redis client you already have, `redis` +(node-redis v4 or later) or `ioredis`, and has no dependencies of its own. + +```sh +npm install @vxnsin/inkan @inkanjs/redis redis # or ioredis +``` + +```ts +import { inkan, rateLimit } from "@vxnsin/inkan"; +import { createClient } from "redis"; +import { redisStore } from "@inkanjs/redis"; + +const client = createClient({ url: process.env.REDIS_URL }); +await client.connect(); +const store = redisStore(client); // or redisStore(new Redis(url)) with ioredis + +const app = inkan({ workers: "auto" }) + .register(rateLimit({ max: 100, window: 60_000, store })) + .job("/exports", { store }, async (job) => buildExport(job)); +``` + +One store serves everything that keeps state; each part keeps its keys apart by a prefix of +its own: + +```ts +import { session } from "@inkanjs/session"; +import { auth } from "@inkanjs/auth"; +import { idempotency } from "@inkanjs/idempotency"; + +app + .register(session({ secret, store })) + .register(auth({ /* … */ store })) + .register(idempotency({ store })); +``` + +With `workers`, inkan refuses jobs kept in memory; this store is `shared: true`, so jobs run +in any worker, and progress and cancels reach the others through publish/subscribe. + +## Options + +`redisStore(client, options?)` + +| option | default | | +| --- | --- | --- | +| `prefix` | `"inkan:"` | put in front of every key and channel; ends in `:` (added when missing); give two apps on one server their own | +| `subscriber` | `client.duplicate()` | the connection SUBSCRIBE runs on; made at the first `subscribe`, closed by `close()`. One you pass is yours to open and close | + +`store.close()` closes only the subscriber connection the store made. The client you passed +in stays open: quit it yourself when the app stops. + +## What it sends + +| method | Redis | +| --- | --- | +| `get` | `GET` | +| `set` | `SET key json PX ttl`, or plain `SET`, which clears an old TTL | +| `delete` | `DEL` | +| `add` | `SET key json NX [PX ttl]` | +| `take` | `GETDEL`; on a server before 6.2 a Lua `GET` + `DEL` instead | +| `increment` | one Lua script: `INCRBY` (`INCRBYFLOAT` for a fraction), `PEXPIRE` only when the key was not there | +| `publish` | `PUBLISH prefix+channel json` | +| `subscribe` | `SUBSCRIBE` on the second connection, one per channel however many listeners | + +The client is told apart by its methods: `call` is ioredis, `sendCommand` node-redis. Any +other object is refused. + +## Guarantees + +- `add`, `take` and `increment` are each one atomic step on the server, so they hold + across every process and every server that uses the same Redis and prefix. +- A key with a TTL is gone once it is over (Redis expires keys to the millisecond). TTLs are + rounded up to whole milliseconds, never down, so a key never goes early. +- `increment` gives a key that was not there (or whose TTL was over) the TTL, and keeps + the TTL of one that was: a rate-limit window stays fixed. +- Values are JSON: `get` returns what `JSON.stringify` made of the value, `undefined` for a + missing key, `null` only for a stored `null`. +- The suite from `@vxnsin/inkan/testing` (`storeConformance`) passes on both clients. + +## Security + +- **Publish/subscribe has no authentication.** A message is whatever arrives on the channel: + anyone who can `PUBLISH` to `` on the server can send one, and inkan's + jobs act on them (a forged message can cancel a running job or report false progress). Use + a Redis ACL user for the app that may only touch its prefix: `+@all ~inkan:* &inkan:*` (or + narrower), and no other user on the same channels. Do not share the server with tenants + you do not trust. +- **Writing the store is trusting it.** Whoever can write the keys can forge sessions, + idempotent replies, job records and rate-limit counts, and whoever can read them reads + every session. Give the app's Redis user access to the store's keys (`~inkan:*`) and + channels (`&inkan:*`) and nothing else, and nobody else write access to them. +- **Values are stored as JSON text**, as they are given: not encrypted, not signed. Keep + secrets out of them or encrypt them yourself; keys are stored as given too (under the + prefix), so do not put secrets in keys. +- **Errors carry no keys or values.** A failing command rejects with a new `Error` that has + the server's message only; ioredis's `command` (with the key and value in its arguments) + is not passed on. Log them freely. + +## Limits + +- **Keys and the prefix.** The prefix (default `"inkan:"`) ends in `:`; one without gets + it (`"app"` is `"app:"`), so prefix `"app"` with key `"1x"` and prefix `"app1"` with key + `"x"` stay apart. `""` puts nothing in front. Do not give one use a prefix that starts + another's (`"app:"` and `"app:sessions:"` can meet). A key or channel holding NUL (`\0`) or a + lone UTF-16 surrogate is refused with a `TypeError`, in all four inkan stores alike. +- **Publish/subscribe is Redis Pub/Sub:** at most once, nothing kept. A message published + while a subscriber is reconnecting is lost. inkan's jobs live with that (the job record + in the store is the truth, messages only make others hear sooner). +- `increment` on a key whose value is not a JSON number (a string, even `"5"`, a boolean, + `null`, an object) rejects with a `TypeError` and leaves the value as it was, as the other + stores do. +- Redis Cluster works with ioredis' `Cluster` and node-redis' `createCluster` as far as the + commands go (every command touches one key), but is not tested here. +- Memory: keys without a TTL stay until deleted. Set `maxmemory-policy` with that in mind; + `volatile-*` policies only ever evict keys that have a TTL. diff --git a/packages/redis/index.d.ts b/packages/redis/index.d.ts new file mode 100644 index 0000000..3a8adc2 --- /dev/null +++ b/packages/redis/index.d.ts @@ -0,0 +1,47 @@ +import type { Store } from "@vxnsin/inkan"; + +/** A connected client from `redis` (node-redis v4 or later): `createClient(...)` after `connect()`. */ +export interface NodeRedisClient { + sendCommand(args: any[], ...rest: any[]): Promise; + duplicate(...args: any[]): any; +} + +/** A client from `ioredis`: `new Redis(...)`. */ +export interface IoRedisClient { + call(command: string, ...args: any[]): Promise; + duplicate(...args: any[]): any; +} + +/** Either client; which one it is shows by its methods (`call` is ioredis, `sendCommand` node-redis). */ +export type RedisClient = NodeRedisClient | IoRedisClient; + +export type RedisStoreOptions = { + /** + * Put in front of every key and channel. Give two apps on one server their own. Ends in `:` + * (one is added when missing); `""` for none. Default `"inkan:"`. + */ + prefix?: string; + /** + * The connection SUBSCRIBE runs on, which can do nothing else while it listens. Default: a + * `client.duplicate()` made at the first `subscribe`, and closed by `close()`. One given + * here is the app's to open and close. + */ + subscriber?: RedisClient; +}; + +/** A `Store` in Redis. `publish`/`subscribe` are always there, `shared` is true. */ +export interface RedisStore extends Store { + readonly shared: true; + publish(channel: string, message: unknown): Promise; + subscribe(channel: string, fn: (message: unknown) => void): Promise<() => void>; + /** Closes the subscriber connection the store made itself. The client passed in stays open. */ + close(): Promise; +} + +/** + * A `Store` in Redis on the app's own client (node-redis v4+ or ioredis), seen by every + * process on the same server and prefix. `add` is `SET NX`, `take` is `GETDEL` (a Lua script + * before Redis 6.2), `increment` one Lua script that gives only a new key its TTL. + */ +export declare function redisStore(client: RedisClient, options?: RedisStoreOptions): RedisStore; +export default redisStore; diff --git a/packages/redis/index.js b/packages/redis/index.js new file mode 100644 index 0000000..2f215fc --- /dev/null +++ b/packages/redis/index.js @@ -0,0 +1,246 @@ +// A `Store` (from @vxnsin/inkan) in Redis, on a client the app already has: node-redis (v4 and +// later) or ioredis. Nothing to install besides the client the user picked. +// +// set SET key json [PX ttl] (without PX the TTL is cleared) +// add SET key json NX [PX ttl] +// take GETDEL key (a Lua GET + DEL before Redis 6.2) +// increment one Lua script: INCRBY, PEXPIRE only for a key that was not there +// publish PUBLISH; subscribe on a second connection, made only when asked + +/** INCRBY (INCRBYFLOAT for a step or a number that is not whole), and the TTL only on a new key. */ +const INCREMENT = `local existed = redis.call('EXISTS', KEYS[1]) +local n = redis.pcall('INCRBY', KEYS[1], ARGV[1]) +if type(n) == 'table' and n.err then + n = redis.pcall('INCRBYFLOAT', KEYS[1], ARGV[1]) + if type(n) == 'table' and n.err then + if string.find(n.err, 'not a valid float', 1, true) then return redis.error_reply('ERR inkan: not a number') end + return n + end +end +if existed == 0 and ARGV[2] ~= '' then redis.call('PEXPIRE', KEYS[1], ARGV[2]) end +return n`; + +/** GETDEL for a server older than 6.2: a script runs as one step, like a MULTI. */ +const TAKE = `local v = redis.call('GET', KEYS[1]) +if v then redis.call('DEL', KEYS[1]) end +return v`; + +const text = (reply) => (reply == null ? reply : typeof reply === "string" ? reply : Buffer.isBuffer(reply) ? reply.toString("utf8") : String(reply)); +const parse = (reply) => { + const s = text(reply); + return s == null ? undefined : JSON.parse(s); +}; +const json = (value) => { + const s = JSON.stringify(value); + if (s === undefined) throw new TypeError("redisStore: undefined is not a value a store keeps"); + return s; +}; +/** A key or channel as given, or a TypeError: a string without NUL and without a lone surrogate. */ +const checked = (key, what = "key") => { + if (typeof key !== "string") throw new TypeError(`redisStore: a ${what} is a string, not ${typeof key}`); + if (key.includes("\0") || !key.isWellFormed()) throw new TypeError(`redisStore: a ${what} cannot hold NUL or a lone surrogate`); + return key; +}; + +/** The prefix: "" for none, otherwise ending in ":" (added when missing), so a key never runs into another prefix. */ +const prefixOf = (prefix) => { + checked(prefix, "prefix"); + return prefix === "" || prefix.endsWith(":") ? prefix : `${prefix}:`; +}; + +/** Ten years in ms: the longest TTL taken. */ +const MAX_TTL = 315_576_000_000; +/** PX takes whole milliseconds: rounding up never lets a key go early. */ +const px = (ttl) => { + if (typeof ttl !== "number" || !Number.isFinite(ttl)) throw new TypeError(`redisStore: a TTL is a finite number of milliseconds, not ${String(ttl)}`); + if (ttl <= 0 || ttl > MAX_TTL) throw new RangeError(`redisStore: a TTL is more than 0 and at most ten years (${MAX_TTL} ms), not ${ttl}`); + return String(Math.ceil(ttl)); +}; +const unknownCommand = (err) => /unknown command|ERR unknown/i.test(String(err?.message ?? err)); + +/** `send(name, ...args)` for either client: ioredis has `call`, node-redis `sendCommand([...])`. */ +function commands(client) { + if (client && typeof client.call === "function") return (...args) => client.call(...args); + if (client && typeof client.sendCommand === "function") return (...args) => client.sendCommand(args); + throw new TypeError("redisStore: pass a client from `redis` (node-redis v4 or later) or `ioredis`"); +} + +/** + * A `Store` in Redis on `client` (node-redis v4+ or ioredis, connected), shared by every + * process that uses the same server and prefix. + */ +export function redisStore(client, options = {}) { + const send = commands(client); + const prefix = prefixOf(options.prefix ?? "inkan:"); + const k = (key, what) => prefix + checked(key, what); + + let getdel = true; // until the server says it does not know GETDEL + + // publish/subscribe: one subscriber connection, one Redis subscription per channel + /** @type {Map void>, ready: Promise }>} */ + const channels = new Map(); + let sub; // { conn, owned, ioredis } once made + let making; + const deliver = (channel, message) => { + const fns = channels.get(channel)?.fns; + if (!fns || !fns.size) return; + let m; + try { + m = JSON.parse(text(message)); + } catch { + return; // not one of ours + } + for (const fn of [...fns]) fn(m); + }; + const subscriber = () => + (making ??= (async () => { + const owned = !options.subscriber; + const conn = options.subscriber ?? client.duplicate(); + const ioredis = typeof conn.call === "function"; + if (ioredis) { + if (conn.status === "wait") await conn.connect(); + conn.on("message", (channel, message) => deliver(channel, message)); + } else if (!conn.isOpen) { + await conn.connect(); + } + // node-redis takes the listener per SUBSCRIBE; one per store, kept so UNSUBSCRIBE removes only it + const listener = (message, channel) => deliver(channel, message); + return (sub = { conn, owned, ioredis, listener }); + })().catch((err) => { + making = undefined; + throw err; + })); + const unsubscribe = (channel) => { + if (!sub) return Promise.resolve(); + return sub.ioredis ? sub.conn.unsubscribe(channel) : sub.conn.unsubscribe(channel, sub.listener); + }; + + return sanitized({ + shared: true, + + async get(key) { + return parse(await send("GET", k(key))); + }, + + async set(key, value, ttl) { + const v = json(value); + if (ttl === undefined) await send("SET", k(key), v); + else await send("SET", k(key), v, "PX", px(ttl)); + }, + + async delete(key) { + await send("DEL", k(key)); + }, + + async add(key, value, ttl) { + const v = json(value); + const reply = ttl === undefined ? await send("SET", k(key), v, "NX") : await send("SET", k(key), v, "NX", "PX", px(ttl)); + return reply != null; + }, + + async take(key) { + if (getdel) { + try { + return parse(await send("GETDEL", k(key))); + } catch (err) { + if (!unknownCommand(err)) throw err; + getdel = false; + } + } + return parse(await send("EVAL", TAKE, "1", k(key))); + }, + + async increment(key, by = 1, ttl) { + if (typeof by !== "number" || !Number.isFinite(by)) throw new TypeError(`redisStore: increment takes a finite number, not ${by}`); + const t = ttl === undefined ? "" : px(ttl); + let reply; + try { + reply = await send("EVAL", INCREMENT, "1", k(key), String(by), t); + } catch (err) { + if (/inkan: not a number/.test(String(err?.message))) throw new TypeError("redisStore: increment on a key whose value is not a number"); + throw err; + } + return Number(text(reply)); + }, + + async publish(channel, message) { + await send("PUBLISH", k(channel, "channel"), json(message)); + }, + + async subscribe(channel, fn) { + const name = k(channel, "channel"); + let entry = channels.get(name); + if (!entry) { + // SUBSCRIBE once per channel; a second subscribe meanwhile waits for the same one + const ready = (async () => { + const { conn, ioredis, listener } = await subscriber(); + if (ioredis) await conn.subscribe(name); + else await conn.subscribe(name, listener); + })(); + channels.set(name, (entry = { fns: new Set(), ready })); + const mine = entry; + ready.catch(() => { + if (channels.get(name) === mine) channels.delete(name); + }); + } + await entry.ready; + const { fns } = entry; + // a wrapper of its own: the same function subscribed twice is called twice, removed once + const one = (m) => fn(m); + fns.add(one); + let done = false; + return () => { + if (done) return; + done = true; + fns.delete(one); + if (!fns.size && channels.get(name) === entry) { + channels.delete(name); + unsubscribe(name).catch(() => {}); + } + }; + }, + + /** + * Ends what the store opened itself: the subscriber connection it duplicated. The client + * passed in (and a `subscriber` passed in) stays open; the app closes those. + */ + async close() { + const names = [...channels.keys()]; + channels.clear(); + const s = sub ?? (making ? await making.catch(() => undefined) : undefined); + sub = undefined; + making = undefined; + if (!s) return; + if (!s.owned) { + for (const name of names) await (s.ioredis ? s.conn.unsubscribe(name) : s.conn.unsubscribe(name, s.listener)).catch(() => {}); + return; + } + if (s.ioredis) await s.conn.quit().catch(() => s.conn.disconnect()); + else if (typeof s.conn.close === "function") await s.conn.close().catch(() => s.conn.destroy?.()); + else await s.conn.quit().catch(() => s.conn.disconnect?.()); + }, + }); +} + +/** + * The store with every method's errors cleaned: ioredis puts the command with its key and + * value on its errors (`command: { name, args }`), and an app logs errors. The error thrown + * instead keeps the server's message (Redis names no key or value in it) and nothing else. + * The store's own errors pass as they are. + */ +function sanitized(store) { + for (const name of ["get", "set", "delete", "add", "take", "increment", "publish", "subscribe"]) { + const method = store[name]; + store[name] = async (...args) => { + try { + return await method(...args); + } catch (err) { + if (err instanceof Error && err.message.startsWith("redisStore:")) throw err; + throw new Error(`redisStore: ${String(err?.message ?? err)}`); + } + }; + } + return store; +} + +export default redisStore; diff --git a/packages/redis/package.json b/packages/redis/package.json new file mode 100644 index 0000000..74cdefd --- /dev/null +++ b/packages/redis/package.json @@ -0,0 +1,52 @@ +{ + "name": "@inkanjs/redis", + "version": "0.1.0", + "description": "A Redis store for inkan: jobs, sessions, auth, idempotency keys and rate limits shared by every process, on the client you already have.", + "type": "module", + "license": "MIT", + "author": "Vensin (https://github.com/vxnsin)", + "repository": { + "type": "git", + "url": "git+https://github.com/inkanjs/middleware.git", + "directory": "packages/redis" + }, + "keywords": [ + "inkan", + "store", + "redis", + "ioredis", + "session", + "rate-limit", + "jobs" + ], + "exports": { + ".": { + "types": "./index.d.ts", + "default": "./index.js" + } + }, + "files": [ + "index.js", + "index.d.ts", + "README.md", + "CHANGELOG.md" + ], + "engines": { + "node": ">=22.18" + }, + "peerDependencies": { + "@vxnsin/inkan": ">=0.8.0 <1.0.0" + }, + "devDependencies": { + "@vxnsin/inkan": "^0.8.0", + "ioredis": "^6.0.0", + "redis": "^6.3.0", + "redis-memory-server": "^0.17.1" + }, + "scripts": { + "test": "node --import ../../scripts/no-deprecations.js --test test.ts" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/redis/test.ts b/packages/redis/test.ts new file mode 100644 index 0000000..0dd1ee6 --- /dev/null +++ b/packages/redis/test.ts @@ -0,0 +1,225 @@ +import { after, describe, it, test } from "node:test"; +import assert from "node:assert/strict"; +import { randomUUID } from "node:crypto"; +import { spawn } from "node:child_process"; +import { inspect } from "node:util"; +import { createClient } from "redis"; +import { Redis } from "ioredis"; +import { storeConformance } from "@vxnsin/inkan/testing"; +import { redisStore, type RedisClient, type RedisStore } from "./index.js"; + +// The server: REDIS_URL when set (CI runs a redis:7 service), else one redis-memory-server +// downloads once (a build of Redis; Memurai on Windows). Offline and without either: skipped. +async function server(): Promise<{ url: string; stop(): Promise } | string> { + if (process.env.REDIS_URL) return { url: process.env.REDIS_URL, stop: async () => {} }; + try { + const { RedisMemoryServer } = await import("redis-memory-server"); + const s = await RedisMemoryServer.create(); + return { url: `redis://${await s.getHost()}:${await s.getPort()}`, stop: () => s.stop() }; + } catch (err) { + return `no Redis to test against: set REDIS_URL, or let redis-memory-server download one (${(err instanceof Error ? err.message : String(err))})`; + } +} + +const srv = await server(); +if (typeof srv === "string") { + console.log(`# skipped: ${srv}`); + test("@inkanjs/redis against a real server", { skip: srv }, () => {}); +} else { + const node = createClient({ url: srv.url }); + node.on("error", () => {}); + await node.connect(); + const io = new Redis(srv.url); + after(async () => { + await node.close(); + await io.quit(); + await srv.stop(); + }); + + const fresh = () => `test:${randomUUID()}:`; + const close = (s: unknown) => (s as RedisStore).close(); + + storeConformance(() => redisStore(node, { prefix: fresh() }), { name: "redisStore on node-redis", close }); + storeConformance(() => redisStore(io, { prefix: fresh() }), { name: "redisStore on ioredis", close }); + + // Redis before 6.2 has no GETDEL: the store goes on with a script + const old = new Proxy(io, { + get(target, p) { + if (p === "call") return (cmd: string, ...args: unknown[]) => (/^getdel$/i.test(cmd) ? Promise.reject(new Error("ERR unknown command 'GETDEL'")) : (target.call as any)(cmd, ...args)); + const v = Reflect.get(target, p, target); + return typeof v === "function" ? v.bind(target) : v; + }, + }) as unknown as RedisClient; + storeConformance(() => redisStore(old, { prefix: fresh() }), { name: "redisStore without GETDEL (Redis < 6.2)", close }); + + describe("redisStore", () => { + it("rejects a client it does not know", () => { + assert.throws(() => redisStore({} as never), /node-redis|ioredis/); + }); + + it("keeps keys under the prefix, JSON as the value, PX as the TTL", async () => { + const prefix = fresh(); + const store = redisStore(io, { prefix }); + await store.set("a", { n: 1 }, 10_000); + assert.equal(await io.get(`${prefix}a`), '{"n":1}'); + const pttl = await io.pttl(`${prefix}a`); + assert.ok(pttl > 9000 && pttl <= 10_000, `pttl ${pttl}`); + await store.set("a", 2); + assert.equal(await io.pttl(`${prefix}a`), -1, "set without a TTL clears it"); + assert.equal(await redisStore(io).get(`${prefix}a`), undefined, "the default prefix is inkan:"); + await store.close(); + }); + + it("a prefix ends in ':' (one is added); keys with NUL or a lone surrogate are refused", async () => { + const base = `p${randomUUID()}`; + const a = redisStore(io, { prefix: base }); + const b = redisStore(io, { prefix: `${base}1` }); + await a.set("1x", "a"); + await b.set("x", "b"); + assert.equal(await a.get("1x"), "a", "prefix p + key 1x is not prefix p1 + key x"); + assert.equal(await io.get(`${base}:1x`), '"a"'); + assert.throws(() => redisStore(io, { prefix: "a\0" }), TypeError); + for (const bad of ["a\0b", "\uD800", "x\uDC00y"]) { + await assert.rejects(a.set(bad, 1), TypeError, JSON.stringify(bad)); + await assert.rejects(a.get(bad), TypeError); + await assert.rejects(a.increment(bad), TypeError); + await assert.rejects(a.publish(bad, 1), TypeError); + await assert.rejects(a.subscribe(bad, () => {}), TypeError); + } + await assert.rejects(a.get(1 as never), TypeError); + await io.del(`${base}:1x`, `${base}1:x`); + }); + + it("increment gives a new key the TTL and keeps it after; numbers that are not whole count too", async () => { + const prefix = fresh(); + const store = redisStore(node, { prefix }); + assert.equal(await store.increment("n", 1, 50_000), 1); + await io.pexpire(`${prefix}n`, 20_000); + assert.equal(await store.increment("n", 1, 50_000), 2); + assert.ok((await io.pttl(`${prefix}n`)) <= 20_000, "the TTL is not set again"); + assert.equal(await store.increment("f", 1.5), 1.5); + assert.equal(await store.increment("f", 1), 2.5); + assert.equal(await store.get("f"), 2.5); + await store.close(); + }); + + it("two stores on one server, one per client library, see each other's keys and messages", async () => { + const prefix = fresh(); + const a = redisStore(node, { prefix }); + const b = redisStore(io, { prefix }); + try { + await a.set("shared", { from: "a" }); + assert.deepEqual(await b.get("shared"), { from: "a" }); + const won = await Promise.all(Array.from({ length: 20 }, (_, i) => (i % 2 ? a : b).add("lock", i, 60_000))); + assert.equal(won.filter(Boolean).length, 1); + const seen: unknown[] = []; + const off = await b.subscribe("news", (m) => seen.push(m)); + await a.publish("news", { hello: "b" }); + await until(() => seen.length > 0); + assert.deepEqual(seen, [{ hello: "b" }]); + off(); + } finally { + await a.close(); + await b.close(); + } + }); + + it("another process counts, writes and publishes on the same keys", async () => { + const prefix = fresh(); + const store = redisStore(node, { prefix }); + const seen: unknown[] = []; + const off = await store.subscribe("from-child", (m) => seen.push(m)); + const child = spawn( + process.execPath, + [ + "--input-type=module", + "-e", + `import { Redis } from "ioredis"; + import { redisStore } from "./index.js"; + const client = new Redis(process.env.URL); + const store = redisStore(client, { prefix: process.env.PREFIX }); + await Promise.all(Array.from({ length: 50 }, () => store.increment("count", 1, 60000))); + await store.set("written", "by the child"); + await store.publish("from-child", { done: true }); + await store.close(); + await client.quit();`, + ], + { cwd: import.meta.dirname, env: { ...process.env, URL: srv.url, PREFIX: prefix }, stdio: "inherit" }, + ); + const exited = new Promise((resolve) => child.on("exit", resolve)); + await Promise.all(Array.from({ length: 50 }, () => store.increment("count", 1, 60_000))); + assert.equal(await exited, 0); + assert.equal(await store.get("count"), 100); + assert.equal(await store.get("written"), "by the child"); + await until(() => seen.length > 0); + assert.deepEqual(seen, [{ done: true }]); + off(); + await store.close(); + }); + + it("subscribes on a subscriber of the app's own, and leaves it open on close", async () => { + const sub = node.duplicate(); + await sub.connect(); + const store = redisStore(node, { prefix: fresh(), subscriber: sub }); + const seen: unknown[] = []; + await store.subscribe("c", (m) => seen.push(m)); + await store.publish("c", 1); + await until(() => seen.length > 0); + assert.deepEqual(seen, [1]); + await store.close(); + assert.equal(sub.isOpen, true); + await sub.close(); + }); + + it("increment on a value that is not a number throws a TypeError and leaves it", async () => { + for (const client of [io, node]) { + const store = redisStore(client, { prefix: fresh() }); + for (const value of ["not a number", "5", true, null, { n: 1 }, [1]]) { + await store.set("s", value, 60_000); + await assert.rejects(store.increment("s", 2), TypeError, `${JSON.stringify(value)} is not a number`); + assert.deepEqual(await store.get("s"), value, "and it stays as it was"); + } + await store.set("s", 5.5); + assert.equal(await store.increment("s", 2), 7.5, "a number counts on"); + } + }); + + it("a server error carries no key: no command, no arguments", async () => { + for (const client of [io, node]) { + const prefix = `secret-key-1:${randomUUID()}:`; + const store = redisStore(client, { prefix }); + await io.lpush(`${prefix}list`, "x"); // a key of another type: WRONGTYPE + for (const call of [() => store.get("list"), () => store.take("list"), () => store.increment("list")]) { + const err = await call().then( + () => assert.fail("WRONGTYPE"), + (e: unknown) => e as Error & Record, + ); + assert.match(err.message, /WRONGTYPE/); + assert.equal(err.command, undefined); + const all = inspect(err, { depth: 10, showHidden: true }); + assert.ok(!all.includes("secret-key-1"), all); + } + await io.del(`${prefix}list`); + } + }); + + it("refuses undefined, and a TTL that is not a finite number of ms up to ten years", async () => { + const store = redisStore(io, { prefix: fresh() }); + await assert.rejects(store.set("x", undefined), TypeError); + await assert.rejects(store.set("x", 1, "10" as never), TypeError); + await assert.rejects(store.increment("x", Infinity), TypeError); + for (const ttl of [NaN, Infinity]) await assert.rejects(store.set("x", 1, ttl), TypeError, String(ttl)); + for (const ttl of [0, -5, 315_576_000_001]) { + await assert.rejects(store.set("x", 1, ttl), RangeError, String(ttl)); + await assert.rejects(store.add("x", 1, ttl), RangeError, String(ttl)); + await assert.rejects(store.increment("x", 1, ttl), RangeError, String(ttl)); + } + assert.equal(await store.get("x"), undefined, "nothing was written"); + }); + }); +} + +async function until(ok: () => boolean, ms = 5000) { + const end = Date.now() + ms; + while (!ok() && Date.now() < end) await new Promise((r) => setTimeout(r, 10)); +} diff --git a/packages/secure-headers/CHANGELOG.md b/packages/secure-headers/CHANGELOG.md index c2724e0..c66be87 100644 --- a/packages/secure-headers/CHANGELOG.md +++ b/packages/secure-headers/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## 0.2.0 + +- Works with inkan 0.8 (`inkan: ">=0.8.0 <1.0.0"`). +- HSTS goes by `ctx.secure` by default instead of the socket's `encrypted` flag, so it is sent + behind a proxy the app trusts (`trustProxy` and `x-forwarded-proto: https`) as well, and no + longer reaches into `ctx.req`. + ## 0.1.0 - `secureHeaders()`: content-security-policy, strict-transport-security over HTTPS, diff --git a/packages/secure-headers/README.md b/packages/secure-headers/README.md index d5b00e3..5302fb6 100644 --- a/packages/secure-headers/README.md +++ b/packages/secure-headers/README.md @@ -65,7 +65,7 @@ secureHeaders({ | --- | --- | --- | | `csp` | `{}` | `false` for none; `directives` over the defaults (`false` drops one, `true` for one without sources), `useDefaults`, `reportOnly` | | `hsts` | `{}` | `false` for none; `maxAge`, `includeSubDomains` (true), `preload` (false), `force` (false) | -| `https` | the socket is a TLS socket | `(ctx) => boolean`: whether the request came over HTTPS | +| `https` | `ctx.secure` | `(ctx) => boolean`: whether the request came over HTTPS | | `skip` | none | `(ctx) => boolean`: answers to leave alone | | `extra` | none | more headers, by name | | each header above | | a string, or `false` | @@ -76,9 +76,9 @@ secureHeaders({ read. An answer whose handler (or template) read it gets `'nonce-…'` in `script-src` and `style-src`; every other answer gets the plain policy. With only `default-src` in the policy, `script-src` and `style-src` are written out from it to take the nonce. -- **HSTS** goes out only over HTTPS: the socket is a TLS socket, or `https(ctx)` says so. - Behind a proxy that ends TLS, pass `https: (ctx) => ctx.headers["x-forwarded-proto"] === "https"` - (only when the proxy sets that header itself) or `hsts: { force: true }`. +- **HSTS** goes out only over HTTPS: `ctx.secure` is true, or `https(ctx)` says so. + Behind a proxy that ends TLS, give the app `trustProxy` (then `ctx.secure` follows the + proxy's `x-forwarded-proto`), or pass `hsts: { force: true }`. - **Your headers win.** A header the handler or a hook before this one set stays as it is, so one route can send its own policy. - **inkan's own pages keep working.** `/docs`, `/openapi.json` and the inspector are answered diff --git a/packages/secure-headers/index.d.ts b/packages/secure-headers/index.d.ts index 2296774..98c6a29 100644 --- a/packages/secure-headers/index.d.ts +++ b/packages/secure-headers/index.d.ts @@ -29,7 +29,7 @@ export type SecureHeadersOptions = { csp?: CspOptions | boolean; /** strict-transport-security, sent over HTTPS only unless `force`. */ hsts?: HstsOptions | false; - /** Whether a request came over HTTPS. Default: the socket is a TLS socket. */ + /** Whether a request came over HTTPS. Default `ctx.secure`: TLS, or `x-forwarded-proto` from a proxy the app's `trustProxy` trusts. */ https?: (ctx: Context) => boolean; /** x-content-type-options. Default `nosniff`. */ noSniff?: string | false; diff --git a/packages/secure-headers/index.js b/packages/secure-headers/index.js index 195cef4..2d0dfc7 100644 --- a/packages/secure-headers/index.js +++ b/packages/secure-headers/index.js @@ -79,7 +79,7 @@ export function secureHeaders(options = {}) { if (h.preload) hsts += "; preload"; } const forceHsts = Boolean(options.hsts && options.hsts.force); - const isHttps = options.https ?? ((ctx) => Boolean(ctx.req?.socket?.encrypted)); + const isHttps = options.https ?? ((ctx) => ctx.secure === true); // the rest is the same on every answer: worked out once const fixed = Object.entries({ @@ -111,7 +111,7 @@ export function secureHeaders(options = {}) { }); return app.decorateRequest("cspNonce", () => randomBytes(16).toString("base64")); }, - { name: "secure-headers", shared: true, inkan: ">=0.7.0 <0.8.0" }, + { name: "secure-headers", shared: true, inkan: ">=0.8.0 <1.0.0" }, ); } diff --git a/packages/secure-headers/package.json b/packages/secure-headers/package.json index 2b43c39..b5186e0 100644 --- a/packages/secure-headers/package.json +++ b/packages/secure-headers/package.json @@ -1,6 +1,6 @@ { "name": "@inkanjs/secure-headers", - "version": "0.1.0", + "version": "0.2.0", "description": "Security headers for inkan: a strict Content-Security-Policy with per-request nonces, HSTS and the rest, each one yours to change.", "type": "module", "license": "MIT", @@ -35,13 +35,13 @@ "node": ">=22.18" }, "peerDependencies": { - "@vxnsin/inkan": ">=0.7.0 <0.8.0" + "@vxnsin/inkan": ">=0.8.0 <1.0.0" }, "devDependencies": { - "@vxnsin/inkan": "^0.7.0" + "@vxnsin/inkan": "^0.8.0" }, "scripts": { - "test": "node --test test.ts" + "test": "node --import ../../scripts/no-deprecations.js --test test.ts" }, "publishConfig": { "access": "public" diff --git a/packages/secure-headers/test.ts b/packages/secure-headers/test.ts index 67a14c5..9168ede 100644 --- a/packages/secure-headers/test.ts +++ b/packages/secure-headers/test.ts @@ -109,6 +109,12 @@ test("hsts: over HTTPS, or always with force, never twice", async () => { assert.equal((await behindProxy.inject({ url: "/x", headers: { "x-forwarded-proto": "https" } })).headers["strict-transport-security"], "max-age=600"); assert.equal((await behindProxy.inject({ url: "/x" })).headers["strict-transport-security"], undefined); + const trusted = inkan({ ...quiet, trustProxy: true }).register(secureHeaders()).get("/x", () => ({})); + assert.equal((await trusted.inject({ url: "/x", headers: { "x-forwarded-proto": "https" } })).headers["strict-transport-security"], "max-age=31536000; includeSubDomains", "ctx.secure by default"); + assert.equal((await trusted.inject({ url: "/x" })).headers["strict-transport-security"], undefined); + const untrusted = inkan(quiet).register(secureHeaders()).get("/x", () => ({})); + assert.equal((await untrusted.inject({ url: "/x", headers: { "x-forwarded-proto": "https" } })).headers["strict-transport-security"], undefined, "no trustProxy: the header is nobody's word"); + const off = inkan(quiet).register(secureHeaders({ hsts: false, https: () => true })).get("/x", () => ({})); assert.equal((await off.inject({ url: "/x" })).headers["strict-transport-security"], undefined); }); diff --git a/packages/session/CHANGELOG.md b/packages/session/CHANGELOG.md index db79c85..e665daa 100644 --- a/packages/session/CHANGELOG.md +++ b/packages/session/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 0.2.0 + +- **Any inkan `Store`** (inkan 0.8) for `store`: `memoryStore()` from @vxnsin/inkan, or an + adapter such as @inkanjs/redis. The plugin's own `memoryStore` is gone; `memoryStore` and + `SessionStore` are re-exported from the core, deprecated. A store that is not one is + refused when the plugin is made. +- **The TTL a store gets is in milliseconds** (it was seconds). Records are `{ d, c, f? }` + as before; flash messages work as before. +- Needs inkan `>=0.8.0 <1.0.0`. + ## 0.1.0 - `session()`: `ctx.session`, typed, signed with HMAC-SHA256 in the cookie, encrypted with diff --git a/packages/session/README.md b/packages/session/README.md index 0ca7762..6aa46e1 100644 --- a/packages/session/README.md +++ b/packages/session/README.md @@ -44,29 +44,31 @@ in it. A cookie session lasts until it expires even after `destroy()` if someone copy; when you need logout to end a session for certain, use a store. ```ts -import { memoryStore, session } from "@inkanjs/session"; +import { memoryStore } from "@vxnsin/inkan"; +import { session } from "@inkanjs/session"; app.register(session({ secret, store: memoryStore({ max: 50_000 }) })); ``` -`memoryStore` lives in the process: right for development and one process. For more, -write the three methods against Redis or your database: +The store is an inkan `Store`. `memoryStore` from @vxnsin/inkan lives in the process: right +for development and one process. For more, any `Store` works, such as @inkanjs/redis, @inkanjs/mongo, @inkanjs/mysql and @inkanjs/postgres (coming with inkan 0.8): ```ts -const store: SessionStore = { - get: async (id) => JSON.parse((await redis.get(`sess:${id}`)) ?? "null") ?? undefined, - set: async (id, record, ttl) => void (await redis.set(`sess:${id}`, JSON.stringify(record), "EX", ttl)), // ttl in seconds - delete: async (id) => void (await redis.del(`sess:${id}`)), -}; +import { redisStore } from "@inkanjs/redis"; + +app.register(session({ secret, store: redisStore(redis) })); ``` +The plugin uses its `get`, `set` and `delete`, keeps a `SessionRecord` (`{ d, c, f? }`, plain +JSON) per session, and gives it a TTL of `cookie.maxAge` in milliseconds. + ## Options | option | default | | | --- | --- | --- | | `secret` | required | at least 32 bytes (a string as UTF-8, or bytes); a list rotates them (below) | | `encrypt` | `false` | AES-256-GCM instead of a signature; ignored with a store | -| `store` | none | `{ get(id), set(id, record, ttl), delete(id) }`, each may be async; `ttl` in **seconds**, as in @inkanjs/auth; `record` is plain JSON | +| `store` | none | any inkan `Store`; TTLs in ms, as every `Store` takes them; records are plain JSON | | `rolling` | `false` | write the cookie on every answer, so a session lasts `maxAge` after the last request | | `maxLifetime` | none | seconds a session lasts at most from its start (or `regenerate()`), however active; with `rolling` you want one | | `cookie.name` | `"sid"` | | diff --git a/packages/session/index.d.ts b/packages/session/index.d.ts index d79e0eb..66e029b 100644 --- a/packages/session/index.d.ts +++ b/packages/session/index.d.ts @@ -1,16 +1,10 @@ -import type { Plugin } from "@vxnsin/inkan"; +import type { memoryStore as coreMemoryStore, Plugin, Store } from "@vxnsin/inkan"; /** What a store keeps per session: the data, when the session began (ms), and flash messages not read yet. Plain JSON. */ export type SessionRecord = { d: Record; c: number; f?: Record }; -/** Where the data lives when the cookie only holds an id. Every method may return a promise. */ -export interface SessionStore { - /** The record kept under `id`, or undefined when there is none (any more). */ - get(id: string): unknown | Promise; - /** Keeps `record` under `id` for `ttl` **seconds** (as @inkanjs/auth's store does). */ - set(id: string, record: SessionRecord, ttl: number): void | Promise; - delete(id: string): void | Promise; -} +/** @deprecated Use `Store` from "@vxnsin/inkan"; TTLs are in milliseconds since 0.2. */ +export type SessionStore = Store; export type SessionCookie = { /** Default `"sid"`. */ @@ -37,8 +31,12 @@ export type SessionOptions = { cookie?: SessionCookie; /** AES-256-GCM instead of a signature: the client cannot read the data either. Default false. Ignored with a store. */ encrypt?: boolean; - /** Keep the data here and only a random id in the cookie. See `memoryStore()`. */ - store?: SessionStore; + /** + * Keep the data in this inkan `Store` (`memoryStore()` from @vxnsin/inkan, Redis, a + * database, …) and only a random id in the cookie. A `SessionRecord` per session, for + * `maxAge` (in ms, as every Store takes it). + */ + store?: Store; /** Write the cookie (and the store's entry) on every answer, so the session lasts `maxAge` after the last request. Default false. */ rolling?: boolean; /** @@ -90,5 +88,5 @@ export declare function session = Record }>; export default session; -/** A store in this process's memory, at most `max` sessions (default 10000), the least recently written dropped first. */ -export declare function memoryStore(options?: { max?: number }): SessionStore & { readonly size: number }; +/** @deprecated Use `memoryStore` from "@vxnsin/inkan": this is the same function, re-exported. */ +export declare const memoryStore: typeof coreMemoryStore; diff --git a/packages/session/index.js b/packages/session/index.js index 1d651a9..49eba5b 100644 --- a/packages/session/index.js +++ b/packages/session/index.js @@ -2,7 +2,7 @@ // // signed the data in the cookie, readable by the client, HMAC-SHA256 against changes // encrypted the data in the cookie, AES-256-GCM: neither readable nor changeable -// store the data on the server, the cookie holds only a random id (signed too) +// store the data in an inkan Store, the cookie holds only a random id (signed too) // // app.register(session({ secret: process.env.SESSION_SECRET })) // .post("/login", (ctx) => { ctx.session.regenerate(); ctx.session.set("user", 7); }) @@ -91,32 +91,8 @@ function codec(keys, name, encrypt) { }; } -/** A store in this process's memory: for development and single-process apps. */ -export function memoryStore({ max = 10_000 } = {}) { - const map = new Map(); // id → { data, until }; oldest first - return { - get(id) { - const e = map.get(id); - if (!e) return undefined; - if (e.until <= Date.now()) { - map.delete(id); - return undefined; - } - return e.data; - }, - set(id, data, ttl) { - map.delete(id); // to the end: the most recently used goes last - map.set(id, { data, until: Date.now() + ttl * 1000 }); // ttl in seconds - while (map.size > max) map.delete(map.keys().next().value); - }, - delete(id) { - map.delete(id); - }, - get size() { - return map.size; - }, - }; -} +/** @deprecated the core's `memoryStore` from @vxnsin/inkan, re-exported for 0.1 code. */ +export { memoryStore } from "@vxnsin/inkan"; /** One request's session. */ export class Session { @@ -273,6 +249,9 @@ const dataOf = (d) => Object.assign(Object.create(null), d && typeof d === "obje export function session(options) { const { secret, encrypt = false, store, rolling = false, maxLifetime } = options ?? {}; const keys = keysOf(secret); + if (store !== undefined && (store === null || typeof store !== "object" || ["get", "set", "delete"].some((m) => typeof store[m] !== "function"))) { + throw new TypeError("session: store must be an inkan Store, such as memoryStore() from @vxnsin/inkan (TTLs in ms)"); + } const c = options.cookie ?? {}; const name = c.name ?? "sid"; const maxAge = c.maxAge ?? 86_400; @@ -339,7 +318,7 @@ export function session(options) { if (store) { if (origin && origin !== s.id) await store.delete(origin); // regenerated: the old id is worth nothing now const f = s._flash(); - await store.set(s.id, f ? { d: s.toJSON(), c: s._created(), f } : { d: s.toJSON(), c: s._created() }, life); // ttl in seconds + await store.set(s.id, f ? { d: s.toJSON(), c: s._created(), f } : { d: s.toJSON(), c: s._created() }, life * 1000); // a Store's TTL is in ms value = cookie.write(s.id); } else { value = cookie.write(JSON.stringify({ i: s.id, e: Date.now() + life * 1000, c: s._created(), d: s.toJSON(), f: s._flash() })); @@ -357,7 +336,7 @@ export function session(options) { return fromCookie(ctx.cookies[name]) ?? new Session(Object.create(null), newId(), true); }); }, - { name: "session", shared: true, inkan: ">=0.7.0 <0.8.0" }, + { name: "session", shared: true, inkan: ">=0.8.0 <1.0.0" }, ); } diff --git a/packages/session/package.json b/packages/session/package.json index 037f3e8..de22c9b 100644 --- a/packages/session/package.json +++ b/packages/session/package.json @@ -1,6 +1,6 @@ { "name": "@inkanjs/session", - "version": "0.1.0", + "version": "0.2.0", "description": "Cookie sessions for inkan without dependencies: signed or encrypted cookies, or a store behind a random id.", "type": "module", "license": "MIT", @@ -34,13 +34,13 @@ "node": ">=22.18" }, "peerDependencies": { - "@vxnsin/inkan": ">=0.7.0 <0.8.0" + "@vxnsin/inkan": ">=0.8.0 <1.0.0" }, "devDependencies": { - "@vxnsin/inkan": "^0.7.0" + "@vxnsin/inkan": "^0.8.0" }, "scripts": { - "test": "node --test test.ts" + "test": "node --import ../../scripts/no-deprecations.js --test test.ts" }, "publishConfig": { "access": "public" diff --git a/packages/session/test.ts b/packages/session/test.ts index 1317767..97c66e3 100644 --- a/packages/session/test.ts +++ b/packages/session/test.ts @@ -1,7 +1,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { inkan } from "@vxnsin/inkan"; -import { memoryStore, session, type SessionStore } from "./index.js"; +import { inkan, memoryStore, type Store } from "@vxnsin/inkan"; +import { memoryStore as deprecatedMemoryStore, session } from "./index.js"; const quiet = { log: false as const }; const SECRET = "a-secret-for-tests-only-0123456789abcdef"; @@ -132,50 +132,85 @@ test("regenerate gives a new id for the same data", async () => { assert.equal(me.body.user, 7); }); -function spyStore() { - const inner = memoryStore(); +const METHODS = ["get", "set", "delete", "add", "take", "increment"]; + +/** + * Any Store, wrapped: only the six methods of the interface are there, every call and every + * property read is noted, and the ids it holds are counted. + */ +function spyStore(inner: Store = memoryStore()) { const calls: string[] = []; - const store: SessionStore & { size: number } = { + const ttls: (number | undefined)[] = []; + const touched = new Set(); + const live = new Set(); + const plain: Store = { async get(id) { calls.push("get"); return inner.get(id); }, - async set(id, data, ttl) { + async set(id, value, ttl) { calls.push("set"); - inner.set(id, data, ttl); + ttls.push(ttl); + live.add(id); + await inner.set(id, value, ttl); }, async delete(id) { calls.push("delete " + id.slice(0, 4)); - inner.delete(id); + live.delete(id); + await inner.delete(id); + }, + async add(id, value, ttl) { + calls.push("add"); + return inner.add(id, value, ttl); + }, + async take(id) { + calls.push("take"); + return inner.take(id); + }, + async increment(id, by, ttl) { + calls.push("increment"); + return inner.increment(id, by, ttl); + }, + }; + const store = new Proxy(plain, { + get(target, prop, receiver) { + touched.add(prop); + return Reflect.get(target, prop, receiver); }, + }); + return { + store, + calls, + ttls, + touched, get size() { - return inner.size; + return live.size; }, }; - return { store, calls }; } test("store: the cookie holds only a signed id, regenerate and destroy forget the old one", async () => { - const { store, calls } = spyStore(); + const spy = spyStore(); + const { store, calls } = spy; const app = shop({ secret: SECRET, store }); const cart = await send(app, "POST", "/cart"); const sid = cookieOf(cart)!; assert.doesNotMatch(Buffer.from(sid.split(".")[0]!, "base64url").toString(), /sencha/); - assert.equal(store.size, 1); + assert.equal(spy.size, 1); assert.equal((await send(app, "GET", "/me", sid)).body.isNew, false); const oldId = (await send(app, "GET", "/me", sid)).body.id as string; const login = await send(app, "POST", "/login", sid); assert.notEqual(login.body.id, oldId); assert.ok(calls.includes("delete " + oldId.slice(0, 4)), "the old id is deleted"); - assert.equal(store.size, 1); + assert.equal(spy.size, 1); assert.equal((await send(app, "GET", "/me", sid)).body.isNew, true, "the old cookie is worth nothing"); const fresh = cookieOf(login)!; assert.equal((await send(app, "GET", "/me", fresh)).body.user, 7); const out = await send(app, "POST", "/logout", fresh); assert.match(out.cookies[0]!, /^sid=; .*Max-Age=0/); - assert.equal(store.size, 0); + assert.equal(spy.size, 0); assert.equal((await send(app, "GET", "/me", fresh)).body.isNew, true); calls.length = 0; @@ -189,12 +224,15 @@ test("store: an entry lasts maxAge, and memoryStore keeps at most max", async (t const store = memoryStore({ max: 2 }); const app = shop({ secret: SECRET, store, cookie: { maxAge: 10 } }); const a = cookieOf(await send(app, "POST", "/login"))!; - t.mock.timers.tick(11_000); + const id = Buffer.from(a.split(".")[0]!, "base64url").toString(); + t.mock.timers.tick(9_000); + assert.deepEqual(((await store.get(id)) as { d: object }).d, { user: 7 }, "kept for maxAge, in ms"); + t.mock.timers.tick(2_000); + assert.equal(await store.get(id), undefined, "and gone from the store after it"); assert.equal((await send(app, "GET", "/me", a)).body.isNew, true, "expired in the store"); const ids = []; for (let i = 0; i < 3; i++) ids.push(cookieOf(await send(app, "POST", "/login"))!); - assert.equal(store.size, 2); assert.equal((await send(app, "GET", "/me", ids[0])).body.isNew, true, "the oldest went first"); assert.equal((await send(app, "GET", "/me", ids[2])).body.user, 7); }); @@ -239,13 +277,31 @@ test("secrets are counted in bytes, and bytes are taken as they are", async () = assert.equal((await send(app, "GET", "/me", sid)).body.user, 7); }); -test("store: the ttl a store gets is in seconds, as in @inkanjs/auth", async () => { - const ttls: number[] = []; - const inner = memoryStore(); - const store: SessionStore = { get: (id) => inner.get(id), set: (id, data, ttl) => (ttls.push(ttl), inner.set(id, data, ttl)), delete: (id) => inner.delete(id) }; +test("store: the ttl a store gets is in ms, as every inkan Store takes it", async () => { + const { store, ttls } = spyStore(); const app = shop({ secret: SECRET, store, cookie: { maxAge: 3600 } }); await send(app, "POST", "/login"); - assert.deepEqual(ttls, [3600]); + assert.deepEqual(ttls, [3_600_000]); +}); + +test("store: any Store will do, and only get, set and delete of it are used", async () => { + const { store, calls, ttls, touched } = spyStore(); + const app = shop({ secret: SECRET, store, rolling: true }); + const sid = cookieOf(await send(app, "POST", "/cart"))!; + const login = cookieOf(await send(app, "POST", "/login", sid))!; + assert.equal((await send(app, "GET", "/me", login)).body.user, 7); + await send(app, "POST", "/logout", login); + const flash = flashing({ secret: SECRET, store }); + const f = cookieOf(await send(flash as never, "POST", "/save"))!; + assert.deepEqual((await send(flash as never, "GET", "/all", f)).body.messages, { success: ["Saved.", "Mailed."], warning: ["Low stock."] }); + assert.deepEqual([...new Set(calls.map((c) => c.split(" ")[0]))].sort(), ["delete", "get", "set"]); + assert.deepEqual([...touched].filter((p) => !METHODS.includes(String(p))), [], "nothing but the Store interface"); + assert.ok(ttls.every((ttl) => ttl === 86_400_000), ttls.join()); +}); + +test("store: one that is not a Store is refused when the plugin is made; memoryStore is the core's", () => { + assert.throws(() => session({ secret: SECRET, store: { get() {} } as never }), /store must be an inkan Store/); + assert.equal(deprecatedMemoryStore, memoryStore); }); test("maxLifetime: rolling keeps a session alive, but never past its absolute lifetime", async (t) => { @@ -326,8 +382,7 @@ test("flash: a broken record in the store gives only its strings", async () => { const app = flashing({ secret: SECRET, store }); const sid = cookieOf(await send(app as never, "POST", "/save"))!; const id = Buffer.from(sid.split(".")[0]!, "base64url").toString(); - const rec = store.get(id) as { d: object; c: number; f: unknown }; - rec.f = { success: ["ok", 1, { x: 1 }], other: "nope", warning: [] }; - store.set(id, rec as never, 60); + const rec = (await store.get(id)) as { d: object; c: number; f: unknown }; + await store.set(id, { ...rec, f: { success: ["ok", 1, { x: 1 }], other: "nope", warning: [] } }, 60_000); assert.deepEqual((await send(app as never, "GET", "/all", sid)).body.messages, { success: ["ok"] }); }); diff --git a/scripts/no-deprecations.js b/scripts/no-deprecations.js new file mode 100644 index 0000000..e814656 --- /dev/null +++ b/scripts/no-deprecations.js @@ -0,0 +1,18 @@ +// Loaded into every test process (`node --import`): a test file that makes inkan print a +// deprecation warning fails, so the packages stay on the current names. inkan says it once +// per process through console.warn, as "inkan: is deprecated, use . …". + +const warn = console.warn; +const seen = []; + +console.warn = (...args) => { + const text = args.map(String).join(" "); + if (/^inkan: .* is deprecated, use /.test(text)) seen.push(text); + warn.apply(console, args); +}; + +process.on("exit", () => { + if (seen.length === 0) return; + process.stderr.write(`\n${seen.length} deprecation warning(s) from inkan:\n${seen.map((s) => ` ${s}`).join("\n")}\n`); + process.exitCode = 1; +});