From 7a2a4c4381f4a85161ace981f9a14b403effeadd Mon Sep 17 00:00:00 2001 From: Anurag Date: Thu, 18 Jun 2026 20:28:17 +0530 Subject: [PATCH 1/3] feat: ./.ipfs storage migration introduced --- package-lock.json | 3 ++ package.json | 3 +- packages/node/src/index.ts | 2 + packages/node/src/node.ts | 28 +++++------ packages/node/src/pins.ts | 50 +++++++++++++++++++ packages/node/src/repo-config.ts | 82 ++++++++++++++++++++++++++++++++ packages/node/src/repo.ts | 9 ++++ packages/node/src/types.ts | 12 +++++ 8 files changed, 171 insertions(+), 18 deletions(-) create mode 100644 packages/node/src/pins.ts create mode 100644 packages/node/src/repo-config.ts create mode 100644 packages/node/src/repo.ts diff --git a/package-lock.json b/package-lock.json index 67c15a2..ed70636 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4146,6 +4146,9 @@ "dependencies": { "@ipfs-meshkit/core": "*", "@ipfs-meshkit/node": "*" + }, + "devDependencies": { + "@types/node": "^25.9.3" } }, "packages/node": { diff --git a/package.json b/package.json index dc1e0e5..9c8f028 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,8 @@ ], "scripts": { "build": "tsc --build", - "clean": "npm run clean --workspaces --if-present" + "clean": "npm run clean --workspaces --if-present", + "test:persistence": "npm run test:persistence --workspace=meshkit-dummy-app" }, "repository": { "type": "git", diff --git a/packages/node/src/index.ts b/packages/node/src/index.ts index 48953aa..ea70141 100644 --- a/packages/node/src/index.ts +++ b/packages/node/src/index.ts @@ -1,3 +1,5 @@ export type { IPFSNodeHandle, StartIPFSNodeOptions } from './types.js'; export { MeshkitNodeError } from './types.js'; +export { DEFAULT_REPO, resolveRepoPath } from './repo.js'; +export { listPins } from './pins.js'; export { startIPFSNode, stopIPFSNode } from './node.js'; diff --git a/packages/node/src/node.ts b/packages/node/src/node.ts index 9238e85..5dfc729 100644 --- a/packages/node/src/node.ts +++ b/packages/node/src/node.ts @@ -1,9 +1,12 @@ import { spawn, type ChildProcess } from 'node:child_process'; import { MeshkitNodeError, type IPFSNodeHandle, type StartIPFSNodeOptions } from './types.js'; import { isKuboHealthy, waitForKubo } from './health.js'; +import { resolveRepoPath } from './repo.js'; +import { ensureRepoConfigured } from './repo-config.js'; const DEFAULT_HOST = '127.0.0.1'; const DEFAULT_PORT = 5001; +const DEFAULT_GATEWAY_PORT = 8080; const DEFAULT_READY_TIMEOUT_MS = 30_000; function buildApiUrl(host: string, port: number): string { @@ -40,19 +43,9 @@ async function shutdownKubo(apiUrl: string): Promise { } } -function spawnKuboDaemon( - binary: string, - options: { repo?: string; init: boolean }, -): ChildProcess { - const args = ['daemon']; - if (options.init) { - args.push('--init'); - } - - const env = options.repo ? { ...process.env, IPFS_PATH: options.repo } : process.env; - - return spawn(binary, args, { - env, +function spawnKuboDaemon(binary: string, options: { repo: string }): ChildProcess { + return spawn(binary, ['daemon'], { + env: { ...process.env, IPFS_PATH: options.repo }, stdio: ['ignore', 'pipe', 'pipe'], }); } @@ -91,7 +84,9 @@ export async function startIPFSNode( const init = options.init ?? true; const readyTimeoutMs = options.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS; const ipfsBinary = options.ipfsBinary ?? 'ipfs'; + const gatewayPort = options.gatewayPort ?? DEFAULT_GATEWAY_PORT; const apiUrl = buildApiUrl(host, port); + const repo = resolveRepoPath(options.repo); if (await isKuboHealthy(apiUrl)) { return { @@ -104,11 +99,9 @@ export async function startIPFSNode( } await assertIpfsBinary(ipfsBinary); + await ensureRepoConfigured(ipfsBinary, repo, host, port, gatewayPort, init); - const child = spawnKuboDaemon(ipfsBinary, { - ...(options.repo !== undefined ? { repo: options.repo } : {}), - init, - }); + const child = spawnKuboDaemon(ipfsBinary, { repo }); let startupError: Error | undefined; @@ -137,6 +130,7 @@ export async function startIPFSNode( return { url: apiUrl, + repo, managed: true, async stop() { try { diff --git a/packages/node/src/pins.ts b/packages/node/src/pins.ts new file mode 100644 index 0000000..393424a --- /dev/null +++ b/packages/node/src/pins.ts @@ -0,0 +1,50 @@ +import { MeshkitNodeError } from './types.js'; + +interface PinLsLine { + Cid?: string; + Keys?: Record; + Pins?: string[]; +} + +/** + * List all pinned CIDs on a Kubo node. Useful for migration scripts and backups. + */ +export async function listPins(apiUrl: string): Promise { + const url = new URL('/api/v0/pin/ls', apiUrl); + url.searchParams.set('type', 'all'); + url.searchParams.set('stream', 'true'); + + const response = await fetch(url, { method: 'POST' }); + if (!response.ok) { + throw new MeshkitNodeError( + `Failed to list pins at ${apiUrl} (HTTP ${response.status}).`, + ); + } + + const body = await response.text(); + const cids = new Set(); + + for (const line of body.split('\n')) { + const trimmed = line.trim(); + if (!trimmed) { + continue; + } + + const parsed = JSON.parse(trimmed) as PinLsLine; + if (parsed.Cid) { + cids.add(parsed.Cid); + } + if (parsed.Keys) { + for (const cid of Object.keys(parsed.Keys)) { + cids.add(cid); + } + } + if (parsed.Pins) { + for (const cid of parsed.Pins) { + cids.add(cid); + } + } + } + + return [...cids]; +} diff --git a/packages/node/src/repo-config.ts b/packages/node/src/repo-config.ts new file mode 100644 index 0000000..f918d55 --- /dev/null +++ b/packages/node/src/repo-config.ts @@ -0,0 +1,82 @@ +import { spawn } from 'node:child_process'; +import { access } from 'node:fs/promises'; +import { join } from 'node:path'; +import { MeshkitNodeError } from './types.js'; + +function runIpfsCommand( + binary: string, + repo: string, + args: string[], +): Promise { + return new Promise((resolve, reject) => { + const child = spawn(binary, args, { + env: { ...process.env, IPFS_PATH: repo }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + + let stderr = ''; + child.stderr?.on('data', (chunk: Buffer) => { + stderr += chunk.toString(); + }); + + child.once('error', (error) => { + reject(new MeshkitNodeError(`ipfs ${args.join(' ')} failed.`, { cause: error })); + }); + + child.once('exit', (code) => { + if (code === 0) { + resolve(); + return; + } + reject( + new MeshkitNodeError( + `ipfs ${args.join(' ')} exited with code ${code ?? 'unknown'}: ${stderr.trim()}`, + ), + ); + }); + }); +} + +async function repoExists(repo: string): Promise { + try { + await access(join(repo, 'config')); + return true; + } catch { + return false; + } +} + +function formatApiAddress(host: string, port: number): string { + if (host.includes(':')) { + return `/ip6/${host}/tcp/${port}`; + } + return `/ip4/${host}/tcp/${port}`; +} + +/** + * Initialize the repo if needed and align Kubo's API listen address with Meshkit options. + */ +export async function ensureRepoConfigured( + binary: string, + repo: string, + host: string, + port: number, + gatewayPort: number, + init: boolean, +): Promise { + const exists = await repoExists(repo); + + if (!exists) { + if (!init) { + throw new MeshkitNodeError( + `Kubo repo does not exist at ${repo}. Set init: true or create the repo first.`, + ); + } + await runIpfsCommand(binary, repo, ['init']); + } + + const apiAddress = formatApiAddress(host, port); + const gatewayAddress = formatApiAddress(host, gatewayPort); + await runIpfsCommand(binary, repo, ['config', 'Addresses.API', apiAddress]); + await runIpfsCommand(binary, repo, ['config', 'Addresses.Gateway', gatewayAddress]); +} diff --git a/packages/node/src/repo.ts b/packages/node/src/repo.ts new file mode 100644 index 0000000..5419c91 --- /dev/null +++ b/packages/node/src/repo.ts @@ -0,0 +1,9 @@ +import { resolve } from 'node:path'; + +/** Default Kubo repo directory relative to the process cwd. */ +export const DEFAULT_REPO = '.ipfs'; + +/** Resolve a repo path to an absolute filesystem path. */ +export function resolveRepoPath(repo: string = DEFAULT_REPO): string { + return resolve(repo); +} diff --git a/packages/node/src/types.ts b/packages/node/src/types.ts index a8e2bf7..fba9963 100644 --- a/packages/node/src/types.ts +++ b/packages/node/src/types.ts @@ -16,6 +16,12 @@ export interface StartIPFSNodeOptions { */ port?: number; + /** + * Kubo HTTP gateway port. Defaults to `8080`. + * Set a unique port when running multiple local Kubo instances. + */ + gatewayPort?: number; + /** * Filesystem path for the Kubo repo (`IPFS_PATH`). * When omitted, Kubo uses its default repo location. @@ -44,6 +50,12 @@ export interface IPFSNodeHandle { /** Kubo RPC API base URL, e.g. `http://127.0.0.1:5001`. */ readonly url: string; + /** + * Absolute path to the Kubo repo (`IPFS_PATH`) when Meshkit spawned the daemon. + * Omitted when attaching to an existing daemon (`managed: false`). + */ + readonly repo?: string; + /** * `true` when Meshkit spawned this daemon; `false` when an existing daemon was reused. */ From 52fb1a6e650307b0b970a10959662cd951e94e5a Mon Sep 17 00:00:00 2001 From: Anurag Date: Thu, 18 Jun 2026 20:30:13 +0530 Subject: [PATCH 2/3] feat: ./.ipfs node reference implemented --- packages/meshkit/package.json | 3 ++ packages/meshkit/src/index.ts | 12 ++++++- packages/meshkit/src/init.ts | 10 +++--- packages/meshkit/src/shutdown.ts | 57 ++++++++++++++++++++++++++++++++ packages/meshkit/tsconfig.json | 3 +- 5 files changed, 79 insertions(+), 6 deletions(-) create mode 100644 packages/meshkit/src/shutdown.ts diff --git a/packages/meshkit/package.json b/packages/meshkit/package.json index f8da62b..91c1508 100644 --- a/packages/meshkit/package.json +++ b/packages/meshkit/package.json @@ -23,5 +23,8 @@ "dependencies": { "@ipfs-meshkit/core": "*", "@ipfs-meshkit/node": "*" + }, + "devDependencies": { + "@types/node": "^25.9.3" } } diff --git a/packages/meshkit/src/index.ts b/packages/meshkit/src/index.ts index b509285..39d48be 100644 --- a/packages/meshkit/src/index.ts +++ b/packages/meshkit/src/index.ts @@ -9,7 +9,14 @@ export type { IPFSNodeHandle, StartIPFSNodeOptions, } from '@ipfs-meshkit/node'; -export { MeshkitNodeError, startIPFSNode, stopIPFSNode } from '@ipfs-meshkit/node'; +export { + DEFAULT_REPO, + MeshkitNodeError, + listPins, + resolveRepoPath, + startIPFSNode, + stopIPFSNode, +} from '@ipfs-meshkit/node'; export type { LocalNodeOption, @@ -17,3 +24,6 @@ export type { MeshkitBootstrapResult, } from './init.js'; export { init } from './init.js'; + +export type { GracefulShutdownOptions } from './shutdown.js'; +export { setupGracefulShutdown } from './shutdown.js'; diff --git a/packages/meshkit/src/init.ts b/packages/meshkit/src/init.ts index 8fe0914..e782d2e 100644 --- a/packages/meshkit/src/init.ts +++ b/packages/meshkit/src/init.ts @@ -1,5 +1,5 @@ import { Meshkit as CoreMeshkit } from '@ipfs-meshkit/core'; -import { startIPFSNode } from '@ipfs-meshkit/node'; +import { DEFAULT_REPO, startIPFSNode } from '@ipfs-meshkit/node'; import type { StartIPFSNodeOptions, IPFSNodeHandle } from '@ipfs-meshkit/node'; import type { Meshkit, MeshkitInitOptions } from '@ipfs-meshkit/core'; import { MeshkitError } from '@ipfs-meshkit/core'; @@ -14,7 +14,7 @@ export interface MeshkitBootstrapOptions extends Omit void | Promise; + + /** Exit the process after shutdown. Defaults to `true`. */ + exit?: boolean; + + /** Exit code. Defaults to `0`. */ + exitCode?: number; +} + +/** + * Register SIGINT (Ctrl+C) and SIGTERM handlers that stop a managed Kubo daemon + * gracefully so the repo on disk (e.g. `./.ipfs`) is left in a consistent state. + */ +export function setupGracefulShutdown( + localNode: IPFSNodeHandle | undefined, + options: GracefulShutdownOptions = {}, +): void { + let shuttingDown = false; + const signals: NodeJS.Signals[] = ['SIGINT', 'SIGTERM']; + + const handle = (signal: NodeJS.Signals) => { + if (shuttingDown) { + return; + } + shuttingDown = true; + + void (async () => { + try { + if (options.onShutdown) { + await options.onShutdown(); + } + if (localNode?.managed) { + await stopIPFSNode(localNode); + } + } catch (error) { + console.error(`Error during ${signal} shutdown:`, error); + if (options.exit !== false) { + process.exit(1); + } + return; + } + + if (options.exit !== false) { + process.exit(options.exitCode ?? 0); + } + })(); + }; + + for (const signal of signals) { + process.on(signal, () => handle(signal)); + } +} diff --git a/packages/meshkit/tsconfig.json b/packages/meshkit/tsconfig.json index 79ce283..898e55b 100644 --- a/packages/meshkit/tsconfig.json +++ b/packages/meshkit/tsconfig.json @@ -3,7 +3,8 @@ "compilerOptions": { "rootDir": "./src", "outDir": "./dist", - "composite": true + "composite": true, + "types": ["node"] }, "include": ["src/**/*"], "references": [ From 7382ecb733597f491992b96686b12f767b4b110b Mon Sep 17 00:00:00 2001 From: Anurag Date: Thu, 18 Jun 2026 20:31:13 +0530 Subject: [PATCH 3/3] test: examples/ updated for testing --- .gitignore | 4 +- README.md | 43 +++++---- examples/dummy-app/.gitignore | 2 + examples/dummy-app/package.json | 3 +- examples/dummy-app/server.js | 19 ++-- examples/dummy-app/test-persistence.ts | 119 +++++++++++++++++++++++++ 6 files changed, 165 insertions(+), 25 deletions(-) create mode 100644 examples/dummy-app/.gitignore create mode 100644 examples/dummy-app/test-persistence.ts diff --git a/.gitignore b/.gitignore index 2bba0dc..eb8d0c3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ node_modules dist *.tsbuildinfo -*.tgz \ No newline at end of file +*.tgz +.ipfs +.ipfs-test diff --git a/README.md b/README.md index 1e96f41..6932163 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ Install [Kubo](https://docs.ipfs.tech/install/) (`ipfs` on your PATH). Meshkit c ```bash npm install npm run build +npm run test:persistence # repo survive shutdown + restart ``` ## Usage @@ -40,40 +41,52 @@ npm install @ipfs-meshkit/meshkit ### Node.js — automatic local Kubo +`localNode: true` stores pinned data in `./.ipfs` (relative to where you start the process). Add `.ipfs` to `.gitignore`. + ```typescript import { readFile, writeFile } from 'node:fs/promises'; -import { init, stopIPFSNode } from '@ipfs-meshkit/meshkit'; +import { init, listPins, setupGracefulShutdown } from '@ipfs-meshkit/meshkit'; const { meshkit, localNode } = await init({ localNode: true }); +setupGracefulShutdown(localNode); // Ctrl+C flushes Kubo; ./.ipfs stays on disk + const pdf = await readFile('./invoice.pdf'); const cid = await meshkit.upload(pdf); await meshkit.pin(cid); +console.log('repo:', localNode?.repo); +console.log('pins:', await listPins(meshkit.activeNodes[0]!)); + const retrieved = await meshkit.retrieve(cid); await writeFile('./invoice-copy.pdf', retrieved); - -if (localNode?.managed) { - await stopIPFSNode(localNode); -} ``` -### Node.js — server bootstrap +### Migrating servers (AWS → GCP) + +1. Stop the server gracefully (`setupGracefulShutdown` or `stopIPFSNode`) +2. Copy the `./.ipfs` directory (tar, EBS snapshot, S3, etc.) +3. Restore on the new host and start with the same repo path: ```typescript -import { startIPFSNode, stopIPFSNode, init } from '@ipfs-meshkit/meshkit'; +const { meshkit, localNode } = await init({ + localNode: { repo: './.ipfs', init: false }, +}); +``` + +Use `listPins()` to export CIDs as a backup manifest for re-pinning. -const kubo = await startIPFSNode(); -const { meshkit } = await init({ nodes: [kubo.url] }); +### Node.js — server bootstrap -// ... your HTTP server ... +```typescript +import { init, setupGracefulShutdown } from '@ipfs-meshkit/meshkit'; -process.on('SIGTERM', async () => { - if (kubo.managed) { - await stopIPFSNode(kubo); - } - process.exit(0); +const { meshkit, localNode } = await init({ localNode: true }); +setupGracefulShutdown(localNode, { + onShutdown: async () => { /* close HTTP server, DB, etc. */ }, }); + +// ... app.listen(3000) ... ``` `startIPFSNode` reuses an existing daemon on `127.0.0.1:5001` when one is already healthy. diff --git a/examples/dummy-app/.gitignore b/examples/dummy-app/.gitignore new file mode 100644 index 0000000..83a8950 --- /dev/null +++ b/examples/dummy-app/.gitignore @@ -0,0 +1,2 @@ +.ipfs +.ipfs-test diff --git a/examples/dummy-app/package.json b/examples/dummy-app/package.json index 764fe3e..088c338 100644 --- a/examples/dummy-app/package.json +++ b/examples/dummy-app/package.json @@ -4,7 +4,8 @@ "private": true, "type": "module", "scripts": { - "start": "node server.js" + "start": "node server.js", + "test:persistence": "node --experimental-strip-types test-persistence.ts" }, "dependencies": { "@ipfs-meshkit/meshkit": "*" diff --git a/examples/dummy-app/server.js b/examples/dummy-app/server.js index 37e075c..688def2 100644 --- a/examples/dummy-app/server.js +++ b/examples/dummy-app/server.js @@ -1,25 +1,28 @@ -import { init, stopIPFSNode } from '@ipfs-meshkit/meshkit'; +import { init, listPins, setupGracefulShutdown } from '@ipfs-meshkit/meshkit'; async function bootstrap() { const { meshkit, localNode } = await init({ localNode: true }); + setupGracefulShutdown(localNode); + console.log('Meshkit ready'); + console.log(' repo:', localNode?.repo ?? '(attached to external daemon)'); console.log(' active nodes:', meshkit.activeNodes); console.log(' kubo managed:', localNode?.managed ?? false); + console.log(' Ctrl+C stops Kubo gracefully — ./.ipfs data stays on disk'); const text = `hello from dummy-app @ ${new Date().toISOString()}`; const cid = await meshkit.upload(new TextEncoder().encode(text)); - console.log(' uploaded cid:', cid); + await meshkit.pin(cid); + console.log(' uploaded & pinned cid:', cid); + + const pins = await listPins(meshkit.activeNodes[0]); + console.log(' pinned cids:', pins.length); const retrieved = new TextDecoder().decode(await meshkit.retrieve(cid)); console.log(' retrieved:', retrieved); - if (localNode?.managed) { - await stopIPFSNode(localNode); - console.log(' kubo stopped'); - } - - console.log('dummy-app ok'); + console.log('\nServer running — press Ctrl+C to shut down gracefully'); } bootstrap().catch((err) => { diff --git a/examples/dummy-app/test-persistence.ts b/examples/dummy-app/test-persistence.ts new file mode 100644 index 0000000..c0b11a8 --- /dev/null +++ b/examples/dummy-app/test-persistence.ts @@ -0,0 +1,119 @@ +import { rm, access } from 'node:fs/promises'; +import { join } from 'node:path'; +import { + init, + listPins, + stopIPFSNode, + resolveRepoPath, +} from '@ipfs-meshkit/meshkit'; + +const TEST_REPO = '.ipfs-test'; +const TEST_PORT = 15_001; +const TEST_GATEWAY_PORT = 18_001; +const TEST_HOST = '127.0.0.1'; +const TEST_URL = `http://${TEST_HOST}:${TEST_PORT}`; +const repoPath = resolveRepoPath(TEST_REPO); + +let passed = 0; +let failed = 0; + +function assert(condition: boolean, message: string): void { + if (condition) { + console.log(` ✓ ${message}`); + passed += 1; + return; + } + console.error(` ✗ ${message}`); + failed += 1; +} + +async function pathExists(path: string): Promise { + try { + await access(path); + return true; + } catch { + return false; + } +} + +async function phaseOne(): Promise<{ cid: string; payload: string }> { + console.log('\n[phase 1] start, upload, pin, list pins, graceful stop'); + + const { meshkit, localNode } = await init({ + localNode: { + repo: TEST_REPO, + host: TEST_HOST, + port: TEST_PORT, + gatewayPort: TEST_GATEWAY_PORT, + }, + }); + + assert(localNode?.managed === true, 'spawned managed Kubo'); + assert(localNode?.repo === repoPath, `repo path is ${repoPath}`); + assert(meshkit.activeNodes[0] === TEST_URL, 'connected to test node'); + + const payload = `persistence-test-${Date.now()}`; + const cid = await meshkit.upload(new TextEncoder().encode(payload)); + await meshkit.pin(cid); + + const pins = await listPins(localNode!.url); + assert(pins.includes(cid), `pin list contains ${cid}`); + + if (localNode?.managed) { + await stopIPFSNode(localNode); + } + + assert(await pathExists(join(repoPath, 'blocks')), 'repo blocks dir exists on disk after stop'); + + return { cid, payload }; +} + +async function phaseTwo(cid: string, originalPayload: string): Promise { + console.log('\n[phase 2] restart same repo, verify data survived shutdown'); + + const { meshkit, localNode } = await init({ + localNode: { + repo: TEST_REPO, + host: TEST_HOST, + port: TEST_PORT, + gatewayPort: TEST_GATEWAY_PORT, + init: false, + }, + }); + + assert(localNode?.managed === true, 'respawned Kubo on same repo'); + + const retrieved = new TextDecoder().decode(await meshkit.retrieve(cid)); + assert(retrieved === originalPayload, 'retrieved same bytes after restart'); + + const pins = await listPins(localNode!.url); + assert(pins.includes(cid), 'pin survived restart'); + + if (localNode?.managed) { + await stopIPFSNode(localNode); + } +} + +async function main(): Promise { + console.log('meshkit persistence integration test'); + + if (await pathExists(repoPath)) { + await rm(repoPath, { recursive: true, force: true }); + } + + const { cid, payload } = await phaseOne(); + await phaseTwo(cid, payload); + + await rm(repoPath, { recursive: true, force: true }); + + console.log(`\n${passed} passed, ${failed} failed`); + if (failed > 0) { + process.exit(1); + } + console.log('all persistence tests ok'); +} + +main().catch((error) => { + console.error(error); + process.exit(1); +});