From 1e9714704d9afaa3838e6cb658d6372a3f75bebc Mon Sep 17 00:00:00 2001 From: Sun-Young Kim Date: Sun, 23 Aug 2026 21:01:53 +0000 Subject: [PATCH] fix(server): harden liftOver/TransVar subprocesses and auto-restart the API container TransVar and liftOver were spawned with no timeout, so a hung binary (bad input, disk stall reading the reference files) left the request hanging forever - the same failure mode just fixed for external HTTP calls in f0f33a3. Both now get a timeout with SIGKILL as the fallback. liftover.js also leaked its temporary BED files on every error path, only cleaning up on full success; over time this fills /tmp and starves the host. Cleanup now runs in a finally block regardless of outcome. Finally, the server container had no restart policy, so any crash left the API down until someone SSHed in to run docker compose up again. restart: unless-stopped lets Docker recover it automatically. Co-Authored-By: Claude Sonnet 5 --- docker-compose.yml | 1 + server/utils/liftover.js | 145 +++++++++++++++++++++------------------ server/utils/transvar.js | 8 ++- 3 files changed, 87 insertions(+), 67 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6fc0b71..316fea4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,7 @@ services: server: image: zhandongliulab/marrvel-server + restart: unless-stopped working_dir: /MARRVEL/server volumes: - "${LOCAL_BASE}/.env:/MARRVEL/server/.env" diff --git a/server/utils/liftover.js b/server/utils/liftover.js index 16c9a1d..c2747b1 100644 --- a/server/utils/liftover.js +++ b/server/utils/liftover.js @@ -1,85 +1,97 @@ const fs = require('fs'); const config = require('../config'); +const cleanupTempFile = (filePath) => { + try { + fs.unlinkSync(filePath); + } catch (error) { + if (error.code !== 'ENOENT') { + console.error(`Error cleaning up temp file ${filePath}: ${error.message}`); + } + } +}; + exports.liftover = async (chr, pos, fromOrg, fromDb, toOrg, toDb, minMatch, isMultiRegionAllowed, minQuery, minChain, minBlocks, isThickFudgeSet) => { - // generate input BED file with random name - const inputBed = `/tmp/${Math.random().toString(36).substring(2, 15)}.bed`; - fs.writeFileSync(inputBed, `chr${chr}\t${pos}\t${pos}\n`); - // generate output BED file with random name - const outputBed = `/tmp/${Math.random().toString(36).substring(2, 15)}.bed`; - // generate unlifted BED file with random name - const unliftedBed = `/tmp/${Math.random().toString(36).substring(2, 15)}.bed`; - if (!config.liftoverCmdTool[fromOrg][fromDb][toOrg][toDb]) { throw new Error('liftOver chain file is not configured properly'); } - const cmdArgs = [ - inputBed, - config.liftoverCmdTool[fromOrg][fromDb][toOrg][toDb], - outputBed, - unliftedBed - ]; - if (minBlocks) { - cmdArgs.push(`-minBlocks=${minBlocks}`); - } - if (isThickFudgeSet) { - cmdArgs.push('-fudgeThick'); - } - if (minMatch) { - cmdArgs.push(`-minMatch=${minMatch}`); - } - if (isMultiRegionAllowed) { - cmdArgs.push('-multiple'); - } - if (minQuery) { - cmdArgs.push(`-minSizeQ=${minQuery}`); - } - if (minChain) { - cmdArgs.push(`-minChainT=${minChain}`); - } - // run liftOver command line tool - try { - await runLiftover(cmdArgs); - } catch (error) { - console.error(`Error occurred while running liftOver: ${error.message}`); - return { - message: 'Error occurred while running liftOver' - }; - } - // read output BED file - let lifted = null; + // generate input/output/unlifted BED files with random names + const inputBed = `/tmp/${Math.random().toString(36).substring(2, 15)}.bed`; + const outputBed = `/tmp/${Math.random().toString(36).substring(2, 15)}.bed`; + const unliftedBed = `/tmp/${Math.random().toString(36).substring(2, 15)}.bed`; + try { - const output = fs.readFileSync(outputBed, 'utf8'); - lifted = (output.split('\n')[0] || '').split('\t'); - } catch (error) { - console.error(`Error reading output BED file: ${error.message}`); - return { - message: 'Error reading output BED file' - }; - } - if (!lifted || lifted.length < 2) { + fs.writeFileSync(inputBed, `chr${chr}\t${pos}\t${pos}\n`); + + const cmdArgs = [ + inputBed, + config.liftoverCmdTool[fromOrg][fromDb][toOrg][toDb], + outputBed, + unliftedBed + ]; + if (minBlocks) { + cmdArgs.push(`-minBlocks=${minBlocks}`); + } + if (isThickFudgeSet) { + cmdArgs.push('-fudgeThick'); + } + if (minMatch) { + cmdArgs.push(`-minMatch=${minMatch}`); + } + if (isMultiRegionAllowed) { + cmdArgs.push('-multiple'); + } + if (minQuery) { + cmdArgs.push(`-minSizeQ=${minQuery}`); + } + if (minChain) { + cmdArgs.push(`-minChainT=${minChain}`); + } + + // run liftOver command line tool + try { + await runLiftover(cmdArgs); + } catch (error) { + console.error(`Error occurred while running liftOver: ${error.message}`); + return { + message: 'Error occurred while running liftOver' + }; + } + // read output BED file + let lifted = null; + try { + const output = fs.readFileSync(outputBed, 'utf8'); + lifted = (output.split('\n')[0] || '').split('\t'); + } catch (error) { + console.error(`Error reading output BED file: ${error.message}`); + return { + message: 'Error reading output BED file' + }; + } + if (!lifted || lifted.length < 2) { + return { + message: 'No lifted position found' + }; + } return { - message: 'No lifted position found' + inputChr: chr, + inputPos: pos, + chr: lifted[0].replace('chr', ''), + pos: parseInt(lifted[1]) }; + } finally { + cleanupTempFile(inputBed); + cleanupTempFile(outputBed); + cleanupTempFile(unliftedBed); } - // clean up temporary files - fs.unlinkSync(inputBed); - fs.unlinkSync(outputBed); - fs.unlinkSync(unliftedBed); - return { - inputChr: chr, - inputPos: pos, - chr: lifted[0].replace('chr', ''), - pos: parseInt(lifted[1]) - }; }; const runLiftover = (args) => { return new Promise((resolve, reject) => { const spawn = require('child_process').spawn; - const child = spawn(config.liftoverCmdTool.path, args); + const child = spawn(config.liftoverCmdTool.path, args, { timeout: 15000, killSignal: 'SIGKILL' }); let stdout = ''; let stderr = ''; child.stdout.on('data', (data) => { @@ -88,7 +100,10 @@ const runLiftover = (args) => { child.stderr.on('data', (data) => { stderr += data.toString(); }); - child.on('close', (code) => { + child.on('close', (code, signal) => { + if (signal) { + return reject(new Error(`liftOver process was killed (signal: ${signal})`)); + } if (code !== 0) { return reject(new Error(`liftOver process exited with code ${code}: ${stderr}`)); } diff --git a/server/utils/transvar.js b/server/utils/transvar.js index 6801af1..b3f12f3 100644 --- a/server/utils/transvar.js +++ b/server/utils/transvar.js @@ -24,7 +24,7 @@ const appendGene = (data) => { const executeTransvar = (option) => { return new Promise((resolve, reject) => { const { spawn } = require('child_process'); - const proc = spawn(transvarPath, option); + const proc = spawn(transvarPath, option, { timeout: 30000, killSignal: 'SIGKILL' }); let stdout = ''; const stderr = []; @@ -37,7 +37,11 @@ const executeTransvar = (option) => { stderr.push(err); }); - proc.on('close', (code) => { + proc.on('close', (code, signal) => { + if (signal) { + reject(new Error(`transvar process was killed (signal: ${signal})`)); + return; + } resolve({ code, stdout,