[codex] Add price source retry backoff

README.md

@@ -133,6 +133,7 @@ The application reads configurations from the `.env` file at the root.
 | `PRICE_REFRESH_INTERVAL_SECONDS` | Refresh interval in seconds | 30 | No |
 | `PRICE_STALE_THRESHOLD_MINUTES` | Stale threshold in minutes | 5 | No |
 | `PRICE_ANOMALY_THRESHOLD_PCT` | Anomaly detection threshold % | 20 | No |
+| `PRICE_SOURCE_RETRY_COUNT` | Number of retries for transient price-source HTTP failures | 3 | No |
 | `ADMIN_API_KEY` | Bootstrap admin bearer token for API key management | empty | Yes, for protected endpoints |
 | `LOG_LEVEL` | Logging level: `debug`, `info`, `warn`, or `error` | info | No |
 

src/config.js

@@ -38,6 +38,7 @@ const env = cleanEnv(rawEnv, {
   PRICE_REFRESH_INTERVAL_SECONDS: num({ default: 30 }),
   PRICE_STALE_THRESHOLD_MINUTES: num({ default: 5 }),
   PRICE_ANOMALY_THRESHOLD_PCT: num({ default: 20 }),
+  PRICE_SOURCE_RETRY_COUNT: num({ default: 3 }),
   LOG_LEVEL: str({
     default: 'info',
     choices: ['debug', 'info', 'warn', 'error'],
@@ -74,6 +75,7 @@ module.exports = {
     refreshInterval: env.PRICE_REFRESH_INTERVAL_SECONDS,
     staleThresholdMinutes: env.PRICE_STALE_THRESHOLD_MINUTES,
     anomalyThresholdPercent: env.PRICE_ANOMALY_THRESHOLD_PCT,
+    sourceRetryCount: env.PRICE_SOURCE_RETRY_COUNT,
   },
   auth: {
     adminApiKey: env.ADMIN_API_KEY,

src/services/sources/coingecko.js

@@ -1,6 +1,7 @@
 const axios = require('axios');
 const config = require('../../config');
 const logger = require('../../logger');
+const { fetchWithRetry } = require('../../utils/fetchWithRetry');
 
 const STELLAR_COINGECKO_MAP = {
   XLM: 'stellar',
@@ -32,12 +33,14 @@ async function fetchPrice(assetCode) {
 
   try {
     const client = getClient();
-    const response = await client.get('/simple/price', {
+    const response = await fetchWithRetry('/simple/price', {
+      client,
+      label: 'coingecko',
       params: {
         ids: coinId,
         vs_currencies: 'usd',
       },
-    });
+    }, config.price?.sourceRetryCount);
 
     const price = response.data[coinId]?.usd;
     if (price === undefined || price === null) {

src/services/sources/coinmarketcap.js

@@ -1,6 +1,7 @@
 const axios = require('axios');
 const config = require('../../config');
 const logger = require('../../logger');
+const { fetchWithRetry } = require('../../utils/fetchWithRetry');
 
 let apiClient = null;
 
@@ -58,12 +59,14 @@ async function fetchPrice(assetCode, issuer = null) {
   try {
     const client = getClient();
     const lookupKey = market.id ? String(market.id) : market.symbol;
-    const response = await client.get('/cryptocurrency/quotes/latest', {
+    const response = await fetchWithRetry('/cryptocurrency/quotes/latest', {
+      client,
+      label: 'coinmarketcap',
       params: {
         ...(market.id ? { id: market.id } : { symbol: market.symbol }),
         convert: 'USD',
       },
-    });
+    }, config.price?.sourceRetryCount);
 
     const data = response.data?.data?.[lookupKey];
     if (!data || !data.quote?.USD?.price) {

src/services/sources/stellarDex.js

@@ -1,6 +1,7 @@
 const { Server } = require('stellar-sdk');
 const config = require('../../config');
 const logger = require('../../logger');
+const { fetchWithRetry } = require('../../utils/fetchWithRetry');
 
 let server = null;
 
@@ -13,6 +14,14 @@ function getServer() {
 
 const XLM_ASSET = { native: true };
 
+function fetchOrderBook(horizon, base, counter, label) {
+  return fetchWithRetry(
+    () => horizon.orderbook(base, counter).limit(1).call(),
+    { label },
+    config.price?.sourceRetryCount
+  );
+}
+
 async function fetchPrice(assetCode, issuer) {
   try {
     const horizon = getServer();
@@ -28,7 +37,12 @@ async function fetchPrice(assetCode, issuer) {
       counter = XLM_ASSET;
     }
 
-    const orderBook = await horizon.orderbook(base, counter === XLM_ASSET ? undefined : counter).limit(1).call();
+    const orderBook = await fetchOrderBook(
+      horizon,
+      base,
+      counter === XLM_ASSET ? undefined : counter,
+      'stellar_dex orderbook'
+    );
 
     if (!orderBook.bids || orderBook.bids.length === 0) {
       return null;
@@ -53,10 +67,12 @@ async function fetchPrice(assetCode, issuer) {
 async function getXlmUsdPrice(horizon) {
   try {
     const usdcIssuer = config.stellar.usdcIssuer;
-    const orderBook = await horizon
-      .orderbook(XLM_ASSET, { code: 'USDC', issuer: usdcIssuer })
-      .limit(1)
-      .call();
+    const orderBook = await fetchOrderBook(
+      horizon,
+      XLM_ASSET,
+      { code: 'USDC', issuer: usdcIssuer },
+      'stellar_dex xlm_usdc'
+    );
 
     if (!orderBook.bids || orderBook.bids.length === 0) {
       return null;

src/utils/fetchWithRetry.js

@@ -0,0 +1,94 @@
+'use strict';
+
+const axios = require('axios');
+const logger = require('../logger');
+
+const NON_RETRYABLE_STATUSES = new Set([400, 401, 403]);
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function parseRetryAfter(headerValue, now = Date.now()) {
+  if (!headerValue) return null;
+
+  const seconds = Number(headerValue);
+  if (Number.isFinite(seconds) && seconds >= 0) {
+    return seconds * 1000;
+  }
+
+  const retryAt = Date.parse(headerValue);
+  if (Number.isNaN(retryAt)) {
+    return null;
+  }
+
+  return Math.max(0, retryAt - now);
+}
+
+function isRetryableStatus(status) {
+  if (!status) return true;
+  if (NON_RETRYABLE_STATUSES.has(status)) return false;
+  return status === 429 || status >= 500;
+}
+
+function getRetryDelay(err, attempt, baseDelayMs) {
+  const status = err.response?.status;
+  const retryAfter = status === 429
+    ? parseRetryAfter(err.response?.headers?.['retry-after'])
+    : null;
+
+  if (retryAfter !== null) {
+    return retryAfter;
+  }
+
+  return baseDelayMs * Math.pow(4, attempt);
+}
+
+async function fetchWithRetry(target, options = {}, retries = 3, baseDelayMs = 500) {
+  const {
+    client = axios,
+    logger: retryLogger = logger,
+    sleep: sleepFn = sleep,
+    label,
+    ...requestOptions
+  } = options;
+
+  for (let attempt = 0; attempt <= retries; attempt += 1) {
+    try {
+      if (typeof target === 'function') {
+        return await target();
+      }
+
+      return await client.get(target, {
+        timeout: 10000,
+        ...requestOptions,
+      });
+    } catch (err) {
+      const status = err.response?.status;
+      const retryable = isRetryableStatus(status);
+
+      if (!retryable || attempt === retries) {
+        throw err;
+      }
+
+      const delayMs = getRetryDelay(err, attempt, baseDelayMs);
+      retryLogger.debug('Retrying price source request', {
+        label: label || (typeof target === 'string' ? target : 'custom-request'),
+        attempt: attempt + 2,
+        maxAttempts: retries + 1,
+        delayMs,
+        status: status || null,
+      });
+
+      await sleepFn(delayMs);
+    }
+  }
+
+  return null;
+}
+
+module.exports = {
+  fetchWithRetry,
+  isRetryableStatus,
+  parseRetryAfter,
+};

test/coinmarketcap.test.js

@@ -28,6 +28,9 @@ jest.mock('../src/config', () => ({
       [`USDC:${mockUsdcIssuer}`]: { id: 3408 },
     },
   },
+  price: {
+    sourceRetryCount: 0,
+  },
 }));
 
 jest.mock('../src/logger', () => mockLogger);
@@ -73,6 +76,7 @@ describe('CoinMarketCap source', () => {
       timeout: 10000,
     });
     expect(mockGet).toHaveBeenCalledWith('/cryptocurrency/quotes/latest', {
+      timeout: 10000,
       params: {
         symbol: 'XLM',
         convert: 'USD',
@@ -101,6 +105,7 @@ describe('CoinMarketCap source', () => {
 
     expect(price).toBe(1.0003);
     expect(mockGet).toHaveBeenCalledWith('/cryptocurrency/quotes/latest', {
+      timeout: 10000,
       params: {
         id: 3408,
         convert: 'USD',

test/config.test.js

@@ -69,6 +69,7 @@ describe('configuration validation', () => {
         refreshInterval: 30,
         staleThresholdMinutes: 5,
         anomalyThresholdPercent: 20,
+        sourceRetryCount: 3,
       },
     });
   });

test/fetchWithRetry.test.js

@@ -0,0 +1,96 @@
+'use strict';
+
+const {
+  fetchWithRetry,
+  isRetryableStatus,
+  parseRetryAfter,
+} = require('../src/utils/fetchWithRetry');
+
+function buildError(status, headers = {}) {
+  const err = new Error(status ? `HTTP ${status}` : 'network failed');
+  if (status) {
+    err.response = { status, headers };
+  }
+  return err;
+}
+
+describe('fetchWithRetry', () => {
+  test('succeeds on the second attempt after a retryable network error', async () => {
+    const client = {
+      get: jest
+        .fn()
+        .mockRejectedValueOnce(buildError())
+        .mockResolvedValueOnce({ data: { ok: true } }),
+    };
+    const sleep = jest.fn(async () => {});
+    const logger = { debug: jest.fn() };
+
+    const response = await fetchWithRetry('/price', { client, sleep, logger }, 3, 500);
+
+    expect(response.data.ok).toBe(true);
+    expect(client.get).toHaveBeenCalledTimes(2);
+    expect(sleep).toHaveBeenCalledWith(500);
+    expect(logger.debug).toHaveBeenCalledWith(
+      'Retrying price source request',
+      expect.objectContaining({ attempt: 2, maxAttempts: 4, delayMs: 500, status: null })
+    );
+  });
+
+  test('throws after the maximum retry count is exhausted', async () => {
+    const client = {
+      get: jest.fn().mockRejectedValue(buildError(503)),
+    };
+    const sleep = jest.fn(async () => {});
+
+    await expect(fetchWithRetry('/price', { client, sleep, logger: { debug: jest.fn() } }, 2, 500))
+      .rejects
+      .toThrow('HTTP 503');
+
+    expect(client.get).toHaveBeenCalledTimes(3);
+    expect(sleep).toHaveBeenCalledTimes(2);
+    expect(sleep).toHaveBeenNthCalledWith(1, 500);
+    expect(sleep).toHaveBeenNthCalledWith(2, 2000);
+  });
+
+  test('does not retry non-retryable 401 responses', async () => {
+    const client = {
+      get: jest.fn().mockRejectedValue(buildError(401)),
+    };
+    const sleep = jest.fn(async () => {});
+
+    await expect(fetchWithRetry('/price', { client, sleep, logger: { debug: jest.fn() } }))
+      .rejects
+      .toThrow('HTTP 401');
+
+    expect(client.get).toHaveBeenCalledTimes(1);
+    expect(sleep).not.toHaveBeenCalled();
+  });
+
+  test('honors Retry-After seconds for 429 responses', async () => {
+    const client = {
+      get: jest
+        .fn()
+        .mockRejectedValueOnce(buildError(429, { 'retry-after': '2' }))
+        .mockResolvedValueOnce({ data: { ok: true } }),
+    };
+    const sleep = jest.fn(async () => {});
+
+    await fetchWithRetry('/price', { client, sleep, logger: { debug: jest.fn() } });
+
+    expect(sleep).toHaveBeenCalledWith(2000);
+  });
+
+  test('classifies retryable and non-retryable statuses', () => {
+    expect(isRetryableStatus(undefined)).toBe(true);
+    expect(isRetryableStatus(429)).toBe(true);
+    expect(isRetryableStatus(500)).toBe(true);
+    expect(isRetryableStatus(401)).toBe(false);
+    expect(isRetryableStatus(403)).toBe(false);
+  });
+
+  test('parses Retry-After dates', () => {
+    const delay = parseRetryAfter('Sat, 04 Jul 2026 00:00:05 GMT', Date.parse('Sat, 04 Jul 2026 00:00:00 GMT'));
+
+    expect(delay).toBe(5000);
+  });
+});