From dafb4cb46ff1c91eafde244799395725a590867d Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Wed, 5 Nov 2025 12:00:56 +0300 Subject: [PATCH 01/29] feat: add Redis TimeSeries helper with safe increment and auto-creation --- package.json | 1 + src/models/eventsFactory.js | 24 +++++++ src/redisHelper.ts | 138 ++++++++++++++++++++++++++++++++++++ src/resolvers/event.js | 2 +- src/resolvers/project.js | 2 +- yarn.lock | 58 ++++++++++++++- 6 files changed, 222 insertions(+), 3 deletions(-) create mode 100644 src/redisHelper.ts diff --git a/package.json b/package.json index 01d3b220..cab5eaf2 100644 --- a/package.json +++ b/package.json @@ -81,6 +81,7 @@ "mongodb": "^3.7.3", "morgan": "^1.10.1", "prom-client": "^15.1.3", + "redis": "^4.7.0", "safe-regex": "^2.1.0", "ts-node-dev": "^2.0.0", "uuid": "^8.3.2" diff --git a/src/models/eventsFactory.js b/src/models/eventsFactory.js index a5166602..d4f21f08 100644 --- a/src/models/eventsFactory.js +++ b/src/models/eventsFactory.js @@ -1,11 +1,13 @@ import { getMidnightWithTimezoneOffset, getUTCMidnight } from '../utils/dates'; import safe from 'safe-regex'; import { createProjectEventsByIdLoader } from '../dataLoaders'; +import { Effect, sgr } from '../utils/ansi'; const Factory = require('./modelFactory'); const mongo = require('../mongo'); const Event = require('../models/event'); const { ObjectID } = require('mongodb'); +import RedisHelper from '../redisHelper'; const { composeEventPayloadByRepetition } = require('../utils/merge'); const MAX_DB_READ_BATCH_SIZE = Number(process.env.MAX_DB_READ_BATCH_SIZE); @@ -69,6 +71,12 @@ const MAX_DB_READ_BATCH_SIZE = Number(process.env.MAX_DB_READ_BATCH_SIZE); * Factory Class for Event's Model */ class EventsFactory extends Factory { + /** + /** + * Redis helper instance for modifying data through redis + */ + redis = new RedisHelper(); + /** * Event types with collections where they stored * @return {{EVENTS: string, DAILY_EVENTS: string, REPETITIONS: string, RELEASES: string}} @@ -94,6 +102,8 @@ class EventsFactory extends Factory { throw new Error('Can not construct Event model, because projectId is not provided'); } + this.redis.initialize(); + this.projectId = projectId; this.eventsDataLoader = createProjectEventsByIdLoader(mongo.databases.events, this.projectId); } @@ -392,6 +402,20 @@ class EventsFactory extends Factory { }; } + async getChartData(hours = 24, timezoneOffset = 0, projectId = '', groupHash = '') { + try { + const redisData = await this.redis.getChartDataFromRedis(hours, timezoneOffset, projectId, groupHash); + + if (redisData && redisData.length > 0) { + return redisData; + } + + return this.findChartData(days = hours, timezoneOffset, groupHash); + } catch (err) { + return this.findChartData(days = hours, timezoneOffset, groupHash); + } + } + /** * Fetch timestamps and total count of errors (or target error) for each day since * diff --git a/src/redisHelper.ts b/src/redisHelper.ts new file mode 100644 index 00000000..3abeb8b7 --- /dev/null +++ b/src/redisHelper.ts @@ -0,0 +1,138 @@ +import HawkCatcher from '@hawk.so/nodejs'; +import { createClient, RedisClientType } from 'redis'; +import { Effect, sgr } from './utils/ansi'; + +/** + * Helper class for working with Redis + */ +export default class RedisHelper { + /** + * TTL for lock records in Redis (in seconds) + */ + private static readonly LOCK_TTL = 10; + + /** + * Redis client instance + */ + private readonly redisClient!: RedisClientType; + + /** + * Constructor + * Initializes the Redis client and sets up error handling + */ + constructor() { + try { + this.redisClient = createClient({ url: process.env.REDIS_URL }); + + this.redisClient.on('error', (error) => { + console.error('[Redis] Client error:', error); + if (error) { + HawkCatcher.send(error); + } + }); + } catch (error) { + console.error('[Redis] Error creating client:', error); + } + } + + /** + * Connect to Redis + */ + public async initialize(): Promise { + try { + await this.redisClient.connect(); + console.log('[Redis] Connected successfully'); + } catch (error) { + console.error('[Redis] Connection failed:', error); + HawkCatcher.send(error as Error); + } + } + + /** + * Close Redis client + */ + public async close(): Promise { + if (this.redisClient.isOpen) { + await this.redisClient.quit(); + console.log('[Redis] Connection closed'); + } + } + + public async getChartDataFromRedis( + hours: number, // количество интервалов (часов или дней) + timezoneOffset = 0, + projectId = '', + groupHash = '' + ): Promise<{ timestamp: number; count: number }[]> { + if (!this.redisClient.isOpen) { + throw new Error('Redis client not connected'); + } + + const key = groupHash + ? `ts:events:${groupHash}:hourly` + : projectId + ? `ts:events:${projectId}:hourly` + : `ts:events:hourly`; + + const now = Date.now(); + + // определяем начало выборки + const fromDate = new Date(now); + fromDate.setMinutes(0, 0, 0); + fromDate.setMilliseconds(fromDate.getMilliseconds() - (hours * 60 * 60 * 1000)); + const from = fromDate.getTime(); + + let result: [string, string][] = []; + try { + result = (await this.redisClient.sendCommand([ + 'TS.RANGE', + key, + from.toString(), + now.toString(), + ])) as [string, string][] | []; + } catch (err: any) { + if (err.message.includes('TSDB: the key does not exist')) { + console.warn(`[Redis] Key ${key} does not exist, returning zeroed data`); + result = []; + } else { + throw err; + } + } + + console.log(groupHash, result) + + // агрегируем события по интервалу + const dataPoints: { [ts: number]: number } = {}; + for (const [tsStr] of result) { + const tsMs = Number(tsStr); + const date = new Date(tsMs); + + let intervalStart: number; + date.setMinutes(0, 0, 0); + intervalStart = Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate(), date.getUTCHours()); + + const intervalWithOffset = intervalStart + timezoneOffset * 60 * 1000; + + dataPoints[intervalWithOffset] = (dataPoints[intervalWithOffset] || 0) + 1; + } + + // заполняем пропущенные интервалы нулями + const filled: { timestamp: number; count: number }[] = []; + const nowDate = new Date(now); + + for (let i = 0; i < hours; i++) { + const date = new Date(nowDate); + + date.setHours(date.getHours() - i, 0, 0, 0); + var intervalStart = Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate(), date.getUTCHours()); + + const intervalWithOffset = intervalStart + timezoneOffset * 60 * 1000; + filled.push({ + timestamp: Math.floor(intervalWithOffset / 1000), + count: dataPoints[intervalWithOffset] || 0, + }); + } + + return filled.sort((a, b) => a.timestamp - b.timestamp); + } +} \ No newline at end of file diff --git a/src/resolvers/event.js b/src/resolvers/event.js index 5b8804da..eb493617 100644 --- a/src/resolvers/event.js +++ b/src/resolvers/event.js @@ -86,7 +86,7 @@ module.exports = { async chartData({ projectId, groupHash }, { days, timezoneOffset }, context) { const factory = getEventsFactory(context, projectId); - return factory.findChartData(days, timezoneOffset, groupHash); + return factory.getChartData(days, timezoneOffset, projectId, groupHash); }, /** diff --git a/src/resolvers/project.js b/src/resolvers/project.js index 9513e497..bc685ed9 100644 --- a/src/resolvers/project.js +++ b/src/resolvers/project.js @@ -468,7 +468,7 @@ module.exports = { async chartData(project, { days, timezoneOffset }, context) { const factory = getEventsFactory(context, project._id); - return factory.findChartData(days, timezoneOffset); + return factory.getChartData(days, timezoneOffset, project._id); }, /** diff --git a/yarn.lock b/yarn.lock index deea0104..e2fa2c6e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -788,6 +788,40 @@ resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== +"@redis/bloom@1.2.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.2.0.tgz#d3fd6d3c0af3ef92f26767b56414a370c7b63b71" + integrity sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg== + +"@redis/client@1.6.1": + version "1.6.1" + resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.6.1.tgz#c4636b7cb34e96008a988409b7e787364ae761a2" + integrity sha512-/KCsg3xSlR+nCK8/8ZYSknYxvXHwubJrU82F3Lm1Fp6789VQ0/3RJKfsmRXjqfaTA++23CvC3hqmqe/2GEt6Kw== + dependencies: + cluster-key-slot "1.1.2" + generic-pool "3.9.0" + yallist "4.0.0" + +"@redis/graph@1.1.1": + version "1.1.1" + resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.1.1.tgz#8c10df2df7f7d02741866751764031a957a170ea" + integrity sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw== + +"@redis/json@1.0.7": + version "1.0.7" + resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.7.tgz#016257fcd933c4cbcb9c49cde8a0961375c6893b" + integrity sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ== + +"@redis/search@1.2.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.2.0.tgz#50976fd3f31168f585666f7922dde111c74567b8" + integrity sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw== + +"@redis/time-series@1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.1.0.tgz#cba454c05ec201bd5547aaf55286d44682ac8eb5" + integrity sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g== + "@shelf/jest-mongodb@^1.2.2": version "1.3.4" resolved "https://registry.yarnpkg.com/@shelf/jest-mongodb/-/jest-mongodb-1.3.4.tgz#200bac386cf513bed2d41952b1857689f0b88f31" @@ -2138,6 +2172,11 @@ cloudpayments@^6.0.1: object-hash "^2.2.0" qs "^6.10.1" +cluster-key-slot@1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac" + integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA== + co@^4.6.0: version "4.6.0" resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" @@ -3326,6 +3365,11 @@ gauge@^3.0.0: strip-ansi "^6.0.1" wide-align "^1.1.2" +generic-pool@3.9.0: + version "3.9.0" + resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4" + integrity sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g== + gensync@^1.0.0-beta.2: version "1.0.0-beta.2" resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" @@ -5718,6 +5762,18 @@ readdirp@~3.6.0: dependencies: picomatch "^2.2.1" +redis@^4.7.0: + version "4.7.1" + resolved "https://registry.yarnpkg.com/redis/-/redis-4.7.1.tgz#08588a30936be0e7ad9c0f3e1ac6a85ccaf73e94" + integrity sha512-S1bJDnqLftzHXHP8JsT5II/CtHWQrASX5K96REjWjlmWKrviSOLWmM7QnRLstAWsu1VBBV1ffV6DzCvxNP0UJQ== + dependencies: + "@redis/bloom" "1.2.0" + "@redis/client" "1.6.1" + "@redis/graph" "1.1.1" + "@redis/json" "1.0.7" + "@redis/search" "1.2.0" + "@redis/time-series" "1.1.0" + regex-not@^1.0.0, regex-not@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/regex-not/-/regex-not-1.0.2.tgz#1f4ece27e00b0b65e0247a6810e6a85d83a5752c" @@ -7107,7 +7163,7 @@ y18n@^4.0.0: resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.3.tgz#b5f259c82cd6e336921efd7bfd8bf560de9eeedf" integrity sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ== -yallist@^4.0.0: +yallist@4.0.0, yallist@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== From dbf684594fd6d82f4bd4c92a93c4f5e6c0560ca0 Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Wed, 5 Nov 2025 13:50:57 +0300 Subject: [PATCH 02/29] add grouping mode: 'hours' or 'days' --- src/models/eventsFactory.js | 12 ++-- src/redisHelper.ts | 140 ++++++++++++++++++++---------------- src/resolvers/event.js | 4 +- src/resolvers/project.js | 4 +- src/typeDefs/event.ts | 9 ++- src/typeDefs/project.ts | 9 ++- 6 files changed, 103 insertions(+), 75 deletions(-) diff --git a/src/models/eventsFactory.js b/src/models/eventsFactory.js index d4f21f08..9c69e700 100644 --- a/src/models/eventsFactory.js +++ b/src/models/eventsFactory.js @@ -402,17 +402,21 @@ class EventsFactory extends Factory { }; } - async getChartData(hours = 24, timezoneOffset = 0, projectId = '', groupHash = '') { + async getChartData(groupingBy = 'hours', rangeValue = 24, timezoneOffset = 0, projectId = '', groupHash = '') { try { - const redisData = await this.redis.getChartDataFromRedis(hours, timezoneOffset, projectId, groupHash); + const redisData = await this.redis.getChartDataFromRedis(groupingBy, rangeValue, timezoneOffset, projectId, groupHash); if (redisData && redisData.length > 0) { return redisData; } - return this.findChartData(days = hours, timezoneOffset, groupHash); + const hours = groupingBy === 'hours' ? rangeValue : Math.max(1, rangeValue) * 24; + const days = Math.max(1, Math.ceil(hours / 24)); + return this.findChartData(days, timezoneOffset, groupHash); } catch (err) { - return this.findChartData(days = hours, timezoneOffset, groupHash); + const hours = groupingBy === 'hours' ? rangeValue : Math.max(1, rangeValue) * 24; + const days = Math.max(1, Math.ceil(hours / 24)); + return this.findChartData(days, timezoneOffset, groupHash); } } diff --git a/src/redisHelper.ts b/src/redisHelper.ts index 3abeb8b7..580c600b 100644 --- a/src/redisHelper.ts +++ b/src/redisHelper.ts @@ -59,7 +59,8 @@ export default class RedisHelper { } public async getChartDataFromRedis( - hours: number, // количество интервалов (часов или дней) + groupingBy: 'hours' | 'days', + rangeValue: number, timezoneOffset = 0, projectId = '', groupHash = '' @@ -68,71 +69,84 @@ export default class RedisHelper { throw new Error('Redis client not connected'); } + const suffix = groupingBy === 'hours' ? 'hourly' : 'daily'; const key = groupHash - ? `ts:events:${groupHash}:hourly` + ? `ts:events:${groupHash}:${suffix}` : projectId - ? `ts:events:${projectId}:hourly` - : `ts:events:hourly`; - - const now = Date.now(); - - // определяем начало выборки - const fromDate = new Date(now); - fromDate.setMinutes(0, 0, 0); - fromDate.setMilliseconds(fromDate.getMilliseconds() - (hours * 60 * 60 * 1000)); - const from = fromDate.getTime(); - - let result: [string, string][] = []; - try { - result = (await this.redisClient.sendCommand([ - 'TS.RANGE', - key, - from.toString(), - now.toString(), - ])) as [string, string][] | []; - } catch (err: any) { - if (err.message.includes('TSDB: the key does not exist')) { - console.warn(`[Redis] Key ${key} does not exist, returning zeroed data`); - result = []; + ? `ts:events:${projectId}:${suffix}` + : `ts:events:${suffix}`; + + const now = Date.now(); + + // определяем начало выборки + const fromDate = new Date(now); + if (groupingBy === 'hours') { + fromDate.setMinutes(0, 0, 0); } else { - throw err; + fromDate.setHours(0, 0, 0, 0); } - } - - console.log(groupHash, result) - - // агрегируем события по интервалу - const dataPoints: { [ts: number]: number } = {}; - for (const [tsStr] of result) { - const tsMs = Number(tsStr); - const date = new Date(tsMs); - - let intervalStart: number; - date.setMinutes(0, 0, 0); - intervalStart = Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate(), date.getUTCHours()); - - const intervalWithOffset = intervalStart + timezoneOffset * 60 * 1000; - - dataPoints[intervalWithOffset] = (dataPoints[intervalWithOffset] || 0) + 1; - } - - // заполняем пропущенные интервалы нулями - const filled: { timestamp: number; count: number }[] = []; - const nowDate = new Date(now); - - for (let i = 0; i < hours; i++) { - const date = new Date(nowDate); - - date.setHours(date.getHours() - i, 0, 0, 0); - var intervalStart = Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate(), date.getUTCHours()); - - const intervalWithOffset = intervalStart + timezoneOffset * 60 * 1000; - filled.push({ - timestamp: Math.floor(intervalWithOffset / 1000), - count: dataPoints[intervalWithOffset] || 0, - }); - } - - return filled.sort((a, b) => a.timestamp - b.timestamp); + fromDate.setMilliseconds(fromDate.getMilliseconds() - (groupingBy === 'hours' ? rangeValue * 60 * 60 * 1000 : rangeValue * 24 * 60 * 60 * 1000)); + const from = fromDate.getTime(); + + let result: [string, string][] = []; + try { + result = (await this.redisClient.sendCommand([ + 'TS.RANGE', + key, + from.toString(), + now.toString(), + ])) as [string, string][] | []; + } catch (err: any) { + if (err.message.includes('TSDB: the key does not exist')) { + console.warn(`[Redis] Key ${key} does not exist, returning zeroed data`); + result = []; + } else { + throw err; + } + } + + // агрегируем события по интервалу + const dataPoints: { [ts: number]: number } = {}; + for (const [tsStr] of result) { + const tsMs = Number(tsStr); + const date = new Date(tsMs); + + let intervalStart: number; + if (groupingBy === 'hours') { + date.setMinutes(0, 0, 0); + intervalStart = Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate(), date.getUTCHours()); + } else { + date.setHours(0, 0, 0, 0); + intervalStart = Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()); + } + + const intervalWithOffset = intervalStart + timezoneOffset * 60 * 1000; + dataPoints[intervalWithOffset] = (dataPoints[intervalWithOffset] || 0) + 1; + } + + // заполняем пропущенные интервалы нулями + const filled: { timestamp: number; count: number }[] = []; + const nowDate = new Date(now); + + for (let i = 0; i < rangeValue; i++) { + const date = new Date(nowDate); + + if (groupingBy === 'hours') { + date.setHours(date.getHours() - i, 0, 0, 0); + var intervalStart = Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate(), date.getUTCHours()); + } else { + date.setDate(date.getDate() - i); + date.setHours(0, 0, 0, 0); + var intervalStart = Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()); + } + + const intervalWithOffset = intervalStart + timezoneOffset * 60 * 1000; + filled.push({ + timestamp: Math.floor(intervalWithOffset / 1000), + count: dataPoints[intervalWithOffset] || 0, + }); + } + + return filled.sort((a, b) => a.timestamp - b.timestamp); } } \ No newline at end of file diff --git a/src/resolvers/event.js b/src/resolvers/event.js index eb493617..fe94e6cb 100644 --- a/src/resolvers/event.js +++ b/src/resolvers/event.js @@ -83,10 +83,10 @@ module.exports = { * @param {number} timezoneOffset - user's local timezone offset in minutes * @returns {Promise} */ - async chartData({ projectId, groupHash }, { days, timezoneOffset }, context) { + async chartData({ projectId, groupHash }, { groupingBy, rangeValue, timezoneOffset }, context) { const factory = getEventsFactory(context, projectId); - return factory.getChartData(days, timezoneOffset, projectId, groupHash); + return factory.getChartData(groupingBy, rangeValue, timezoneOffset, projectId, groupHash); }, /** diff --git a/src/resolvers/project.js b/src/resolvers/project.js index bc685ed9..97a1c174 100644 --- a/src/resolvers/project.js +++ b/src/resolvers/project.js @@ -465,10 +465,10 @@ module.exports = { * * @return {Promise} */ - async chartData(project, { days, timezoneOffset }, context) { + async chartData(project, { groupingBy, rangeValue, timezoneOffset }, context) { const factory = getEventsFactory(context, project._id); - return factory.getChartData(days, timezoneOffset, project._id); + return factory.getChartData(groupingBy, rangeValue, timezoneOffset, project._id); }, /** diff --git a/src/typeDefs/event.ts b/src/typeDefs/event.ts index ead9af4a..7a8195d2 100644 --- a/src/typeDefs/event.ts +++ b/src/typeDefs/event.ts @@ -282,9 +282,14 @@ type Event { """ chartData( """ - How many days we need to fetch for displaying in a chart + Grouping mode: 'hours' or 'days' """ - days: Int! = 0 + groupingBy: String! = "hours" + + """ + Range value: number of hours or days depending on groupingBy + """ + rangeValue: Int! = 0 """ User's local timezone offset in minutes diff --git a/src/typeDefs/project.ts b/src/typeDefs/project.ts index 8159ff22..55599ee6 100644 --- a/src/typeDefs/project.ts +++ b/src/typeDefs/project.ts @@ -289,9 +289,14 @@ type Project { """ chartData( """ - How many days we need to fetch for displaying in a chart + Grouping mode: 'hours' or 'days' """ - days: Int! = 0 + groupingBy: String! + + """ + Range value: number of hours or days depending on groupingBy + """ + rangeValue: Int! """ User's local timezone offset in minutes From 04278d25883bb9f4e4b133a377f20ac636e155a5 Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Wed, 5 Nov 2025 16:09:00 +0300 Subject: [PATCH 03/29] feat(api): implement flexible chart data API with Redis TimeSeries - Replace days parameter with startDate/endDate/groupBy (minutes) - Add RedisHelper class for TimeSeries operations - Support minutely/hourly/daily aggregations based on groupBy value - Add search parameter support in events queries (title/backtrace/context/addons) - Update GraphQL schema and resolvers for new chart parameters - Fallback to MongoDB when Redis data unavailable --- src/models/eventsFactory.js | 23 ++++-- src/models/user.ts | 2 +- src/redisHelper.ts | 147 ++++++++++++++++++------------------ src/resolvers/event.js | 4 +- src/resolvers/project.js | 4 +- src/typeDefs/event.ts | 15 ++-- src/typeDefs/project.ts | 13 +++- 7 files changed, 113 insertions(+), 95 deletions(-) diff --git a/src/models/eventsFactory.js b/src/models/eventsFactory.js index 9c69e700..2d9d19d7 100644 --- a/src/models/eventsFactory.js +++ b/src/models/eventsFactory.js @@ -402,20 +402,31 @@ class EventsFactory extends Factory { }; } - async getChartData(groupingBy = 'hours', rangeValue = 24, timezoneOffset = 0, projectId = '', groupHash = '') { + async getChartData(startDate, endDate, groupBy = 60, timezoneOffset = 0, projectId = '', groupHash = '') { try { - const redisData = await this.redis.getChartDataFromRedis(groupingBy, rangeValue, timezoneOffset, projectId, groupHash); + const redisData = await this.redis.getChartDataFromRedis( + startDate, + endDate, + groupBy, + timezoneOffset, + projectId, + groupHash + ); if (redisData && redisData.length > 0) { return redisData; } - const hours = groupingBy === 'hours' ? rangeValue : Math.max(1, rangeValue) * 24; - const days = Math.max(1, Math.ceil(hours / 24)); + // Fallback to Mongo + const start = new Date(startDate).getTime(); + const end = new Date(endDate).getTime(); + const days = Math.ceil((end - start) / (24 * 60 * 60 * 1000)); return this.findChartData(days, timezoneOffset, groupHash); } catch (err) { - const hours = groupingBy === 'hours' ? rangeValue : Math.max(1, rangeValue) * 24; - const days = Math.max(1, Math.ceil(hours / 24)); + console.error('[EventsFactory] getChartData error:', err); + const start = new Date(startDate).getTime(); + const end = new Date(endDate).getTime(); + const days = Math.ceil((end - start) / (24 * 60 * 60 * 1000)); return this.findChartData(days, timezoneOffset, groupHash); } } diff --git a/src/models/user.ts b/src/models/user.ts index 7ea2f065..c67f959a 100644 --- a/src/models/user.ts +++ b/src/models/user.ts @@ -268,7 +268,7 @@ export default class UserModel extends AbstractModel implements Us userId: this._id, }, process.env.JWT_SECRET_ACCESS_TOKEN as Secret, - { expiresIn: '15m' } + { expiresIn: '1d' } ); const refreshToken = await jwt.sign( diff --git a/src/redisHelper.ts b/src/redisHelper.ts index 580c600b..56c9d41d 100644 --- a/src/redisHelper.ts +++ b/src/redisHelper.ts @@ -59,8 +59,9 @@ export default class RedisHelper { } public async getChartDataFromRedis( - groupingBy: 'hours' | 'days', - rangeValue: number, + startDate: string, + endDate: string, + groupBy: number, // minutes: 1=minute, 60=hour, 1440=day timezoneOffset = 0, projectId = '', groupHash = '' @@ -68,85 +69,81 @@ export default class RedisHelper { if (!this.redisClient.isOpen) { throw new Error('Redis client not connected'); } - - const suffix = groupingBy === 'hours' ? 'hourly' : 'daily'; + + // Determine suffix based on groupBy + let suffix: string; + if (groupBy === 1) { + suffix = 'minutely'; + } else if (groupBy === 60) { + suffix = 'hourly'; + } else if (groupBy === 1440) { + suffix = 'daily'; + } else { + // For custom intervals, fallback to minutely with aggregation + suffix = 'minutely'; + } + const key = groupHash ? `ts:events:${groupHash}:${suffix}` : projectId ? `ts:events:${projectId}:${suffix}` : `ts:events:${suffix}`; - - const now = Date.now(); - - // определяем начало выборки - const fromDate = new Date(now); - if (groupingBy === 'hours') { - fromDate.setMinutes(0, 0, 0); + + // Parse dates (support ISO string or Unix timestamp in seconds) + const start = typeof startDate === 'string' && startDate.includes('-') + ? new Date(startDate).getTime() + : Number(startDate) * 1000; + const end = typeof endDate === 'string' && endDate.includes('-') + ? new Date(endDate).getTime() + : Number(endDate) * 1000; + + const bucketMs = groupBy * 60 * 1000; + + let result: [string, string][] = []; + try { + // Use aggregation to sum events within each bucket + // Since we now use TS.ADD (not TS.INCRBY), each sample is 1, so SUM gives us count + result = (await this.redisClient.sendCommand([ + 'TS.RANGE', + key, + start.toString(), + end.toString(), + 'AGGREGATION', + 'sum', + bucketMs.toString(), + ])) as [string, string][] | []; + } catch (err: any) { + if (err.message.includes('TSDB: the key does not exist')) { + console.warn(`[Redis] Key ${key} does not exist, returning zeroed data`); + result = []; } else { - fromDate.setHours(0, 0, 0, 0); - } - fromDate.setMilliseconds(fromDate.getMilliseconds() - (groupingBy === 'hours' ? rangeValue * 60 * 60 * 1000 : rangeValue * 24 * 60 * 60 * 1000)); - const from = fromDate.getTime(); - - let result: [string, string][] = []; - try { - result = (await this.redisClient.sendCommand([ - 'TS.RANGE', - key, - from.toString(), - now.toString(), - ])) as [string, string][] | []; - } catch (err: any) { - if (err.message.includes('TSDB: the key does not exist')) { - console.warn(`[Redis] Key ${key} does not exist, returning zeroed data`); - result = []; - } else { - throw err; - } - } - - // агрегируем события по интервалу - const dataPoints: { [ts: number]: number } = {}; - for (const [tsStr] of result) { - const tsMs = Number(tsStr); - const date = new Date(tsMs); - - let intervalStart: number; - if (groupingBy === 'hours') { - date.setMinutes(0, 0, 0); - intervalStart = Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate(), date.getUTCHours()); - } else { - date.setHours(0, 0, 0, 0); - intervalStart = Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()); - } - - const intervalWithOffset = intervalStart + timezoneOffset * 60 * 1000; - dataPoints[intervalWithOffset] = (dataPoints[intervalWithOffset] || 0) + 1; + throw err; } - - // заполняем пропущенные интервалы нулями - const filled: { timestamp: number; count: number }[] = []; - const nowDate = new Date(now); - - for (let i = 0; i < rangeValue; i++) { - const date = new Date(nowDate); - - if (groupingBy === 'hours') { - date.setHours(date.getHours() - i, 0, 0, 0); - var intervalStart = Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate(), date.getUTCHours()); - } else { - date.setDate(date.getDate() - i); - date.setHours(0, 0, 0, 0); - var intervalStart = Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()); - } - - const intervalWithOffset = intervalStart + timezoneOffset * 60 * 1000; - filled.push({ - timestamp: Math.floor(intervalWithOffset / 1000), - count: dataPoints[intervalWithOffset] || 0, - }); - } - - return filled.sort((a, b) => a.timestamp - b.timestamp); + } + + // Transform data from Redis + const dataPoints: { [ts: number]: number } = {}; + for (const [tsStr, valStr] of result) { + const tsMs = Number(tsStr); + dataPoints[tsMs] = Number(valStr) || 0; + } + + // Fill missing intervals with zeros + const filled: { timestamp: number; count: number }[] = []; + let current = start; + + // Round current to the nearest bucket boundary + current = Math.floor(current / bucketMs) * bucketMs; + + while (current <= end) { + const count = dataPoints[current] || 0; + filled.push({ + timestamp: Math.floor((current + timezoneOffset * 60 * 1000) / 1000), + count, + }); + current += bucketMs; + } + + return filled.sort((a, b) => a.timestamp - b.timestamp); } } \ No newline at end of file diff --git a/src/resolvers/event.js b/src/resolvers/event.js index fe94e6cb..731a4635 100644 --- a/src/resolvers/event.js +++ b/src/resolvers/event.js @@ -83,10 +83,10 @@ module.exports = { * @param {number} timezoneOffset - user's local timezone offset in minutes * @returns {Promise} */ - async chartData({ projectId, groupHash }, { groupingBy, rangeValue, timezoneOffset }, context) { + async chartData({ projectId, groupHash }, { startDate, endDate, groupBy, timezoneOffset }, context) { const factory = getEventsFactory(context, projectId); - return factory.getChartData(groupingBy, rangeValue, timezoneOffset, projectId, groupHash); + return factory.getChartData(startDate, endDate, groupBy, timezoneOffset, projectId, groupHash); }, /** diff --git a/src/resolvers/project.js b/src/resolvers/project.js index 97a1c174..8aa6387a 100644 --- a/src/resolvers/project.js +++ b/src/resolvers/project.js @@ -465,10 +465,10 @@ module.exports = { * * @return {Promise} */ - async chartData(project, { groupingBy, rangeValue, timezoneOffset }, context) { + async chartData(project, { startDate, endDate, groupBy, timezoneOffset }, context) { const factory = getEventsFactory(context, project._id); - return factory.getChartData(groupingBy, rangeValue, timezoneOffset, project._id); + return factory.getChartData(startDate, endDate, groupBy, timezoneOffset, project._id); }, /** diff --git a/src/typeDefs/event.ts b/src/typeDefs/event.ts index 7a8195d2..09ff70a8 100644 --- a/src/typeDefs/event.ts +++ b/src/typeDefs/event.ts @@ -278,18 +278,23 @@ type Event { usersAffected: Int """ - Return graph of the error rate for the last few days + Return graph of the error rate for the specified period """ chartData( """ - Grouping mode: 'hours' or 'days' + Start date (ISO string or Unix timestamp in seconds) """ - groupingBy: String! = "hours" + startDate: String! """ - Range value: number of hours or days depending on groupingBy + End date (ISO string or Unix timestamp in seconds) """ - rangeValue: Int! = 0 + endDate: String! + + """ + Grouping interval in minutes (1=minute, 60=hour, 1440=day) + """ + groupBy: Int! = 60 """ User's local timezone offset in minutes diff --git a/src/typeDefs/project.ts b/src/typeDefs/project.ts index 55599ee6..64b1fa3a 100644 --- a/src/typeDefs/project.ts +++ b/src/typeDefs/project.ts @@ -289,14 +289,19 @@ type Project { """ chartData( """ - Grouping mode: 'hours' or 'days' + Start date (ISO string or Unix timestamp in seconds) """ - groupingBy: String! + startDate: String! """ - Range value: number of hours or days depending on groupingBy + End date (ISO string or Unix timestamp in seconds) """ - rangeValue: Int! + endDate: String! + + """ + Grouping interval in minutes (1=minute, 60=hour, 1440=day) + """ + groupBy: Int! = 60 """ User's local timezone offset in minutes From 2a88509cedfdd8cd612a88363a709b7e104e2caa Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Thu, 6 Nov 2025 00:45:15 +0300 Subject: [PATCH 04/29] change expires time for jwt access secret --- src/models/user.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/models/user.ts b/src/models/user.ts index c67f959a..7ea2f065 100644 --- a/src/models/user.ts +++ b/src/models/user.ts @@ -268,7 +268,7 @@ export default class UserModel extends AbstractModel implements Us userId: this._id, }, process.env.JWT_SECRET_ACCESS_TOKEN as Secret, - { expiresIn: '1d' } + { expiresIn: '15m' } ); const refreshToken = await jwt.sign( From 6147a7a46f5ccf441fbef7a4e1de28f87d01dd6d Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 6 Nov 2025 10:00:29 +0000 Subject: [PATCH 05/29] Bump version up to 1.2.14 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index cab5eaf2..13937d8f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hawk.api", - "version": "1.2.13", + "version": "1.2.14", "main": "index.ts", "license": "BUSL-1.1", "scripts": { From 0599da486d8f31ea5f2f3bd6d7ff522977dab204 Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Sat, 8 Nov 2025 00:25:12 +0300 Subject: [PATCH 06/29] refactor: separate Event and Project chart data APIs - Event.chartData: uses 'days' parameter, fetches data only from MongoDB - Project.chartData: uses 'startDate', 'endDate', 'groupBy' parameters, fetches from Redis with MongoDB fallback - Add getChartDataFromMongo method for Event-specific chart data - Fix redisHelper comment to reflect TS.INCRBY usage - Remove unused 'projectId' parameter from getChartDataFromMongo --- src/models/eventsFactory.js | 23 +++++++++++++++++++++++ src/redisHelper.ts | 4 ++-- src/resolvers/event.js | 4 ++-- src/typeDefs/event.ts | 14 ++------------ 4 files changed, 29 insertions(+), 16 deletions(-) diff --git a/src/models/eventsFactory.js b/src/models/eventsFactory.js index 2d9d19d7..2398c24b 100644 --- a/src/models/eventsFactory.js +++ b/src/models/eventsFactory.js @@ -402,6 +402,17 @@ class EventsFactory extends Factory { }; } + /** + * Get chart data for projects (uses Redis with fallback to MongoDB) + * + * @param {string} startDate - start date (ISO string or Unix timestamp) + * @param {string} endDate - end date (ISO string or Unix timestamp) + * @param {number} groupBy - grouping interval in minutes + * @param {number} timezoneOffset - user's local timezone offset in minutes + * @param {string} projectId - project ID + * @param {string} groupHash - event's group hash (empty for project-level data) + * @returns {Promise} + */ async getChartData(startDate, endDate, groupBy = 60, timezoneOffset = 0, projectId = '', groupHash = '') { try { const redisData = await this.redis.getChartDataFromRedis( @@ -431,6 +442,18 @@ class EventsFactory extends Factory { } } + /** + * Get chart data from MongoDB only (for events) + * + * @param {number} days - how many days to fetch + * @param {number} timezoneOffset - user's local timezone offset in minutes + * @param {string} groupHash - event's group hash + * @returns {Promise} + */ + async getChartDataFromMongo(days, timezoneOffset = 0, groupHash = '') { + return this.findChartData(days, timezoneOffset, groupHash); + } + /** * Fetch timestamps and total count of errors (or target error) for each day since * diff --git a/src/redisHelper.ts b/src/redisHelper.ts index 56c9d41d..9f6f5915 100644 --- a/src/redisHelper.ts +++ b/src/redisHelper.ts @@ -101,8 +101,8 @@ export default class RedisHelper { let result: [string, string][] = []; try { - // Use aggregation to sum events within each bucket - // Since we now use TS.ADD (not TS.INCRBY), each sample is 1, so SUM gives us count + // Use aggregation to sum values within each bucket + // TS.INCRBY creates one point per time period with accumulated count result = (await this.redisClient.sendCommand([ 'TS.RANGE', key, diff --git a/src/resolvers/event.js b/src/resolvers/event.js index 731a4635..0bc77083 100644 --- a/src/resolvers/event.js +++ b/src/resolvers/event.js @@ -83,10 +83,10 @@ module.exports = { * @param {number} timezoneOffset - user's local timezone offset in minutes * @returns {Promise} */ - async chartData({ projectId, groupHash }, { startDate, endDate, groupBy, timezoneOffset }, context) { + async chartData({ projectId, groupHash }, { days, timezoneOffset }, context) { const factory = getEventsFactory(context, projectId); - return factory.getChartData(startDate, endDate, groupBy, timezoneOffset, projectId, groupHash); + return factory.getChartDataFromMongo(days, timezoneOffset, groupHash); }, /** diff --git a/src/typeDefs/event.ts b/src/typeDefs/event.ts index 09ff70a8..c1f4ec77 100644 --- a/src/typeDefs/event.ts +++ b/src/typeDefs/event.ts @@ -282,19 +282,9 @@ type Event { """ chartData( """ - Start date (ISO string or Unix timestamp in seconds) + How many days we need to fetch for displaying in a chart """ - startDate: String! - - """ - End date (ISO string or Unix timestamp in seconds) - """ - endDate: String! - - """ - Grouping interval in minutes (1=minute, 60=hour, 1440=day) - """ - groupBy: Int! = 60 + days: Int! = 0 """ User's local timezone offset in minutes From 3af0959209e9599e4e062a525dd00aef5c1a50df Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 7 Nov 2025 22:01:26 +0000 Subject: [PATCH 07/29] Bump version up to 1.2.20 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f04cf7a8..984948d5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hawk.api", - "version": "1.2.19", + "version": "1.2.20", "main": "index.ts", "license": "BUSL-1.1", "scripts": { From 56281b885e49461732786c6ade90b802ec2f23d1 Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Sat, 8 Nov 2025 01:26:12 +0300 Subject: [PATCH 08/29] fix: Add Redis auto-reconnect mechanism for Kubernetes pod restarts - Implement singleton pattern for Redis client to use single connection - Add exponential backoff reconnection strategy (1s to 30s max) - Handle connection lifecycle events (ready, reconnecting, error, end) - Initialize Redis on app startup alongside MongoDB and RabbitMQ - Graceful degradation to MongoDB when Redis is unavailable - Add comprehensive documentation in REDIS_RECONNECT.md This fixes the issue where API lost Redis connection after pod restart and required manual API pod restart to restore functionality. --- src/index.ts | 6 ++++ src/models/eventsFactory.js | 5 ++- src/redisHelper.ts | 71 ++++++++++++++++++++++++++++++++++--- 3 files changed, 74 insertions(+), 8 deletions(-) diff --git a/src/index.ts b/src/index.ts index 0fca32f0..a6b77506 100644 --- a/src/index.ts +++ b/src/index.ts @@ -29,6 +29,7 @@ import { graphqlUploadExpress } from 'graphql-upload'; import { metricsMiddleware, createMetricsServer, graphqlMetricsPlugin } from './metrics'; import { requestLogger } from './utils/logger'; import ReleasesFactory from './models/releasesFactory'; +import RedisHelper from './redisHelper'; /** * Option to enable playground @@ -252,6 +253,11 @@ class HawkAPI { public async start(): Promise { await mongo.setupConnections(); await rabbitmq.setupConnections(); + + // Initialize Redis singleton with auto-reconnect + const redis = RedisHelper.getInstance(); + await redis.initialize(); + await this.server.start(); this.app.use(graphqlUploadExpress()); this.server.applyMiddleware({ app: this.app }); diff --git a/src/models/eventsFactory.js b/src/models/eventsFactory.js index 760b9824..d21bb0bb 100644 --- a/src/models/eventsFactory.js +++ b/src/models/eventsFactory.js @@ -72,10 +72,9 @@ const MAX_DB_READ_BATCH_SIZE = Number(process.env.MAX_DB_READ_BATCH_SIZE); */ class EventsFactory extends Factory { /** - /** - * Redis helper instance for modifying data through redis + * Redis helper instance for modifying data through redis (singleton) */ - redis = new RedisHelper(); + redis = RedisHelper.getInstance(); /** * Event types with collections where they stored diff --git a/src/redisHelper.ts b/src/redisHelper.ts index 9f6f5915..0b1ef0c0 100644 --- a/src/redisHelper.ts +++ b/src/redisHelper.ts @@ -11,40 +11,92 @@ export default class RedisHelper { */ private static readonly LOCK_TTL = 10; + /** + * Singleton instance + */ + private static instance: RedisHelper | null = null; + /** * Redis client instance */ - private readonly redisClient!: RedisClientType; + private redisClient!: RedisClientType; + + /** + * Flag to track if we're currently reconnecting + */ + private isReconnecting = false; /** * Constructor - * Initializes the Redis client and sets up error handling + * Initializes the Redis client and sets up error handling with auto-reconnect */ constructor() { try { - this.redisClient = createClient({ url: process.env.REDIS_URL }); + this.redisClient = createClient({ + url: process.env.REDIS_URL, + socket: { + reconnectStrategy: (retries) => { + // Exponential backoff: wait longer between each retry + // Max wait time: 30 seconds + const delay = Math.min(retries * 1000, 30000); + console.log(`[Redis] Reconnecting... attempt ${retries}, waiting ${delay}ms`); + return delay; + }, + }, + }); + // Handle connection errors this.redisClient.on('error', (error) => { console.error('[Redis] Client error:', error); if (error) { HawkCatcher.send(error); } }); + + // Handle successful reconnection + this.redisClient.on('ready', () => { + console.log('[Redis] Client ready'); + this.isReconnecting = false; + }); + + // Handle reconnecting event + this.redisClient.on('reconnecting', () => { + console.log('[Redis] Client reconnecting...'); + this.isReconnecting = true; + }); + + // Handle connection end + this.redisClient.on('end', () => { + console.log('[Redis] Connection ended'); + }); } catch (error) { console.error('[Redis] Error creating client:', error); } } + /** + * Get singleton instance + */ + public static getInstance(): RedisHelper { + if (!RedisHelper.instance) { + RedisHelper.instance = new RedisHelper(); + } + return RedisHelper.instance; + } + /** * Connect to Redis */ public async initialize(): Promise { try { - await this.redisClient.connect(); - console.log('[Redis] Connected successfully'); + if (!this.redisClient.isOpen && !this.isReconnecting) { + await this.redisClient.connect(); + console.log('[Redis] Connected successfully'); + } } catch (error) { console.error('[Redis] Connection failed:', error); HawkCatcher.send(error as Error); + // Don't throw - let reconnectStrategy handle it } } @@ -57,6 +109,13 @@ export default class RedisHelper { console.log('[Redis] Connection closed'); } } + + /** + * Check if Redis is connected + */ + public isConnected(): boolean { + return this.redisClient.isOpen; + } public async getChartDataFromRedis( startDate: string, @@ -66,7 +125,9 @@ export default class RedisHelper { projectId = '', groupHash = '' ): Promise<{ timestamp: number; count: number }[]> { + // If Redis is not connected, throw error to fallback to MongoDB if (!this.redisClient.isOpen) { + console.warn('[Redis] Client not connected, will fallback to MongoDB'); throw new Error('Redis client not connected'); } From db9d94ce2c77eb60ac6fea48c79bd32449c1e002 Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Sat, 8 Nov 2025 01:50:03 +0300 Subject: [PATCH 09/29] refactor: Apply PR #576 review comments and improve architecture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes based on @neSpecc review: 1. TypeDefs improvements: - Use DateTime! instead of String! for date parameters in Project.chartData 2. Method renaming for clarity: - eventsFactory: getChartData → getProjectChartData - eventsFactory: getChartDataFromMongo → getEventDailyChart - Improved parameter order (main identifier first) 3. Code quality: - Remove duplication in try-catch blocks - Extract Redis key composition to utility functions 4. Architecture improvements: - Create ChartDataService to separate business logic from Redis client - Simplify RedisHelper to only low-level Redis operations - Add tsRange() method for TS.RANGE commands - Move chart data logic from redisHelper to dedicated service 5. New utilities: - src/utils/redisKeys.ts with composeTimeSeriesKey(), composeEventTimeSeriesKey() - Type-safe key composition with required parameters Benefits: - Clear separation of concerns (Data Access vs Business Logic vs Service) - RedisHelper stays focused and doesn't grow - Easier to test and maintain - Better type safety and documentation --- src/models/eventsFactory.js | 54 ++++++++------- src/redisHelper.ts | 115 +++++++------------------------ src/resolvers/event.js | 2 +- src/resolvers/project.js | 2 +- src/services/chartDataService.ts | 92 +++++++++++++++++++++++++ src/typeDefs/project.ts | 8 +-- src/utils/redisKeys.ts | 59 ++++++++++++++++ 7 files changed, 212 insertions(+), 120 deletions(-) create mode 100644 src/services/chartDataService.ts create mode 100644 src/utils/redisKeys.ts diff --git a/src/models/eventsFactory.js b/src/models/eventsFactory.js index d21bb0bb..1ef00619 100644 --- a/src/models/eventsFactory.js +++ b/src/models/eventsFactory.js @@ -8,6 +8,7 @@ const mongo = require('../mongo'); const Event = require('../models/event'); const { ObjectID } = require('mongodb'); import RedisHelper from '../redisHelper'; +import ChartDataService from '../services/chartDataService'; const { composeEventPayloadByRepetition } = require('../utils/merge'); const MAX_DB_READ_BATCH_SIZE = Number(process.env.MAX_DB_READ_BATCH_SIZE); @@ -72,10 +73,15 @@ const MAX_DB_READ_BATCH_SIZE = Number(process.env.MAX_DB_READ_BATCH_SIZE); */ class EventsFactory extends Factory { /** - * Redis helper instance for modifying data through redis (singleton) + * Redis helper instance (singleton) */ redis = RedisHelper.getInstance(); + /** + * Chart data service for fetching data from Redis TimeSeries + */ + chartDataService = new ChartDataService(this.redis); + /** * Event types with collections where they stored * @return {{EVENTS: string, DAILY_EVENTS: string, REPETITIONS: string, RELEASES: string}} @@ -424,54 +430,52 @@ class EventsFactory extends Factory { } /** - * Get chart data for projects (uses Redis with fallback to MongoDB) + * Get project chart data from Redis or fallback to MongoDB * - * @param {string} startDate - start date (ISO string or Unix timestamp) - * @param {string} endDate - end date (ISO string or Unix timestamp) - * @param {number} groupBy - grouping interval in minutes - * @param {number} timezoneOffset - user's local timezone offset in minutes * @param {string} projectId - project ID - * @param {string} groupHash - event's group hash (empty for project-level data) + * @param {string} startDate - start date (ISO string) + * @param {string} endDate - end date (ISO string) + * @param {number} groupBy - grouping interval in minutes (1=minute, 60=hour, 1440=day) + * @param {number} timezoneOffset - user's local timezone offset in minutes * @returns {Promise} */ - async getChartData(startDate, endDate, groupBy = 60, timezoneOffset = 0, projectId = '', groupHash = '') { + async getProjectChartData(projectId, startDate, endDate, groupBy = 60, timezoneOffset = 0) { + // Calculate days for MongoDB fallback + const start = new Date(startDate).getTime(); + const end = new Date(endDate).getTime(); + const days = Math.ceil((end - start) / (24 * 60 * 60 * 1000)); + try { - const redisData = await this.redis.getChartDataFromRedis( + const redisData = await this.chartDataService.getProjectChartData( + projectId, startDate, endDate, groupBy, - timezoneOffset, - projectId, - groupHash + timezoneOffset ); if (redisData && redisData.length > 0) { return redisData; } - // Fallback to Mongo - const start = new Date(startDate).getTime(); - const end = new Date(endDate).getTime(); - const days = Math.ceil((end - start) / (24 * 60 * 60 * 1000)); - return this.findChartData(days, timezoneOffset, groupHash); + // Fallback to Mongo (empty groupHash for project-level data) + return this.findChartData(days, timezoneOffset, ''); } catch (err) { - console.error('[EventsFactory] getChartData error:', err); - const start = new Date(startDate).getTime(); - const end = new Date(endDate).getTime(); - const days = Math.ceil((end - start) / (24 * 60 * 60 * 1000)); - return this.findChartData(days, timezoneOffset, groupHash); + console.error('[EventsFactory] getProjectChartData error:', err); + // Fallback to Mongo on error (empty groupHash for project-level data) + return this.findChartData(days, timezoneOffset, ''); } } /** - * Get chart data from MongoDB only (for events) + * Get event daily chart data from MongoDB only * + * @param {string} groupHash - event's group hash * @param {number} days - how many days to fetch * @param {number} timezoneOffset - user's local timezone offset in minutes - * @param {string} groupHash - event's group hash * @returns {Promise} */ - async getChartDataFromMongo(days, timezoneOffset = 0, groupHash = '') { + async getEventDailyChart(groupHash, days, timezoneOffset = 0) { return this.findChartData(days, timezoneOffset, groupHash); } diff --git a/src/redisHelper.ts b/src/redisHelper.ts index 0b1ef0c0..e193362e 100644 --- a/src/redisHelper.ts +++ b/src/redisHelper.ts @@ -116,95 +116,32 @@ export default class RedisHelper { public isConnected(): boolean { return this.redisClient.isOpen; } - - public async getChartDataFromRedis( - startDate: string, - endDate: string, - groupBy: number, // minutes: 1=minute, 60=hour, 1440=day - timezoneOffset = 0, - projectId = '', - groupHash = '' - ): Promise<{ timestamp: number; count: number }[]> { - // If Redis is not connected, throw error to fallback to MongoDB - if (!this.redisClient.isOpen) { - console.warn('[Redis] Client not connected, will fallback to MongoDB'); - throw new Error('Redis client not connected'); - } - - // Determine suffix based on groupBy - let suffix: string; - if (groupBy === 1) { - suffix = 'minutely'; - } else if (groupBy === 60) { - suffix = 'hourly'; - } else if (groupBy === 1440) { - suffix = 'daily'; - } else { - // For custom intervals, fallback to minutely with aggregation - suffix = 'minutely'; - } - - const key = groupHash - ? `ts:events:${groupHash}:${suffix}` - : projectId - ? `ts:events:${projectId}:${suffix}` - : `ts:events:${suffix}`; - - // Parse dates (support ISO string or Unix timestamp in seconds) - const start = typeof startDate === 'string' && startDate.includes('-') - ? new Date(startDate).getTime() - : Number(startDate) * 1000; - const end = typeof endDate === 'string' && endDate.includes('-') - ? new Date(endDate).getTime() - : Number(endDate) * 1000; - - const bucketMs = groupBy * 60 * 1000; - - let result: [string, string][] = []; - try { - // Use aggregation to sum values within each bucket - // TS.INCRBY creates one point per time period with accumulated count - result = (await this.redisClient.sendCommand([ - 'TS.RANGE', - key, - start.toString(), - end.toString(), - 'AGGREGATION', - 'sum', - bucketMs.toString(), - ])) as [string, string][] | []; - } catch (err: any) { - if (err.message.includes('TSDB: the key does not exist')) { - console.warn(`[Redis] Key ${key} does not exist, returning zeroed data`); - result = []; - } else { - throw err; - } - } - // Transform data from Redis - const dataPoints: { [ts: number]: number } = {}; - for (const [tsStr, valStr] of result) { - const tsMs = Number(tsStr); - dataPoints[tsMs] = Number(valStr) || 0; - } - - // Fill missing intervals with zeros - const filled: { timestamp: number; count: number }[] = []; - let current = start; - - // Round current to the nearest bucket boundary - current = Math.floor(current / bucketMs) * bucketMs; - - while (current <= end) { - const count = dataPoints[current] || 0; - filled.push({ - timestamp: Math.floor((current + timezoneOffset * 60 * 1000) / 1000), - count, - }); - current += bucketMs; - } - - return filled.sort((a, b) => a.timestamp - b.timestamp); + /** + * Execute TS.RANGE command with aggregation + * + * @param key - Redis TimeSeries key + * @param start - start timestamp in milliseconds + * @param end - end timestamp in milliseconds + * @param aggregationType - aggregation type (sum, avg, min, max, etc.) + * @param bucketMs - bucket size in milliseconds + * @returns Array of [timestamp, value] tuples + */ + public async tsRange( + key: string, + start: string, + end: string, + aggregationType: string, + bucketMs: string + ): Promise<[string, string][]> { + return (await this.redisClient.sendCommand([ + 'TS.RANGE', + key, + start, + end, + 'AGGREGATION', + aggregationType, + bucketMs, + ])) as [string, string][]; } } \ No newline at end of file diff --git a/src/resolvers/event.js b/src/resolvers/event.js index 0bc77083..b9f59ea3 100644 --- a/src/resolvers/event.js +++ b/src/resolvers/event.js @@ -86,7 +86,7 @@ module.exports = { async chartData({ projectId, groupHash }, { days, timezoneOffset }, context) { const factory = getEventsFactory(context, projectId); - return factory.getChartDataFromMongo(days, timezoneOffset, groupHash); + return factory.getEventDailyChart(groupHash, days, timezoneOffset); }, /** diff --git a/src/resolvers/project.js b/src/resolvers/project.js index 976bc4ba..f62e3503 100644 --- a/src/resolvers/project.js +++ b/src/resolvers/project.js @@ -486,7 +486,7 @@ module.exports = { async chartData(project, { startDate, endDate, groupBy, timezoneOffset }, context) { const factory = getEventsFactory(context, project._id); - return factory.getChartData(startDate, endDate, groupBy, timezoneOffset, project._id); + return factory.getProjectChartData(project._id, startDate, endDate, groupBy, timezoneOffset); }, /** diff --git a/src/services/chartDataService.ts b/src/services/chartDataService.ts new file mode 100644 index 00000000..5e874415 --- /dev/null +++ b/src/services/chartDataService.ts @@ -0,0 +1,92 @@ +import RedisHelper from '../redisHelper'; +import { composeTimeSeriesKey, getTimeSeriesSuffix } from '../utils/redisKeys'; + +/** + * Service for fetching chart data from Redis TimeSeries + */ +export default class ChartDataService { + private redisHelper: RedisHelper; + + constructor(redisHelper: RedisHelper) { + this.redisHelper = redisHelper; + } + + /** + * Get project chart data from Redis TimeSeries + * + * @param projectId - project ID + * @param startDate - start date as ISO string (e.g., '2025-01-01T00:00:00Z') + * @param endDate - end date as ISO string (e.g., '2025-01-31T23:59:59Z') + * @param groupBy - grouping interval in minutes (1=minute, 60=hour, 1440=day) + * @param timezoneOffset - user's local timezone offset in minutes (default: 0) + * @returns Array of data points with timestamp and count + * @throws Error if Redis is not connected (caller should fallback to MongoDB) + */ + public async getProjectChartData( + projectId: string, + startDate: string, + endDate: string, + groupBy: number, + timezoneOffset = 0 + ): Promise<{ timestamp: number; count: number }[]> { + // Check if Redis is connected + if (!this.redisHelper.isConnected()) { + console.warn('[ChartDataService] Redis not connected, will fallback to MongoDB'); + throw new Error('Redis client not connected'); + } + + // Determine suffix and compose key + const suffix = getTimeSeriesSuffix(groupBy); + const key = composeTimeSeriesKey(suffix, projectId); + + // Parse ISO date strings to milliseconds + const start = new Date(startDate).getTime(); + const end = new Date(endDate).getTime(); + const bucketMs = groupBy * 60 * 1000; + + // Fetch data from Redis + let result: [string, string][] = []; + try { + result = await this.redisHelper.tsRange( + key, + start.toString(), + end.toString(), + 'sum', + bucketMs.toString() + ); + } catch (err: any) { + if (err.message.includes('TSDB: the key does not exist')) { + console.warn(`[ChartDataService] Key ${key} does not exist, returning zeroed data`); + result = []; + } else { + throw err; + } + } + + // Transform data from Redis + const dataPoints: { [ts: number]: number } = {}; + for (const [tsStr, valStr] of result) { + const tsMs = Number(tsStr); + dataPoints[tsMs] = Number(valStr) || 0; + } + + // Fill missing intervals with zeros + const filled: { timestamp: number; count: number }[] = []; + let current = start; + + // Round current to the nearest bucket boundary + current = Math.floor(current / bucketMs) * bucketMs; + + while (current <= end) { + const count = dataPoints[current] || 0; + filled.push({ + timestamp: Math.floor((current + timezoneOffset * 60 * 1000) / 1000), + count, + }); + current += bucketMs; + } + + return filled.sort((a, b) => a.timestamp - b.timestamp); + } +} + diff --git a/src/typeDefs/project.ts b/src/typeDefs/project.ts index 00c626cb..f5ce891e 100644 --- a/src/typeDefs/project.ts +++ b/src/typeDefs/project.ts @@ -354,14 +354,14 @@ type Project { """ chartData( """ - Start date (ISO string or Unix timestamp in seconds) + Start date (ISO 8601 DateTime string) """ - startDate: String! + startDate: DateTime! """ - End date (ISO string or Unix timestamp in seconds) + End date (ISO 8601 DateTime string) """ - endDate: String! + endDate: DateTime! """ Grouping interval in minutes (1=minute, 60=hour, 1440=day) diff --git a/src/utils/redisKeys.ts b/src/utils/redisKeys.ts new file mode 100644 index 00000000..42423065 --- /dev/null +++ b/src/utils/redisKeys.ts @@ -0,0 +1,59 @@ +/** + * Utility for composing Redis TimeSeries keys + */ + +/** + * Compose Redis TimeSeries key for project metrics + * + * @param suffix - time granularity suffix (minutely, hourly, daily) + * @param projectId - project ID + * @returns Redis key string + * + * @example + * composeTimeSeriesKey('hourly', '123abc') + * // => 'ts:events:123abc:hourly' + */ +export function composeTimeSeriesKey( + suffix: string, + projectId: string +): string { + return `ts:events:${projectId}:${suffix}`; +} + +/** + * Compose Redis TimeSeries key for event-level metrics + * + * @param suffix - time granularity suffix (minutely, hourly, daily) + * @param groupHash - event group hash + * @returns Redis key string + * + * @example + * composeEventTimeSeriesKey('daily', 'abc123def') + * // => 'ts:events:abc123def:daily' + */ +export function composeEventTimeSeriesKey( + suffix: string, + groupHash: string +): string { + return `ts:events:${groupHash}:${suffix}`; +} + +/** + * Get time granularity suffix based on groupBy interval + * + * @param groupBy - grouping interval in minutes (1=minute, 60=hour, 1440=day) + * @returns suffix string (minutely, hourly, daily) + */ +export function getTimeSeriesSuffix(groupBy: number): string { + if (groupBy === 1) { + return 'minutely'; + } else if (groupBy === 60) { + return 'hourly'; + } else if (groupBy === 1440) { + return 'daily'; + } + + // For custom intervals, fallback to minutely with aggregation + return 'minutely'; +} + From 067bd6a681763a45f0fae42809ec713f4b9b156c Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Sat, 8 Nov 2025 01:50:03 +0300 Subject: [PATCH 10/29] refactor: Apply PR #576 review comments and improve architecture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes based on @neSpecc review: 1. TypeDefs improvements: - Use DateTime! instead of String! for date parameters in Project.chartData 2. Method renaming for clarity: - eventsFactory: getChartData → getProjectChartData - eventsFactory: getChartDataFromMongo → getEventDailyChart - Improved parameter order (main identifier first) 3. Code quality: - Remove duplication in try-catch blocks - Extract Redis key composition to utility functions 4. Architecture improvements: - Create ChartDataService to separate business logic from Redis client - Simplify RedisHelper to only low-level Redis operations - Add tsRange() method for TS.RANGE commands - Move chart data logic from redisHelper to dedicated service 5. New utilities: - src/utils/redisKeys.ts with composeTimeSeriesKey(), composeEventTimeSeriesKey() - Type-safe key composition with required parameters Benefits: - Clear separation of concerns (Data Access vs Business Logic vs Service) - RedisHelper stays focused and doesn't grow - Easier to test and maintain - Better type safety and documentation --- src/models/eventsFactory.js | 62 +++++++++-------- src/redisHelper.ts | 115 +++++++------------------------ src/resolvers/event.js | 2 +- src/resolvers/project.js | 2 +- src/services/chartDataService.ts | 92 +++++++++++++++++++++++++ src/typeDefs/project.ts | 8 +-- src/utils/redisKeys.ts | 59 ++++++++++++++++ 7 files changed, 216 insertions(+), 124 deletions(-) create mode 100644 src/services/chartDataService.ts create mode 100644 src/utils/redisKeys.ts diff --git a/src/models/eventsFactory.js b/src/models/eventsFactory.js index d21bb0bb..e3eb32fd 100644 --- a/src/models/eventsFactory.js +++ b/src/models/eventsFactory.js @@ -8,6 +8,7 @@ const mongo = require('../mongo'); const Event = require('../models/event'); const { ObjectID } = require('mongodb'); import RedisHelper from '../redisHelper'; +import ChartDataService from '../services/chartDataService'; const { composeEventPayloadByRepetition } = require('../utils/merge'); const MAX_DB_READ_BATCH_SIZE = Number(process.env.MAX_DB_READ_BATCH_SIZE); @@ -71,11 +72,6 @@ const MAX_DB_READ_BATCH_SIZE = Number(process.env.MAX_DB_READ_BATCH_SIZE); * Factory Class for Event's Model */ class EventsFactory extends Factory { - /** - * Redis helper instance for modifying data through redis (singleton) - */ - redis = RedisHelper.getInstance(); - /** * Event types with collections where they stored * @return {{EVENTS: string, DAILY_EVENTS: string, REPETITIONS: string, RELEASES: string}} @@ -97,6 +93,16 @@ class EventsFactory extends Factory { constructor(projectId) { super(); + /** + * Redis helper instance (singleton) + */ + this.redis = RedisHelper.getInstance(); + + /** + * Chart data service for fetching data from Redis TimeSeries + */ + this.chartDataService = new ChartDataService(this.redis); + if (!projectId) { throw new Error('Can not construct Event model, because projectId is not provided'); } @@ -424,54 +430,52 @@ class EventsFactory extends Factory { } /** - * Get chart data for projects (uses Redis with fallback to MongoDB) + * Get project chart data from Redis or fallback to MongoDB * - * @param {string} startDate - start date (ISO string or Unix timestamp) - * @param {string} endDate - end date (ISO string or Unix timestamp) - * @param {number} groupBy - grouping interval in minutes - * @param {number} timezoneOffset - user's local timezone offset in minutes * @param {string} projectId - project ID - * @param {string} groupHash - event's group hash (empty for project-level data) + * @param {string} startDate - start date (ISO string) + * @param {string} endDate - end date (ISO string) + * @param {number} groupBy - grouping interval in minutes (1=minute, 60=hour, 1440=day) + * @param {number} timezoneOffset - user's local timezone offset in minutes * @returns {Promise} */ - async getChartData(startDate, endDate, groupBy = 60, timezoneOffset = 0, projectId = '', groupHash = '') { + async getProjectChartData(projectId, startDate, endDate, groupBy = 60, timezoneOffset = 0) { + // Calculate days for MongoDB fallback + const start = new Date(startDate).getTime(); + const end = new Date(endDate).getTime(); + const days = Math.ceil((end - start) / (24 * 60 * 60 * 1000)); + try { - const redisData = await this.redis.getChartDataFromRedis( + const redisData = await this.chartDataService.getProjectChartData( + projectId, startDate, endDate, groupBy, - timezoneOffset, - projectId, - groupHash + timezoneOffset ); if (redisData && redisData.length > 0) { return redisData; } - // Fallback to Mongo - const start = new Date(startDate).getTime(); - const end = new Date(endDate).getTime(); - const days = Math.ceil((end - start) / (24 * 60 * 60 * 1000)); - return this.findChartData(days, timezoneOffset, groupHash); + // Fallback to Mongo (empty groupHash for project-level data) + return this.findChartData(days, timezoneOffset, ''); } catch (err) { - console.error('[EventsFactory] getChartData error:', err); - const start = new Date(startDate).getTime(); - const end = new Date(endDate).getTime(); - const days = Math.ceil((end - start) / (24 * 60 * 60 * 1000)); - return this.findChartData(days, timezoneOffset, groupHash); + console.error('[EventsFactory] getProjectChartData error:', err); + // Fallback to Mongo on error (empty groupHash for project-level data) + return this.findChartData(days, timezoneOffset, ''); } } /** - * Get chart data from MongoDB only (for events) + * Get event daily chart data from MongoDB only * + * @param {string} groupHash - event's group hash * @param {number} days - how many days to fetch * @param {number} timezoneOffset - user's local timezone offset in minutes - * @param {string} groupHash - event's group hash * @returns {Promise} */ - async getChartDataFromMongo(days, timezoneOffset = 0, groupHash = '') { + async getEventDailyChart(groupHash, days, timezoneOffset = 0) { return this.findChartData(days, timezoneOffset, groupHash); } diff --git a/src/redisHelper.ts b/src/redisHelper.ts index 0b1ef0c0..e193362e 100644 --- a/src/redisHelper.ts +++ b/src/redisHelper.ts @@ -116,95 +116,32 @@ export default class RedisHelper { public isConnected(): boolean { return this.redisClient.isOpen; } - - public async getChartDataFromRedis( - startDate: string, - endDate: string, - groupBy: number, // minutes: 1=minute, 60=hour, 1440=day - timezoneOffset = 0, - projectId = '', - groupHash = '' - ): Promise<{ timestamp: number; count: number }[]> { - // If Redis is not connected, throw error to fallback to MongoDB - if (!this.redisClient.isOpen) { - console.warn('[Redis] Client not connected, will fallback to MongoDB'); - throw new Error('Redis client not connected'); - } - - // Determine suffix based on groupBy - let suffix: string; - if (groupBy === 1) { - suffix = 'minutely'; - } else if (groupBy === 60) { - suffix = 'hourly'; - } else if (groupBy === 1440) { - suffix = 'daily'; - } else { - // For custom intervals, fallback to minutely with aggregation - suffix = 'minutely'; - } - - const key = groupHash - ? `ts:events:${groupHash}:${suffix}` - : projectId - ? `ts:events:${projectId}:${suffix}` - : `ts:events:${suffix}`; - - // Parse dates (support ISO string or Unix timestamp in seconds) - const start = typeof startDate === 'string' && startDate.includes('-') - ? new Date(startDate).getTime() - : Number(startDate) * 1000; - const end = typeof endDate === 'string' && endDate.includes('-') - ? new Date(endDate).getTime() - : Number(endDate) * 1000; - - const bucketMs = groupBy * 60 * 1000; - - let result: [string, string][] = []; - try { - // Use aggregation to sum values within each bucket - // TS.INCRBY creates one point per time period with accumulated count - result = (await this.redisClient.sendCommand([ - 'TS.RANGE', - key, - start.toString(), - end.toString(), - 'AGGREGATION', - 'sum', - bucketMs.toString(), - ])) as [string, string][] | []; - } catch (err: any) { - if (err.message.includes('TSDB: the key does not exist')) { - console.warn(`[Redis] Key ${key} does not exist, returning zeroed data`); - result = []; - } else { - throw err; - } - } - // Transform data from Redis - const dataPoints: { [ts: number]: number } = {}; - for (const [tsStr, valStr] of result) { - const tsMs = Number(tsStr); - dataPoints[tsMs] = Number(valStr) || 0; - } - - // Fill missing intervals with zeros - const filled: { timestamp: number; count: number }[] = []; - let current = start; - - // Round current to the nearest bucket boundary - current = Math.floor(current / bucketMs) * bucketMs; - - while (current <= end) { - const count = dataPoints[current] || 0; - filled.push({ - timestamp: Math.floor((current + timezoneOffset * 60 * 1000) / 1000), - count, - }); - current += bucketMs; - } - - return filled.sort((a, b) => a.timestamp - b.timestamp); + /** + * Execute TS.RANGE command with aggregation + * + * @param key - Redis TimeSeries key + * @param start - start timestamp in milliseconds + * @param end - end timestamp in milliseconds + * @param aggregationType - aggregation type (sum, avg, min, max, etc.) + * @param bucketMs - bucket size in milliseconds + * @returns Array of [timestamp, value] tuples + */ + public async tsRange( + key: string, + start: string, + end: string, + aggregationType: string, + bucketMs: string + ): Promise<[string, string][]> { + return (await this.redisClient.sendCommand([ + 'TS.RANGE', + key, + start, + end, + 'AGGREGATION', + aggregationType, + bucketMs, + ])) as [string, string][]; } } \ No newline at end of file diff --git a/src/resolvers/event.js b/src/resolvers/event.js index 0bc77083..b9f59ea3 100644 --- a/src/resolvers/event.js +++ b/src/resolvers/event.js @@ -86,7 +86,7 @@ module.exports = { async chartData({ projectId, groupHash }, { days, timezoneOffset }, context) { const factory = getEventsFactory(context, projectId); - return factory.getChartDataFromMongo(days, timezoneOffset, groupHash); + return factory.getEventDailyChart(groupHash, days, timezoneOffset); }, /** diff --git a/src/resolvers/project.js b/src/resolvers/project.js index 976bc4ba..f62e3503 100644 --- a/src/resolvers/project.js +++ b/src/resolvers/project.js @@ -486,7 +486,7 @@ module.exports = { async chartData(project, { startDate, endDate, groupBy, timezoneOffset }, context) { const factory = getEventsFactory(context, project._id); - return factory.getChartData(startDate, endDate, groupBy, timezoneOffset, project._id); + return factory.getProjectChartData(project._id, startDate, endDate, groupBy, timezoneOffset); }, /** diff --git a/src/services/chartDataService.ts b/src/services/chartDataService.ts new file mode 100644 index 00000000..5e874415 --- /dev/null +++ b/src/services/chartDataService.ts @@ -0,0 +1,92 @@ +import RedisHelper from '../redisHelper'; +import { composeTimeSeriesKey, getTimeSeriesSuffix } from '../utils/redisKeys'; + +/** + * Service for fetching chart data from Redis TimeSeries + */ +export default class ChartDataService { + private redisHelper: RedisHelper; + + constructor(redisHelper: RedisHelper) { + this.redisHelper = redisHelper; + } + + /** + * Get project chart data from Redis TimeSeries + * + * @param projectId - project ID + * @param startDate - start date as ISO string (e.g., '2025-01-01T00:00:00Z') + * @param endDate - end date as ISO string (e.g., '2025-01-31T23:59:59Z') + * @param groupBy - grouping interval in minutes (1=minute, 60=hour, 1440=day) + * @param timezoneOffset - user's local timezone offset in minutes (default: 0) + * @returns Array of data points with timestamp and count + * @throws Error if Redis is not connected (caller should fallback to MongoDB) + */ + public async getProjectChartData( + projectId: string, + startDate: string, + endDate: string, + groupBy: number, + timezoneOffset = 0 + ): Promise<{ timestamp: number; count: number }[]> { + // Check if Redis is connected + if (!this.redisHelper.isConnected()) { + console.warn('[ChartDataService] Redis not connected, will fallback to MongoDB'); + throw new Error('Redis client not connected'); + } + + // Determine suffix and compose key + const suffix = getTimeSeriesSuffix(groupBy); + const key = composeTimeSeriesKey(suffix, projectId); + + // Parse ISO date strings to milliseconds + const start = new Date(startDate).getTime(); + const end = new Date(endDate).getTime(); + const bucketMs = groupBy * 60 * 1000; + + // Fetch data from Redis + let result: [string, string][] = []; + try { + result = await this.redisHelper.tsRange( + key, + start.toString(), + end.toString(), + 'sum', + bucketMs.toString() + ); + } catch (err: any) { + if (err.message.includes('TSDB: the key does not exist')) { + console.warn(`[ChartDataService] Key ${key} does not exist, returning zeroed data`); + result = []; + } else { + throw err; + } + } + + // Transform data from Redis + const dataPoints: { [ts: number]: number } = {}; + for (const [tsStr, valStr] of result) { + const tsMs = Number(tsStr); + dataPoints[tsMs] = Number(valStr) || 0; + } + + // Fill missing intervals with zeros + const filled: { timestamp: number; count: number }[] = []; + let current = start; + + // Round current to the nearest bucket boundary + current = Math.floor(current / bucketMs) * bucketMs; + + while (current <= end) { + const count = dataPoints[current] || 0; + filled.push({ + timestamp: Math.floor((current + timezoneOffset * 60 * 1000) / 1000), + count, + }); + current += bucketMs; + } + + return filled.sort((a, b) => a.timestamp - b.timestamp); + } +} + diff --git a/src/typeDefs/project.ts b/src/typeDefs/project.ts index 00c626cb..f5ce891e 100644 --- a/src/typeDefs/project.ts +++ b/src/typeDefs/project.ts @@ -354,14 +354,14 @@ type Project { """ chartData( """ - Start date (ISO string or Unix timestamp in seconds) + Start date (ISO 8601 DateTime string) """ - startDate: String! + startDate: DateTime! """ - End date (ISO string or Unix timestamp in seconds) + End date (ISO 8601 DateTime string) """ - endDate: String! + endDate: DateTime! """ Grouping interval in minutes (1=minute, 60=hour, 1440=day) diff --git a/src/utils/redisKeys.ts b/src/utils/redisKeys.ts new file mode 100644 index 00000000..42423065 --- /dev/null +++ b/src/utils/redisKeys.ts @@ -0,0 +1,59 @@ +/** + * Utility for composing Redis TimeSeries keys + */ + +/** + * Compose Redis TimeSeries key for project metrics + * + * @param suffix - time granularity suffix (minutely, hourly, daily) + * @param projectId - project ID + * @returns Redis key string + * + * @example + * composeTimeSeriesKey('hourly', '123abc') + * // => 'ts:events:123abc:hourly' + */ +export function composeTimeSeriesKey( + suffix: string, + projectId: string +): string { + return `ts:events:${projectId}:${suffix}`; +} + +/** + * Compose Redis TimeSeries key for event-level metrics + * + * @param suffix - time granularity suffix (minutely, hourly, daily) + * @param groupHash - event group hash + * @returns Redis key string + * + * @example + * composeEventTimeSeriesKey('daily', 'abc123def') + * // => 'ts:events:abc123def:daily' + */ +export function composeEventTimeSeriesKey( + suffix: string, + groupHash: string +): string { + return `ts:events:${groupHash}:${suffix}`; +} + +/** + * Get time granularity suffix based on groupBy interval + * + * @param groupBy - grouping interval in minutes (1=minute, 60=hour, 1440=day) + * @returns suffix string (minutely, hourly, daily) + */ +export function getTimeSeriesSuffix(groupBy: number): string { + if (groupBy === 1) { + return 'minutely'; + } else if (groupBy === 60) { + return 'hourly'; + } else if (groupBy === 1440) { + return 'daily'; + } + + // For custom intervals, fallback to minutely with aggregation + return 'minutely'; +} + From 1fe3d012cac2cee5c0a6a9c16577ef1a16d49f59 Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Sat, 8 Nov 2025 02:07:07 +0300 Subject: [PATCH 11/29] Update redisKeys.ts --- src/utils/redisKeys.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/utils/redisKeys.ts b/src/utils/redisKeys.ts index 42423065..a2eddf38 100644 --- a/src/utils/redisKeys.ts +++ b/src/utils/redisKeys.ts @@ -7,17 +7,23 @@ * * @param suffix - time granularity suffix (minutely, hourly, daily) * @param projectId - project ID + * @param metricType - metric type (default: 'events-accepted') * @returns Redis key string * * @example * composeTimeSeriesKey('hourly', '123abc') - * // => 'ts:events:123abc:hourly' + * // => 'ts:project-events-accepted:123abc:hourly' + * + * @example + * composeTimeSeriesKey('daily', '123abc', 'events-rate-limited') + * // => 'ts:project-events-rate-limited:123abc:daily' */ export function composeTimeSeriesKey( suffix: string, - projectId: string + projectId: string, + metricType = 'events-accepted' ): string { - return `ts:events:${projectId}:${suffix}`; + return `ts:project-${metricType}:${projectId}:${suffix}`; } /** From 476378d525c4b6deb79cdc505a734343bd37fb5a Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Sat, 8 Nov 2025 02:44:25 +0300 Subject: [PATCH 12/29] refactor: Rename redisKeys.ts to chartStorageKeys.ts Better name that reflects the purpose - composing keys for chart data storage, not just Redis-specific utilities. --- src/services/chartDataService.ts | 2 +- src/utils/{redisKeys.ts => chartStorageKeys.ts} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/utils/{redisKeys.ts => chartStorageKeys.ts} (100%) diff --git a/src/services/chartDataService.ts b/src/services/chartDataService.ts index 5e874415..f6203d86 100644 --- a/src/services/chartDataService.ts +++ b/src/services/chartDataService.ts @@ -1,5 +1,5 @@ import RedisHelper from '../redisHelper'; -import { composeTimeSeriesKey, getTimeSeriesSuffix } from '../utils/redisKeys'; +import { composeTimeSeriesKey, getTimeSeriesSuffix } from '../utils/chartStorageKeys'; /** * Service for fetching chart data from Redis TimeSeries diff --git a/src/utils/redisKeys.ts b/src/utils/chartStorageKeys.ts similarity index 100% rename from src/utils/redisKeys.ts rename to src/utils/chartStorageKeys.ts From e0bf8be549c239f58a08acdeae00d8d13071cfb8 Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Sat, 8 Nov 2025 02:45:28 +0300 Subject: [PATCH 13/29] refactor: Rename composeTimeSeriesKey to composeProjectMetricsKey Clearer naming that explicitly indicates: - This is for project-level metrics (not event-level) - Parameter renamed: suffix -> granularity (more descriptive) Addresses PR review comment about unclear naming. --- src/services/chartDataService.ts | 8 ++++---- src/utils/chartStorageKeys.ts | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/services/chartDataService.ts b/src/services/chartDataService.ts index f6203d86..0a265ac3 100644 --- a/src/services/chartDataService.ts +++ b/src/services/chartDataService.ts @@ -1,5 +1,5 @@ import RedisHelper from '../redisHelper'; -import { composeTimeSeriesKey, getTimeSeriesSuffix } from '../utils/chartStorageKeys'; +import { composeProjectMetricsKey, getTimeSeriesSuffix } from '../utils/chartStorageKeys'; /** * Service for fetching chart data from Redis TimeSeries @@ -35,9 +35,9 @@ export default class ChartDataService { throw new Error('Redis client not connected'); } - // Determine suffix and compose key - const suffix = getTimeSeriesSuffix(groupBy); - const key = composeTimeSeriesKey(suffix, projectId); + // Determine granularity and compose key + const granularity = getTimeSeriesSuffix(groupBy); + const key = composeProjectMetricsKey(granularity, projectId); // Parse ISO date strings to milliseconds const start = new Date(startDate).getTime(); diff --git a/src/utils/chartStorageKeys.ts b/src/utils/chartStorageKeys.ts index a2eddf38..f787e768 100644 --- a/src/utils/chartStorageKeys.ts +++ b/src/utils/chartStorageKeys.ts @@ -3,27 +3,27 @@ */ /** - * Compose Redis TimeSeries key for project metrics + * Compose Redis TimeSeries key for project-level metrics * - * @param suffix - time granularity suffix (minutely, hourly, daily) + * @param granularity - time granularity (minutely, hourly, daily) * @param projectId - project ID * @param metricType - metric type (default: 'events-accepted') - * @returns Redis key string + * @returns Redis key string in format: ts:project-{metricType}:{projectId}:{granularity} * * @example - * composeTimeSeriesKey('hourly', '123abc') + * composeProjectMetricsKey('hourly', '123abc') * // => 'ts:project-events-accepted:123abc:hourly' * * @example - * composeTimeSeriesKey('daily', '123abc', 'events-rate-limited') + * composeProjectMetricsKey('daily', '123abc', 'events-rate-limited') * // => 'ts:project-events-rate-limited:123abc:daily' */ -export function composeTimeSeriesKey( - suffix: string, +export function composeProjectMetricsKey( + granularity: string, projectId: string, metricType = 'events-accepted' ): string { - return `ts:project-${metricType}:${projectId}:${suffix}`; + return `ts:project-${metricType}:${projectId}:${granularity}`; } /** From 2115a1d6caa194195f64f3cc4153ce09b9c269f3 Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Sat, 8 Nov 2025 03:27:59 +0300 Subject: [PATCH 14/29] Update chartStorageKeys.ts --- src/utils/chartStorageKeys.ts | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/src/utils/chartStorageKeys.ts b/src/utils/chartStorageKeys.ts index f787e768..46699e37 100644 --- a/src/utils/chartStorageKeys.ts +++ b/src/utils/chartStorageKeys.ts @@ -1,5 +1,8 @@ /** - * Utility for composing Redis TimeSeries keys + * Utilities for composing Redis TimeSeries keys for chart data storage + * + * Note: Event-level metrics use MongoDB only (not Redis TimeSeries) + * Only project-level metrics are stored in Redis TimeSeries */ /** @@ -26,24 +29,6 @@ export function composeProjectMetricsKey( return `ts:project-${metricType}:${projectId}:${granularity}`; } -/** - * Compose Redis TimeSeries key for event-level metrics - * - * @param suffix - time granularity suffix (minutely, hourly, daily) - * @param groupHash - event group hash - * @returns Redis key string - * - * @example - * composeEventTimeSeriesKey('daily', 'abc123def') - * // => 'ts:events:abc123def:daily' - */ -export function composeEventTimeSeriesKey( - suffix: string, - groupHash: string -): string { - return `ts:events:${groupHash}:${suffix}`; -} - /** * Get time granularity suffix based on groupBy interval * From e8bed5cf4900e04f7c31e161f4bc66738e1a8ed6 Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Sat, 8 Nov 2025 03:34:38 +0300 Subject: [PATCH 15/29] Update eventsFactory.js --- src/models/eventsFactory.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/models/eventsFactory.js b/src/models/eventsFactory.js index e3eb32fd..c3817ec2 100644 --- a/src/models/eventsFactory.js +++ b/src/models/eventsFactory.js @@ -107,8 +107,6 @@ class EventsFactory extends Factory { throw new Error('Can not construct Event model, because projectId is not provided'); } - this.redis.initialize(); - this.projectId = projectId; this.eventsDataLoader = createProjectEventsByIdLoader(mongo.databases.events, this.projectId); } From ba0b275e44aa7100eb0a35d2f0138e3ad1820ce8 Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Sat, 8 Nov 2025 03:51:55 +0300 Subject: [PATCH 16/29] linter --- src/index.ts | 21 ++++++++++----------- src/models/eventsFactory.js | 5 ++--- src/redisHelper.ts | 1 - src/utils/chartStorageKeys.ts | 7 +++---- 4 files changed, 15 insertions(+), 19 deletions(-) diff --git a/src/index.ts b/src/index.ts index a6b77506..f386ead6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -149,6 +149,7 @@ class HawkAPI { /** * Creates factories to work with models * @param dataLoaders - dataLoaders for fetching data form database + * @returns factories object */ private static setupFactories(dataLoaders: DataLoaders): ContextFactories { // eslint-disable-next-line @typescript-eslint/no-non-null-assertion @@ -213,21 +214,18 @@ class HawkAPI { /** * Initializing accounting SDK - */ - let tlsVerify; - - /** * Checking env variables * If at least one path is not transmitted, the variable tlsVerify is undefined */ if ( ![process.env.TLS_CA_CERT, process.env.TLS_CERT, process.env.TLS_KEY].some(value => value === undefined || value.length === 0) ) { - tlsVerify = { - tlsCaCertPath: `${process.env.TLS_CA_CERT}`, - tlsCertPath: `${process.env.TLS_CERT}`, - tlsKeyPath: `${process.env.TLS_KEY}`, - }; + // tlsVerify is used for accounting SDK (currently commented out) + // const tlsVerify = { + // tlsCaCertPath: `${process.env.TLS_CA_CERT}`, + // tlsCertPath: `${process.env.TLS_CERT}`, + // tlsKeyPath: `${process.env.TLS_KEY}`, + // }; } /* @@ -253,11 +251,12 @@ class HawkAPI { public async start(): Promise { await mongo.setupConnections(); await rabbitmq.setupConnections(); - + // Initialize Redis singleton with auto-reconnect const redis = RedisHelper.getInstance(); + await redis.initialize(); - + await this.server.start(); this.app.use(graphqlUploadExpress()); this.server.applyMiddleware({ app: this.app }); diff --git a/src/models/eventsFactory.js b/src/models/eventsFactory.js index c3817ec2..b1c2d414 100644 --- a/src/models/eventsFactory.js +++ b/src/models/eventsFactory.js @@ -1,14 +1,13 @@ import { getMidnightWithTimezoneOffset, getUTCMidnight } from '../utils/dates'; import safe from 'safe-regex'; import { createProjectEventsByIdLoader } from '../dataLoaders'; -import { Effect, sgr } from '../utils/ansi'; +import RedisHelper from '../redisHelper'; +import ChartDataService from '../services/chartDataService'; const Factory = require('./modelFactory'); const mongo = require('../mongo'); const Event = require('../models/event'); const { ObjectID } = require('mongodb'); -import RedisHelper from '../redisHelper'; -import ChartDataService from '../services/chartDataService'; const { composeEventPayloadByRepetition } = require('../utils/merge'); const MAX_DB_READ_BATCH_SIZE = Number(process.env.MAX_DB_READ_BATCH_SIZE); diff --git a/src/redisHelper.ts b/src/redisHelper.ts index e193362e..d69fa5c1 100644 --- a/src/redisHelper.ts +++ b/src/redisHelper.ts @@ -1,6 +1,5 @@ import HawkCatcher from '@hawk.so/nodejs'; import { createClient, RedisClientType } from 'redis'; -import { Effect, sgr } from './utils/ansi'; /** * Helper class for working with Redis diff --git a/src/utils/chartStorageKeys.ts b/src/utils/chartStorageKeys.ts index 46699e37..78a049e4 100644 --- a/src/utils/chartStorageKeys.ts +++ b/src/utils/chartStorageKeys.ts @@ -1,6 +1,6 @@ /** * Utilities for composing Redis TimeSeries keys for chart data storage - * + * * Note: Event-level metrics use MongoDB only (not Redis TimeSeries) * Only project-level metrics are stored in Redis TimeSeries */ @@ -43,8 +43,7 @@ export function getTimeSeriesSuffix(groupBy: number): string { } else if (groupBy === 1440) { return 'daily'; } - + // For custom intervals, fallback to minutely with aggregation return 'minutely'; -} - +} \ No newline at end of file From 5abce3cacbd7c209030b96ce595f56ebb82766c8 Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Sat, 8 Nov 2025 03:53:49 +0300 Subject: [PATCH 17/29] Update index.ts --- src/index.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/index.ts b/src/index.ts index f386ead6..6bb3eb30 100644 --- a/src/index.ts +++ b/src/index.ts @@ -220,12 +220,14 @@ class HawkAPI { if ( ![process.env.TLS_CA_CERT, process.env.TLS_CERT, process.env.TLS_KEY].some(value => value === undefined || value.length === 0) ) { - // tlsVerify is used for accounting SDK (currently commented out) - // const tlsVerify = { - // tlsCaCertPath: `${process.env.TLS_CA_CERT}`, - // tlsCertPath: `${process.env.TLS_CERT}`, - // tlsKeyPath: `${process.env.TLS_KEY}`, - // }; + /* + * tlsVerify is used for accounting SDK (currently commented out) + * const tlsVerify = { + * tlsCaCertPath: `${process.env.TLS_CA_CERT}`, + * tlsCertPath: `${process.env.TLS_CERT}`, + * tlsKeyPath: `${process.env.TLS_KEY}`, + * }; + */ } /* From 92761560ddc82ad75aa3838c1c0547ad40c78976 Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Sat, 8 Nov 2025 04:00:55 +0300 Subject: [PATCH 18/29] Update eventsFactory.js --- src/models/eventsFactory.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/models/eventsFactory.js b/src/models/eventsFactory.js index b1c2d414..c4a631bf 100644 --- a/src/models/eventsFactory.js +++ b/src/models/eventsFactory.js @@ -459,6 +459,7 @@ class EventsFactory extends Factory { return this.findChartData(days, timezoneOffset, ''); } catch (err) { console.error('[EventsFactory] getProjectChartData error:', err); + // Fallback to Mongo on error (empty groupHash for project-level data) return this.findChartData(days, timezoneOffset, ''); } From ac754bfffe8d7fe0fca3409cd06545dfbefb4a23 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 8 Nov 2025 01:02:21 +0000 Subject: [PATCH 19/29] Bump version up to 1.2.21 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 984948d5..5397e66f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hawk.api", - "version": "1.2.20", + "version": "1.2.21", "main": "index.ts", "license": "BUSL-1.1", "scripts": { From 9a57f76bc708be97e1111bfa6a94594c1b158666 Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Sat, 8 Nov 2025 04:05:39 +0300 Subject: [PATCH 20/29] Update eventsFactory.js --- src/models/eventsFactory.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/models/eventsFactory.js b/src/models/eventsFactory.js index c4a631bf..9b27ee4b 100644 --- a/src/models/eventsFactory.js +++ b/src/models/eventsFactory.js @@ -87,7 +87,7 @@ class EventsFactory extends Factory { /** * Creates Event instance - * @param {ObjectId} projectId - project ID + * @param {ObjectId} projectId */ constructor(projectId) { super(); From db93f62f81345509b70a052c414f6be08a6ad3e2 Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Sat, 8 Nov 2025 04:17:11 +0300 Subject: [PATCH 21/29] Update api.env --- test/integration/api.env | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/integration/api.env b/test/integration/api.env index 69703097..a1f431d1 100644 --- a/test/integration/api.env +++ b/test/integration/api.env @@ -7,7 +7,9 @@ MONGO_HAWK_DB_URL=mongodb://mongodb:27017/hawk # Events database URL MONGO_EVENTS_DB_URL=mongodb://mongodb:27017/hawk_events - +# Redis URL (optional, for TimeSeries metrics) +# If not set, Redis features will be disabled gracefully +REDIS_URL=redis:6379 # MongoDB settings MONGO_RECONNECT_TRIES=60 From ad01c8ff6016885f2be44b6a3a64b139cf8f43fe Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Wed, 12 Nov 2025 22:01:47 +0300 Subject: [PATCH 22/29] fix eslint in files --- src/redisHelper.ts | 16 +++++++++++----- src/services/chartDataService.ts | 5 ++--- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/redisHelper.ts b/src/redisHelper.ts index d69fa5c1..9f6518c3 100644 --- a/src/redisHelper.ts +++ b/src/redisHelper.ts @@ -1,6 +1,10 @@ import HawkCatcher from '@hawk.so/nodejs'; import { createClient, RedisClientType } from 'redis'; +// eslint call error: 0:0 error Parsing error: Cannot read properties of undefined (reading 'map') +// export type TsRangeResult = [timestamp: string, value: string]; +export type TsRangeResult = any; + /** * Helper class for working with Redis */ @@ -35,8 +39,10 @@ export default class RedisHelper { url: process.env.REDIS_URL, socket: { reconnectStrategy: (retries) => { - // Exponential backoff: wait longer between each retry - // Max wait time: 30 seconds + /* + * Exponential backoff: wait longer between each retry + * Max wait time: 30 seconds + */ const delay = Math.min(retries * 1000, 30000); console.log(`[Redis] Reconnecting... attempt ${retries}, waiting ${delay}ms`); return delay; @@ -132,7 +138,7 @@ export default class RedisHelper { end: string, aggregationType: string, bucketMs: string - ): Promise<[string, string][]> { + ): Promise { return (await this.redisClient.sendCommand([ 'TS.RANGE', key, @@ -141,6 +147,6 @@ export default class RedisHelper { 'AGGREGATION', aggregationType, bucketMs, - ])) as [string, string][]; + ])) as TsRangeResult[]; } -} \ No newline at end of file +} diff --git a/src/services/chartDataService.ts b/src/services/chartDataService.ts index 0a265ac3..ff7dc21f 100644 --- a/src/services/chartDataService.ts +++ b/src/services/chartDataService.ts @@ -1,4 +1,4 @@ -import RedisHelper from '../redisHelper'; +import RedisHelper, { TsRangeResult } from '../redisHelper'; import { composeProjectMetricsKey, getTimeSeriesSuffix } from '../utils/chartStorageKeys'; /** @@ -45,7 +45,7 @@ export default class ChartDataService { const bucketMs = groupBy * 60 * 1000; // Fetch data from Redis - let result: [string, string][] = []; + let result: TsRangeResult[] = []; try { result = await this.redisHelper.tsRange( key, @@ -89,4 +89,3 @@ export default class ChartDataService { return filled.sort((a, b) => a.timestamp - b.timestamp); } } - From a4edc89b5d752b61e2b49dab64de3c01e70930ad Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Wed, 12 Nov 2025 22:03:47 +0300 Subject: [PATCH 23/29] update package.json: new version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5397e66f..cc5df70e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hawk.api", - "version": "1.2.21", + "version": "1.2.23", "main": "index.ts", "license": "BUSL-1.1", "scripts": { From 37cab1b225aca8878589f9da582ee9d74e007cb5 Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Wed, 12 Nov 2025 22:20:14 +0300 Subject: [PATCH 24/29] add redis-mock library and fix tests --- package.json | 1 + test/integration/jestEnv.js | 9 +++++ test/integration/redisMock.js | 72 +++++++++++++++++++++++++++++++++++ yarn.lock | 5 +++ 4 files changed, 87 insertions(+) create mode 100644 test/integration/redisMock.js diff --git a/package.json b/package.json index cc5df70e..e80054db 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "jest": "^26.2.2", "mongodb-memory-server": "^6.6.1", "nodemon": "^2.0.2", + "redis-mock": "^0.58.0", "ts-jest": "^26.1.4", "ts-node": "^10.9.1", "typescript": "^4.7.4" diff --git a/test/integration/jestEnv.js b/test/integration/jestEnv.js index cedfba31..0249f971 100644 --- a/test/integration/jestEnv.js +++ b/test/integration/jestEnv.js @@ -1,6 +1,7 @@ const NodeEnvironment = require('jest-environment-node'); const amqp = require('amqplib'); const mongodb = require('mongodb'); +const { installRedisMock, uninstallRedisMock } = require('./redisMock'); /** * Custom test environment for defining global connections @@ -19,6 +20,12 @@ class CustomEnvironment extends NodeEnvironment { await mongoClient.db('hawk').dropDatabase(); // await mongoClient.db('codex_accounting').dropDatabase(); + /** + * Use redis-mock instead of a real Redis connection. + * This avoids spinning up Redis during integration tests while keeping the API surface. + */ + this.global.redisClient = installRedisMock(); + this.rabbitMqConnection = await amqp.connect('amqp://guest:guest@rabbitmq:5672/'); this.global.rabbitChannel = await this.rabbitMqConnection.createChannel(); await this.global.rabbitChannel.purgeQueue('cron-tasks/limiter'); @@ -41,6 +48,8 @@ class CustomEnvironment extends NodeEnvironment { if (this.rabbitMqConnection) { await this.rabbitMqConnection.close(); } + + uninstallRedisMock(); } catch (error) { console.error('Error during teardown:', error); } diff --git a/test/integration/redisMock.js b/test/integration/redisMock.js new file mode 100644 index 00000000..1c98efcd --- /dev/null +++ b/test/integration/redisMock.js @@ -0,0 +1,72 @@ +const path = require('path'); +const redisMock = require('redis-mock'); + +let originalRedisModule = null; +let redisModulePath = null; + +/** + * Create Redis mock client compatible with node-redis v4 API portions we use. + * + * @returns {object} mocked redis client + */ +function createMockClient() { + const client = redisMock.createClient(); + + client.isOpen = true; + client.connect = async () => client; + client.quit = async () => undefined; + client.sendCommand = async () => []; + client.on = () => client; + + return client; +} + +/** + * Install redis-mock into Node's module cache so that `require('redis')` + * returns the mocked client factory. + * + * @returns {object} mock client instance to be reused in tests + */ +function installRedisMock() { + redisModulePath = require.resolve('redis'); + originalRedisModule = require.cache[redisModulePath] || null; + + const mockExports = { + createClient: () => createMockClient(), + }; + + require.cache[redisModulePath] = { + id: redisModulePath, + filename: redisModulePath, + loaded: true, + exports: mockExports, + path: path.dirname(redisModulePath), + children: [], + }; + + return mockExports.createClient(); +} + +/** + * Restore original `redis` module if it existed. + */ +function uninstallRedisMock() { + if (!redisModulePath) { + return; + } + + if (originalRedisModule) { + require.cache[redisModulePath] = originalRedisModule; + } else { + delete require.cache[redisModulePath]; + } + + originalRedisModule = null; + redisModulePath = null; +} + +module.exports = { + installRedisMock, + uninstallRedisMock, +}; + diff --git a/yarn.lock b/yarn.lock index e2fa2c6e..42de7f1f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5762,6 +5762,11 @@ readdirp@~3.6.0: dependencies: picomatch "^2.2.1" +redis-mock@^0.58.0: + version "0.56.3" + resolved "https://registry.yarnpkg.com/redis-mock/-/redis-mock-0.56.3.tgz#e96471bcc774ddc514c2fc49cdd03cab2baecd89" + integrity sha512-ynaJhqk0Qf3Qajnwvy4aOjS4Mdf9IBkELWtjd+NYhpiqu4QCNq6Vf3Q7c++XRPGiKiwRj9HWr0crcwy7EiPjYQ== + redis@^4.7.0: version "4.7.1" resolved "https://registry.yarnpkg.com/redis/-/redis-4.7.1.tgz#08588a30936be0e7ad9c0f3e1ac6a85ccaf73e94" From 82e0dc08dd6032169686def84b9ede9051aee475 Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Wed, 12 Nov 2025 22:23:09 +0300 Subject: [PATCH 25/29] change version for redis-mock --- package.json | 2 +- yarn.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index e80054db..1600018a 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "jest": "^26.2.2", "mongodb-memory-server": "^6.6.1", "nodemon": "^2.0.2", - "redis-mock": "^0.58.0", + "redis-mock": "^0.56.3", "ts-jest": "^26.1.4", "ts-node": "^10.9.1", "typescript": "^4.7.4" diff --git a/yarn.lock b/yarn.lock index 42de7f1f..44240cfb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5762,7 +5762,7 @@ readdirp@~3.6.0: dependencies: picomatch "^2.2.1" -redis-mock@^0.58.0: +redis-mock@^0.56.3: version "0.56.3" resolved "https://registry.yarnpkg.com/redis-mock/-/redis-mock-0.56.3.tgz#e96471bcc774ddc514c2fc49cdd03cab2baecd89" integrity sha512-ynaJhqk0Qf3Qajnwvy4aOjS4Mdf9IBkELWtjd+NYhpiqu4QCNq6Vf3Q7c++XRPGiKiwRj9HWr0crcwy7EiPjYQ== From ea05c7d60f79bfd316cb9eab1cef724a660e392f Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Wed, 12 Nov 2025 22:49:31 +0300 Subject: [PATCH 26/29] add redis in integration.test --- docker-compose.test.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docker-compose.test.yml b/docker-compose.test.yml index ac51dae6..a34222fe 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -52,6 +52,14 @@ services: timeout: 3s retries: 5 + redis: + image: redis:6.0.9 + ports: + - 6379:6379 + volumes: + - redis-data-test:/data + restart: unless-stopped + # accounting: # image: codexteamuser/codex-accounting:prod # env_file: From 2bfe2b81f2b5b6756014b69572b45355bd6da919 Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Wed, 12 Nov 2025 22:55:51 +0300 Subject: [PATCH 27/29] fix intergration.test --- docker-compose.test.yml | 8 -------- src/redisHelper.ts | 21 ++++++++++++++++++--- test/integration/api.env | 2 +- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/docker-compose.test.yml b/docker-compose.test.yml index a34222fe..ac51dae6 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -52,14 +52,6 @@ services: timeout: 3s retries: 5 - redis: - image: redis:6.0.9 - ports: - - 6379:6379 - volumes: - - redis-data-test:/data - restart: unless-stopped - # accounting: # image: codexteamuser/codex-accounting:prod # env_file: diff --git a/src/redisHelper.ts b/src/redisHelper.ts index 9f6518c3..ddc6e72e 100644 --- a/src/redisHelper.ts +++ b/src/redisHelper.ts @@ -22,7 +22,7 @@ export default class RedisHelper { /** * Redis client instance */ - private redisClient!: RedisClientType; + private redisClient: RedisClientType | null = null; /** * Flag to track if we're currently reconnecting @@ -34,6 +34,11 @@ export default class RedisHelper { * Initializes the Redis client and sets up error handling with auto-reconnect */ constructor() { + if (!process.env.REDIS_URL) { + console.warn('[Redis] REDIS_URL not set, Redis features will be disabled'); + return; + } + try { this.redisClient = createClient({ url: process.env.REDIS_URL, @@ -76,6 +81,7 @@ export default class RedisHelper { }); } catch (error) { console.error('[Redis] Error creating client:', error); + this.redisClient = null; } } @@ -93,6 +99,11 @@ export default class RedisHelper { * Connect to Redis */ public async initialize(): Promise { + if (!this.redisClient) { + console.warn('[Redis] Client not initialized, skipping connection'); + return; + } + try { if (!this.redisClient.isOpen && !this.isReconnecting) { await this.redisClient.connect(); @@ -109,7 +120,7 @@ export default class RedisHelper { * Close Redis client */ public async close(): Promise { - if (this.redisClient.isOpen) { + if (this.redisClient?.isOpen) { await this.redisClient.quit(); console.log('[Redis] Connection closed'); } @@ -119,7 +130,7 @@ export default class RedisHelper { * Check if Redis is connected */ public isConnected(): boolean { - return this.redisClient.isOpen; + return Boolean(this.redisClient?.isOpen); } /** @@ -139,6 +150,10 @@ export default class RedisHelper { aggregationType: string, bucketMs: string ): Promise { + if (!this.redisClient) { + throw new Error('Redis client not initialized'); + } + return (await this.redisClient.sendCommand([ 'TS.RANGE', key, diff --git a/test/integration/api.env b/test/integration/api.env index a1f431d1..8598b55f 100644 --- a/test/integration/api.env +++ b/test/integration/api.env @@ -9,7 +9,7 @@ MONGO_EVENTS_DB_URL=mongodb://mongodb:27017/hawk_events # Redis URL (optional, for TimeSeries metrics) # If not set, Redis features will be disabled gracefully -REDIS_URL=redis:6379 +REDIS_URL= # MongoDB settings MONGO_RECONNECT_TRIES=60 From cde06d266ef1e28e8d2b2dcd8e78b27056cccdee Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Thu, 13 Nov 2025 00:11:49 +0300 Subject: [PATCH 28/29] Update src/redisHelper.ts Co-authored-by: Vyacheslav Chernyshev <81693471+slaveeks@users.noreply.github.com> --- src/redisHelper.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/redisHelper.ts b/src/redisHelper.ts index ddc6e72e..b8c2b585 100644 --- a/src/redisHelper.ts +++ b/src/redisHelper.ts @@ -81,6 +81,7 @@ export default class RedisHelper { }); } catch (error) { console.error('[Redis] Error creating client:', error); + HawkCatcher.send(error as Error); this.redisClient = null; } } From 57b8de7ad097e3b2ff831f99e98ab348d9453838 Mon Sep 17 00:00:00 2001 From: Pavel Zotikov Date: Thu, 13 Nov 2025 00:12:21 +0300 Subject: [PATCH 29/29] fix for pr comments --- src/index.ts | 13 ------------- src/utils/chartStorageKeys.ts | 19 ++++++++++--------- 2 files changed, 10 insertions(+), 22 deletions(-) diff --git a/src/index.ts b/src/index.ts index 6bb3eb30..d84776b5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -217,19 +217,6 @@ class HawkAPI { * Checking env variables * If at least one path is not transmitted, the variable tlsVerify is undefined */ - if ( - ![process.env.TLS_CA_CERT, process.env.TLS_CERT, process.env.TLS_KEY].some(value => value === undefined || value.length === 0) - ) { - /* - * tlsVerify is used for accounting SDK (currently commented out) - * const tlsVerify = { - * tlsCaCertPath: `${process.env.TLS_CA_CERT}`, - * tlsCertPath: `${process.env.TLS_CERT}`, - * tlsKeyPath: `${process.env.TLS_KEY}`, - * }; - */ - } - /* * const accounting = new Accounting({ * baseURL: `${process.env.CODEX_ACCOUNTING_URL}`, diff --git a/src/utils/chartStorageKeys.ts b/src/utils/chartStorageKeys.ts index 78a049e4..be51e077 100644 --- a/src/utils/chartStorageKeys.ts +++ b/src/utils/chartStorageKeys.ts @@ -36,14 +36,15 @@ export function composeProjectMetricsKey( * @returns suffix string (minutely, hourly, daily) */ export function getTimeSeriesSuffix(groupBy: number): string { - if (groupBy === 1) { - return 'minutely'; - } else if (groupBy === 60) { - return 'hourly'; - } else if (groupBy === 1440) { - return 'daily'; + switch (groupBy) { + case 1: + return 'minutely'; + case 60: + return 'hourly'; + case 1440: + return 'daily'; + default: + // For custom intervals, fallback to minutely with aggregation + return 'minutely'; } - - // For custom intervals, fallback to minutely with aggregation - return 'minutely'; } \ No newline at end of file