Merge branch 'feature/redis-timeseries-helper' into stage

Author: pavelzotikov

package.json

@@ -28,6 +28,7 @@
     "jest": "^26.2.2",
     "mongodb-memory-server": "^6.6.1",
     "nodemon": "^2.0.2",
+    "redis-mock": "^0.56.3",
     "ts-jest": "^26.1.4",
     "ts-node": "^10.9.1",
     "typescript": "^4.7.4"

src/index.ts

@@ -149,6 +149,7 @@ class HawkAPI {
   /**
    * Creates factories to work with models
    * @param dataLoaders - dataLoaders for fetching data form database
+   * @returns factories object
    */
   private static setupFactories(dataLoaders: DataLoaders): ContextFactories {
     // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
@@ -213,21 +214,20 @@ class HawkAPI {
 
     /**
      * Initializing accounting SDK
-     */
-    let tlsVerify;
-
-    /**
      * Checking env variables
      * If at least one path is not transmitted, the variable tlsVerify is undefined
      */
     if (
       ![process.env.TLS_CA_CERT, process.env.TLS_CERT, process.env.TLS_KEY].some(value => value === undefined || value.length === 0)
     ) {
-      tlsVerify = {
-        tlsCaCertPath: `${process.env.TLS_CA_CERT}`,
-        tlsCertPath: `${process.env.TLS_CERT}`,
-        tlsKeyPath: `${process.env.TLS_KEY}`,
-      };
+      /*
+       * tlsVerify is used for accounting SDK (currently commented out)
+       * const tlsVerify = {
+       *   tlsCaCertPath: `${process.env.TLS_CA_CERT}`,
+       *   tlsCertPath: `${process.env.TLS_CERT}`,
+       *   tlsKeyPath: `${process.env.TLS_KEY}`,
+       * };
+       */
     }
 
     /*
@@ -253,11 +253,12 @@ class HawkAPI {
   public async start(): Promise<void> {
     await mongo.setupConnections();
     await rabbitmq.setupConnections();
-    
+
     // Initialize Redis singleton with auto-reconnect
     const redis = RedisHelper.getInstance();
+
     await redis.initialize();
-    
+
     await this.server.start();
     this.app.use(graphqlUploadExpress());
     this.server.applyMiddleware({ app: this.app });

src/models/eventsFactory.js

@@ -1,14 +1,13 @@
 import { getMidnightWithTimezoneOffset, getUTCMidnight } from '../utils/dates';
 import safe from 'safe-regex';
 import { createProjectEventsByIdLoader } from '../dataLoaders';
-import { Effect, sgr } from '../utils/ansi';
+import RedisHelper from '../redisHelper';
+import ChartDataService from '../services/chartDataService';
 
 const Factory = require('./modelFactory');
 const mongo = require('../mongo');
 const Event = require('../models/event');
 const { ObjectID } = require('mongodb');
-import RedisHelper from '../redisHelper';
-import ChartDataService from '../services/chartDataService';
 const { composeEventPayloadByRepetition } = require('../utils/merge');
 
 const MAX_DB_READ_BATCH_SIZE = Number(process.env.MAX_DB_READ_BATCH_SIZE);
@@ -88,7 +87,7 @@ class EventsFactory extends Factory {
 
   /**
    * Creates Event instance
-   * @param {ObjectId} projectId - project ID
+   * @param {ObjectId} projectId
    */
   constructor(projectId) {
     super();
@@ -107,8 +106,6 @@ class EventsFactory extends Factory {
       throw new Error('Can not construct Event model, because projectId is not provided');
     }
 
-    this.redis.initialize();
-
     this.projectId = projectId;
     this.eventsDataLoader = createProjectEventsByIdLoader(mongo.databases.events, this.projectId);
   }
@@ -462,6 +459,7 @@ class EventsFactory extends Factory {
       return this.findChartData(days, timezoneOffset, '');
     } catch (err) {
       console.error('[EventsFactory] getProjectChartData error:', err);
+
       // Fallback to Mongo on error (empty groupHash for project-level data)
       return this.findChartData(days, timezoneOffset, '');
     }

src/redisHelper.ts

@@ -1,6 +1,9 @@
 import HawkCatcher from '@hawk.so/nodejs';
 import { createClient, RedisClientType } from 'redis';
-import { Effect, sgr } from './utils/ansi';
+
+// eslint call error: 0:0  error  Parsing error: Cannot read properties of undefined (reading 'map')
+// export type TsRangeResult = [timestamp: string, value: string];
+export type TsRangeResult = any;
 
 /**
  * Helper class for working with Redis
@@ -36,8 +39,10 @@ export default class RedisHelper {
         url: process.env.REDIS_URL,
         socket: {
           reconnectStrategy: (retries) => {
-            // Exponential backoff: wait longer between each retry
-            // Max wait time: 30 seconds
+            /*
+             * Exponential backoff: wait longer between each retry
+             * Max wait time: 30 seconds
+             */
             const delay = Math.min(retries * 1000, 30000);
             console.log(`[Redis] Reconnecting... attempt ${retries}, waiting ${delay}ms`);
             return delay;
@@ -133,7 +138,7 @@ export default class RedisHelper {
     end: string,
     aggregationType: string,
     bucketMs: string
-  ): Promise<[string, string][]> {
+  ): Promise<TsRangeResult[]> {
     return (await this.redisClient.sendCommand([
       'TS.RANGE',
       key,
@@ -142,6 +147,6 @@ export default class RedisHelper {
       'AGGREGATION',
       aggregationType,
       bucketMs,
-    ])) as [string, string][];
+    ])) as TsRangeResult[];
   }
-}
+}

src/services/chartDataService.ts

@@ -1,4 +1,4 @@
-import RedisHelper from '../redisHelper';
+import RedisHelper, { TsRangeResult } from '../redisHelper';
 import { composeProjectMetricsKey, getTimeSeriesSuffix } from '../utils/chartStorageKeys';
 
 /**
@@ -45,7 +45,7 @@ export default class ChartDataService {
     const bucketMs = groupBy * 60 * 1000;
 
     // Fetch data from Redis
-    let result: [string, string][] = [];
+    let result: TsRangeResult[] = [];
     try {
       result = await this.redisHelper.tsRange(
         key,
@@ -89,4 +89,3 @@ export default class ChartDataService {
     return filled.sort((a, b) => a.timestamp - b.timestamp);
   }
 }
-

src/utils/chartStorageKeys.ts

@@ -1,5 +1,8 @@
 /**
- * Utility for composing Redis TimeSeries keys
+ * Utilities for composing Redis TimeSeries keys for chart data storage
+ *
+ * Note: Event-level metrics use MongoDB only (not Redis TimeSeries)
+ * Only project-level metrics are stored in Redis TimeSeries
  */
 
 /**
@@ -26,24 +29,6 @@ export function composeProjectMetricsKey(
   return `ts:project-${metricType}:${projectId}:${granularity}`;
 }
 
-/**
- * Compose Redis TimeSeries key for event-level metrics
- *
- * @param suffix - time granularity suffix (minutely, hourly, daily)
- * @param groupHash - event group hash
- * @returns Redis key string
- *
- * @example
- * composeEventTimeSeriesKey('daily', 'abc123def')
- * // => 'ts:events:abc123def:daily'
- */
-export function composeEventTimeSeriesKey(
-  suffix: string,
-  groupHash: string
-): string {
-  return `ts:events:${groupHash}:${suffix}`;
-}
-
 /**
  * Get time granularity suffix based on groupBy interval
  *
@@ -58,8 +43,7 @@ export function getTimeSeriesSuffix(groupBy: number): string {
   } else if (groupBy === 1440) {
     return 'daily';
   }
-  
+
   // For custom intervals, fallback to minutely with aggregation
   return 'minutely';
-}
-
+}

test/integration/api.env

@@ -7,7 +7,9 @@ MONGO_HAWK_DB_URL=mongodb://mongodb:27017/hawk
 # Events database URL
 MONGO_EVENTS_DB_URL=mongodb://mongodb:27017/hawk_events
 
-
+# Redis URL (optional, for TimeSeries metrics)
+# If not set, Redis features will be disabled gracefully
+REDIS_URL=redis:6379
 
 # MongoDB settings
 MONGO_RECONNECT_TRIES=60

test/integration/jestEnv.js

@@ -1,6 +1,7 @@
 const NodeEnvironment = require('jest-environment-node');
 const amqp = require('amqplib');
 const mongodb = require('mongodb');
+const { installRedisMock, uninstallRedisMock } = require('./redisMock');
 
 /**
  * Custom test environment for defining global connections
@@ -19,6 +20,12 @@ class CustomEnvironment extends NodeEnvironment {
     await mongoClient.db('hawk').dropDatabase();
     // await mongoClient.db('codex_accounting').dropDatabase();
 
+    /**
+     * Use redis-mock instead of a real Redis connection.
+     * This avoids spinning up Redis during integration tests while keeping the API surface.
+     */
+    this.global.redisClient = installRedisMock();
+
     this.rabbitMqConnection = await amqp.connect('amqp://guest:guest@rabbitmq:5672/');
     this.global.rabbitChannel = await this.rabbitMqConnection.createChannel();
     await this.global.rabbitChannel.purgeQueue('cron-tasks/limiter');
@@ -41,6 +48,8 @@ class CustomEnvironment extends NodeEnvironment {
       if (this.rabbitMqConnection) {
         await this.rabbitMqConnection.close();
       }
+
+      uninstallRedisMock();
     } catch (error) {
       console.error('Error during teardown:', error);
     }

test/integration/redisMock.js

@@ -0,0 +1,72 @@
+const path = require('path');
+const redisMock = require('redis-mock');
+
+let originalRedisModule = null;
+let redisModulePath = null;
+
+/**
+ * Create Redis mock client compatible with node-redis v4 API portions we use.
+ *
+ * @returns {object} mocked redis client
+ */
+function createMockClient() {
+  const client = redisMock.createClient();
+
+  client.isOpen = true;
+  client.connect = async () => client;
+  client.quit = async () => undefined;
+  client.sendCommand = async () => [];
+  client.on = () => client;
+
+  return client;
+}
+
+/**
+ * Install redis-mock into Node's module cache so that `require('redis')`
+ * returns the mocked client factory.
+ *
+ * @returns {object} mock client instance to be reused in tests
+ */
+function installRedisMock() {
+  redisModulePath = require.resolve('redis');
+  originalRedisModule = require.cache[redisModulePath] || null;
+
+  const mockExports = {
+    createClient: () => createMockClient(),
+  };
+
+  require.cache[redisModulePath] = {
+    id: redisModulePath,
+    filename: redisModulePath,
+    loaded: true,
+    exports: mockExports,
+    path: path.dirname(redisModulePath),
+    children: [],
+  };
+
+  return mockExports.createClient();
+}
+
+/**
+ * Restore original `redis` module if it existed.
+ */
+function uninstallRedisMock() {
+  if (!redisModulePath) {
+    return;
+  }
+
+  if (originalRedisModule) {
+    require.cache[redisModulePath] = originalRedisModule;
+  } else {
+    delete require.cache[redisModulePath];
+  }
+
+  originalRedisModule = null;
+  redisModulePath = null;
+}
+
+module.exports = {
+  installRedisMock,
+  uninstallRedisMock,
+};
+

yarn.lock

@@ -5817,6 +5817,11 @@ readdirp@~3.6.0:
   dependencies:
     picomatch "^2.2.1"
 
+redis-mock@^0.56.3:
+  version "0.56.3"
+  resolved "https://registry.yarnpkg.com/redis-mock/-/redis-mock-0.56.3.tgz#e96471bcc774ddc514c2fc49cdd03cab2baecd89"
+  integrity sha512-ynaJhqk0Qf3Qajnwvy4aOjS4Mdf9IBkELWtjd+NYhpiqu4QCNq6Vf3Q7c++XRPGiKiwRj9HWr0crcwy7EiPjYQ==
+
 redis@^4.7.0:
   version "4.7.1"
   resolved "https://registry.yarnpkg.com/redis/-/redis-4.7.1.tgz#08588a30936be0e7ad9c0f3e1ac6a85ccaf73e94"