fix: Add Redis auto-reconnect mechanism for Kubernetes pod restarts

- Implement singleton pattern for Redis client to use single connection
- Add exponential backoff reconnection strategy (1s to 30s max)
- Handle connection lifecycle events (ready, reconnecting, error, end)
- Initialize Redis on app startup alongside MongoDB and RabbitMQ
- Graceful degradation to MongoDB when Redis is unavailable
- Add comprehensive documentation in REDIS_RECONNECT.md

This fixes the issue where API lost Redis connection after pod restart
and required manual API pod restart to restore functionality.

Author: pavelzotikov

src/index.ts

@@ -29,6 +29,7 @@ import { graphqlUploadExpress } from 'graphql-upload';
 import { metricsMiddleware, createMetricsServer, graphqlMetricsPlugin } from './metrics';
 import { requestLogger } from './utils/logger';
 import ReleasesFactory from './models/releasesFactory';
+import RedisHelper from './redisHelper';
 
 /**
  * Option to enable playground
@@ -252,6 +253,11 @@ class HawkAPI {
   public async start(): Promise<void> {
     await mongo.setupConnections();
     await rabbitmq.setupConnections();
+    
+    // Initialize Redis singleton with auto-reconnect
+    const redis = RedisHelper.getInstance();
+    await redis.initialize();
+    
     await this.server.start();
     this.app.use(graphqlUploadExpress());
     this.server.applyMiddleware({ app: this.app });

src/models/eventsFactory.js

@@ -72,10 +72,9 @@ const MAX_DB_READ_BATCH_SIZE = Number(process.env.MAX_DB_READ_BATCH_SIZE);
  */
 class EventsFactory extends Factory {
   /**
-  /**
-   * Redis helper instance for modifying data through redis
+   * Redis helper instance for modifying data through redis (singleton)
    */
-  redis = new RedisHelper();
+  redis = RedisHelper.getInstance();
 
   /**
    * Event types with collections where they stored

src/redisHelper.ts

@@ -11,40 +11,92 @@ export default class RedisHelper {
    */
   private static readonly LOCK_TTL = 10;
 
+  /**
+   * Singleton instance
+   */
+  private static instance: RedisHelper | null = null;
+
   /**
    * Redis client instance
    */
-  private readonly redisClient!: RedisClientType;
+  private redisClient!: RedisClientType;
+
+  /**
+   * Flag to track if we're currently reconnecting
+   */
+  private isReconnecting = false;
 
   /**
    * Constructor
-   * Initializes the Redis client and sets up error handling
+   * Initializes the Redis client and sets up error handling with auto-reconnect
    */
   constructor() {
     try {
-      this.redisClient = createClient({ url: process.env.REDIS_URL });
+      this.redisClient = createClient({
+        url: process.env.REDIS_URL,
+        socket: {
+          reconnectStrategy: (retries) => {
+            // Exponential backoff: wait longer between each retry
+            // Max wait time: 30 seconds
+            const delay = Math.min(retries * 1000, 30000);
+            console.log(`[Redis] Reconnecting... attempt ${retries}, waiting ${delay}ms`);
+            return delay;
+          },
+        },
+      });
 
+      // Handle connection errors
       this.redisClient.on('error', (error) => {
         console.error('[Redis] Client error:', error);
         if (error) {
           HawkCatcher.send(error);
         }
       });
+
+      // Handle successful reconnection
+      this.redisClient.on('ready', () => {
+        console.log('[Redis] Client ready');
+        this.isReconnecting = false;
+      });
+
+      // Handle reconnecting event
+      this.redisClient.on('reconnecting', () => {
+        console.log('[Redis] Client reconnecting...');
+        this.isReconnecting = true;
+      });
+
+      // Handle connection end
+      this.redisClient.on('end', () => {
+        console.log('[Redis] Connection ended');
+      });
     } catch (error) {
       console.error('[Redis] Error creating client:', error);
     }
   }
 
+  /**
+   * Get singleton instance
+   */
+  public static getInstance(): RedisHelper {
+    if (!RedisHelper.instance) {
+      RedisHelper.instance = new RedisHelper();
+    }
+    return RedisHelper.instance;
+  }
+
   /**
    * Connect to Redis
    */
   public async initialize(): Promise<void> {
     try {
-      await this.redisClient.connect();
-      console.log('[Redis] Connected successfully');
+      if (!this.redisClient.isOpen && !this.isReconnecting) {
+        await this.redisClient.connect();
+        console.log('[Redis] Connected successfully');
+      }
     } catch (error) {
       console.error('[Redis] Connection failed:', error);
       HawkCatcher.send(error as Error);
+      // Don't throw - let reconnectStrategy handle it
     }
   }
 
@@ -57,6 +109,13 @@ export default class RedisHelper {
       console.log('[Redis] Connection closed');
     }
   }
+
+  /**
+   * Check if Redis is connected
+   */
+  public isConnected(): boolean {
+    return this.redisClient.isOpen;
+  }
 
   public async getChartDataFromRedis(
     startDate: string,
@@ -66,7 +125,9 @@ export default class RedisHelper {
     projectId = '',
     groupHash = ''
   ): Promise<{ timestamp: number; count: number }[]> {
+    // If Redis is not connected, throw error to fallback to MongoDB
     if (!this.redisClient.isOpen) {
+      console.warn('[Redis] Client not connected, will fallback to MongoDB');
       throw new Error('Redis client not connected');
     }