chore(devops): set up basic prometheus metrics (#541)

* Initial plan

* Initial plan for Prometheus metrics integration

Co-authored-by: neSpecc <3684889+neSpecc@users.noreply.github.com>

* Add Prometheus metrics integration with prom-client

Co-authored-by: neSpecc <3684889+neSpecc@users.noreply.github.com>

* Add metrics integration tests and documentation

Co-authored-by: neSpecc <3684889+neSpecc@users.noreply.github.com>

* Bump version up to 1.1.42

* Remove package-lock.json and add to .gitignore (use yarn)

Co-authored-by: neSpecc <3684889+neSpecc@users.noreply.github.com>

* Fix yarn.lock to preserve git dependency commit hash

Co-authored-by: neSpecc <3684889+neSpecc@users.noreply.github.com>

* Update Dockerfile.dev and CI workflows to use Node 16 for prom-client compatibility

Co-authored-by: neSpecc <3684889+neSpecc@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: neSpecc <3684889+neSpecc@users.noreply.github.com>
Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

Author: 3 people

.env.sample

@@ -1,6 +1,9 @@
 # API server port
 PORT=4000
 
+# Metrics server port
+METRICS_PORT=9090
+
 # Hawk API database URL
 MONGO_HAWK_DB_URL=mongodb://mongodb:27017/hawk
 

.github/workflows/integration-tests.yml

@@ -10,5 +10,10 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
+      - name: Use Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: '.nvmrc'
+
       - name: Run tests
         run: yarn test:integration

.github/workflows/tests.yml

@@ -9,6 +9,10 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v2
+      - name: Use Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: '.nvmrc'
       - name: Install modules
         run: yarn
       - name: Run tests

.gitignore

@@ -7,3 +7,4 @@ uploads
 globalConfig.json
 coverage
 tls
+package-lock.json

docker/Dockerfile.dev

@@ -1,13 +1,13 @@
-FROM node:14.17.0-alpine as builder
+FROM node:16-alpine as builder
 
 WORKDIR /usr/src/app
-RUN apk add --no-cache git gcc g++ python make musl-dev
+RUN apk add --no-cache git gcc g++ python3 make musl-dev
 
 COPY package.json yarn.lock ./
 
 RUN yarn install
 
-FROM node:14.17.0-alpine
+FROM node:16-alpine
 
 WORKDIR /usr/src/app
 

docs/METRICS.md

@@ -0,0 +1,120 @@
+# Prometheus Metrics
+
+This application exposes Prometheus-compatible metrics on a separate port from the main API server.
+
+## Configuration
+
+The metrics server runs on a separate port configured via the `METRICS_PORT` environment variable:
+
+```bash
+# Default: 9090
+METRICS_PORT=9090
+```
+
+Add this to your `.env` file. See `.env.sample` for reference.
+
+## Metrics Endpoint
+
+The metrics are served at:
+
+```
+http://localhost:9090/metrics
+```
+
+(Replace `9090` with your configured `METRICS_PORT` if different)
+
+## Available Metrics
+
+### Default Node.js Metrics
+
+The following default Node.js metrics are automatically collected:
+
+- **nodejs_version_info** - Node.js version information
+- **process_cpu_user_seconds_total** - Total user CPU time spent in seconds
+- **process_cpu_system_seconds_total** - Total system CPU time spent in seconds
+- **nodejs_heap_size_total_bytes** - Total heap size in bytes
+- **nodejs_heap_size_used_bytes** - Used heap size in bytes
+- **nodejs_external_memory_bytes** - External memory in bytes
+- **nodejs_heap_space_size_total_bytes** - Total heap space size in bytes
+- **nodejs_heap_space_size_used_bytes** - Used heap space size in bytes
+- **nodejs_eventloop_lag_seconds** - Event loop lag in seconds
+- **nodejs_eventloop_lag_min_seconds** - Minimum event loop lag
+- **nodejs_eventloop_lag_max_seconds** - Maximum event loop lag
+- **nodejs_eventloop_lag_mean_seconds** - Mean event loop lag
+- **nodejs_eventloop_lag_stddev_seconds** - Standard deviation of event loop lag
+- **nodejs_eventloop_lag_p50_seconds** - 50th percentile event loop lag
+- **nodejs_eventloop_lag_p90_seconds** - 90th percentile event loop lag
+- **nodejs_eventloop_lag_p99_seconds** - 99th percentile event loop lag
+
+### Custom HTTP Metrics
+
+#### http_request_duration_seconds (Histogram)
+
+Duration of HTTP requests in seconds, labeled by:
+- `method` - HTTP method (GET, POST, etc.)
+- `route` - Request route/path
+- `status_code` - HTTP status code
+
+Buckets: 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10 seconds
+
+#### http_requests_total (Counter)
+
+Total number of HTTP requests, labeled by:
+- `method` - HTTP method (GET, POST, etc.)
+- `route` - Request route/path
+- `status_code` - HTTP status code
+
+## Testing
+
+### Manual Testing
+
+You can test the metrics endpoint using curl:
+
+```bash
+curl http://localhost:9090/metrics
+```
+
+Or run the provided test script:
+
+```bash
+./test-metrics.sh
+```
+
+### Integration Tests
+
+Integration tests for metrics are located in `test/integration/cases/metrics.test.ts`.
+
+Run them with:
+
+```bash
+npm run test:integration
+```
+
+## Implementation Details
+
+The metrics implementation uses the `prom-client` library and consists of:
+
+1. **Metrics Module** (`src/metrics/index.ts`):
+   - Initializes a Prometheus registry
+   - Configures default Node.js metrics collection
+   - Defines custom HTTP metrics (duration histogram and request counter)
+   - Provides middleware for tracking HTTP requests
+   - Creates a separate Express app for serving metrics
+
+2. **Integration** (`src/index.ts`):
+   - Adds metrics middleware to the main Express app
+   - Starts metrics server on a separate port
+   - Keeps metrics server isolated from main API traffic
+
+## Prometheus Configuration
+
+To scrape these metrics with Prometheus, add the following to your `prometheus.yml`:
+
+```yaml
+scrape_configs:
+  - job_name: 'hawk-api'
+    static_configs:
+      - targets: ['localhost:9090']
+```
+
+Adjust the target host and port according to your deployment.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "hawk.api",
-  "version": "1.1.41",
+  "version": "1.1.42",
   "main": "index.ts",
   "license": "BUSL-1.1",
   "scripts": {
@@ -80,6 +80,7 @@
     "mime-types": "^2.1.25",
     "mongodb": "^3.7.3",
     "morgan": "^1.10.1",
+    "prom-client": "^15.1.3",
     "safe-regex": "^2.1.0",
     "ts-node-dev": "^2.0.0",
     "uuid": "^8.3.2"

src/index.ts

@@ -27,6 +27,7 @@ import BusinessOperationsFactory from './models/businessOperationsFactory';
 import schema from './schema';
 import { graphqlUploadExpress } from 'graphql-upload';
 import morgan from 'morgan';
+import { metricsMiddleware, createMetricsServer } from './metrics';
 
 /**
  * Option to enable playground
@@ -48,6 +49,11 @@ class HawkAPI {
    */
   private serverPort = +(process.env.PORT || 4000);
 
+  /**
+   * Port to serve metrics endpoint
+   */
+  private metricsPort = +(process.env.METRICS_PORT || 9090);
+
   /**
    * Express application
    */
@@ -86,6 +92,11 @@ class HawkAPI {
      */
     this.app.use(morgan(process.env.NODE_ENV === 'production' ? 'combined' : 'dev'));
 
+    /**
+     * Add metrics middleware to track HTTP requests
+     */
+    this.app.use(metricsMiddleware);
+
     this.app.use(express.json());
     this.app.use(bodyParser.urlencoded({ extended: false }));
     this.app.use('/static', express.static(`./static`));
@@ -241,6 +252,15 @@ class HawkAPI {
     this.app.use(graphqlUploadExpress());
     this.server.applyMiddleware({ app: this.app });
 
+    // Start metrics server on separate port
+    const metricsApp = createMetricsServer();
+
+    metricsApp.listen(this.metricsPort, () => {
+      console.log(
+        `📊 Metrics server ready at http://localhost:${this.metricsPort}/metrics`
+      );
+    });
+
     return new Promise((resolve) => {
       this.httpServer.listen({ port: this.serverPort }, () => {
         console.log(

src/metrics/index.ts

@@ -0,0 +1,73 @@
+import client from 'prom-client';
+import express from 'express';
+
+/**
+ * Create a Registry to register the metrics
+ */
+const register = new client.Registry();
+
+/**
+ * Add default Node.js metrics (CPU, memory, event loop, etc.)
+ */
+client.collectDefaultMetrics({ register });
+
+/**
+ * HTTP request duration histogram
+ * Tracks request duration by route, method, and status code
+ */
+const httpRequestDuration = new client.Histogram({
+  name: 'http_request_duration_seconds',
+  help: 'Duration of HTTP requests in seconds',
+  labelNames: ['method', 'route', 'status_code'],
+  buckets: [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10],
+  registers: [ register ],
+});
+
+/**
+ * HTTP request counter
+ * Tracks count of HTTP requests by route, method, and status code
+ */
+const httpRequestCounter = new client.Counter({
+  name: 'http_requests_total',
+  help: 'Total number of HTTP requests',
+  labelNames: ['method', 'route', 'status_code'],
+  registers: [ register ],
+});
+
+/**
+ * Express middleware to track HTTP metrics
+ */
+export function metricsMiddleware(req: express.Request, res: express.Response, next: express.NextFunction): void {
+  const start = Date.now();
+
+  // Hook into response finish event to capture metrics
+  res.on('finish', () => {
+    const duration = (Date.now() - start) / 1000; // Convert to seconds
+    const route = req.route ? req.route.path : req.path;
+    const method = req.method;
+    const statusCode = res.statusCode.toString();
+
+    // Record metrics
+    httpRequestDuration.labels(method, route, statusCode).observe(duration);
+    httpRequestCounter.labels(method, route, statusCode).inc();
+  });
+
+  next();
+}
+
+/**
+ * Create metrics server
+ * @returns Express application serving metrics endpoint
+ */
+export function createMetricsServer(): express.Application {
+  const metricsApp = express();
+
+  metricsApp.get('/metrics', async (req, res) => {
+    res.setHeader('Content-Type', register.contentType);
+    const metrics = await register.metrics();
+
+    res.send(metrics);
+  });
+
+  return metricsApp;
+}

src/types/env.d.ts

@@ -11,6 +11,11 @@ declare namespace NodeJS {
      */
     PORT: string;
 
+    /**
+     * Metrics server port
+     */
+    METRICS_PORT: string;
+
     /**
      * MongoDB url
      */