Feature: Provide metrics emitting capabilities using statsd (#39)

package-lock.json

@@ -2102,6 +2102,15 @@
         "tweetnacl": "^0.14.3"
       }
     },
+    "bindings": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
+      "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",
+      "optional": true,
+      "requires": {
+        "file-uri-to-path": "1.0.0"
+      }
+    },
     "bl": {
       "version": "1.2.2",
       "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz",
@@ -3626,6 +3635,12 @@
         "moment": "^2.11.2"
       }
     },
+    "file-uri-to-path": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
+      "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==",
+      "optional": true
+    },
     "fill-range": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz",
@@ -4600,6 +4615,14 @@
       "integrity": "sha512-7T/BxH19zbcCTa8XkMlbK5lTo1WtgkFi3GvdWEyNuc4Vex7/9Dqbnpsf4JMydcfj9HCg4zUWFTL3Za6lapg5/w==",
       "dev": true
     },
+    "hot-shots": {
+      "version": "6.3.0",
+      "resolved": "https://registry.npmjs.org/hot-shots/-/hot-shots-6.3.0.tgz",
+      "integrity": "sha512-9aSojxGXFDQG8EiRtUp7Cd/dG0vgiQ2E/dB/5B59rdEbV8++tqaa2v/OUJW7EYyplETaglnaXsjUA8DB3LFGrw==",
+      "requires": {
+        "unix-dgram": "2.0.x"
+      }
+    },
     "hpkp": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/hpkp/-/hpkp-2.0.0.tgz",
@@ -7973,6 +7996,16 @@
       "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz",
       "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg=="
     },
+    "unix-dgram": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/unix-dgram/-/unix-dgram-2.0.3.tgz",
+      "integrity": "sha512-Bay5CkSLcdypcBCsxvHEvaG3mftzT5FlUnRToPWEAVxwYI8NI/8zSJ/Gknlp86MPhV6hBA8I8TBsETj2tssoHQ==",
+      "optional": true,
+      "requires": {
+        "bindings": "^1.3.0",
+        "nan": "^2.13.2"
+      }
+    },
     "unpipe": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",

package.json

@@ -66,6 +66,7 @@
     "express": "^4.17.1",
     "fast-safe-stringify": "^2.0.6",
     "helmet": "^3.18.0",
+    "hot-shots": "^6.3.0",
     "node-fetch": "^2.6.0",
     "nodegit": "^0.24.3",
     "reflect-metadata": "^0.1.13",

src/app.module.ts

@@ -9,6 +9,7 @@ import {
   ContentController,
   HealthCheckController,
 } from "./controllers";
+import { Telemetry, createTelemetry } from "./core";
 import { LoggingInterceptor } from "./middlewares";
 import {
   AppService,
@@ -44,6 +45,11 @@ import { ContentService } from "./services/content";
       provide: APP_INTERCEPTOR,
       useClass: LoggingInterceptor,
     },
+    {
+      provide: Telemetry,
+      useFactory: (config: Configuration) => createTelemetry(config),
+      inject: [Configuration],
+    },
   ],
 })
 export class AppModule {}

src/config/configuration.ts

@@ -7,9 +7,16 @@ import { testConfig } from "./test";
 
 export type Env = "production" | "development" | "test";
 
+export interface StatsdConfig {
+  readonly host: string | undefined;
+  readonly port: number | undefined;
+}
+
 @Injectable()
 export class Configuration {
   public readonly env: Env;
+  public readonly serviceName: string;
+  public readonly statsd: StatsdConfig;
 
   constructor() {
     const environmentOverrides: Record<Env, Partial<Configuration>> = {
@@ -26,6 +33,8 @@ export class Configuration {
     // Perform validation
     configSchema.validate({ allowed: "strict" });
 
-    this.env = configSchema.get("env");
+    this.env = env;
+    this.serviceName = configSchema.get("serviceName");
+    this.statsd = configSchema.get("statsd");
   }
 }

src/config/schema.ts

@@ -9,4 +9,24 @@ export const configSchema = convict<Configuration>({
     default: "development",
     env: "NODE_ENV",
   },
+  serviceName: {
+    doc: "Name of the service. Used when uploading metrics for example",
+    format: String,
+    default: "git-rest-api",
+    env: "serviceName",
+  },
+  statsd: {
+    host: {
+      doc: "Statsd host to upload metrics using statsd",
+      format: String,
+      default: undefined,
+      env: "statsd_host",
+    },
+    port: {
+      doc: "Statsd port to upload metrics using statsd",
+      format: Number,
+      default: undefined,
+      env: "statsd_port",
+    },
+  },
 });

src/core/index.ts

@@ -1,4 +1,5 @@
 export * from "./repo-auth";
 export * from "./logger";
 export * from "./pagination";
+export * from "./telemetry";
 export * from "./models";

src/core/logger/logger.ts

@@ -9,6 +9,7 @@ export interface LogMetadata {
 
 export class Logger {
   private logger: winston.Logger;
+
   constructor(private context: string) {
     this.logger = WINSTON_LOGGER;
   }

src/core/telemetry/factory.ts

@@ -0,0 +1,20 @@
+import { Configuration } from "../../config";
+import { Logger } from "../logger";
+import { NoopTelemetry } from "./noop-telemetry";
+import { StatsdTelemetry } from "./statsd-telemetry";
+import { Telemetry } from "./telemetry";
+
+export function createTelemetry(config: Configuration): Telemetry {
+  const logger = new Logger("TelemetryFactory");
+  const instance = getTelmetryInstance(config);
+  logger.info(`Resolving telemetry engine from configuration: ${instance.constructor.name}`);
+  return instance;
+}
+
+function getTelmetryInstance(config: Configuration) {
+  if (config.statsd.host && config.statsd.port) {
+    return new StatsdTelemetry(config);
+  } else {
+    return new NoopTelemetry();
+  }
+}

src/core/telemetry/index.ts

@@ -0,0 +1,2 @@
+export * from "./factory";
+export * from "./telemetry";

src/core/telemetry/noop-telemetry.ts

@@ -0,0 +1,7 @@
+import { Metric, Telemetry } from "./telemetry";
+
+export class NoopTelemetry extends Telemetry {
+  public emitMetric(_: Metric): void {
+    // Noop
+  }
+}

src/core/telemetry/statsd-telemetry.ts

@@ -0,0 +1,27 @@
+import { StatsD } from "hot-shots";
+
+import { Configuration } from "../../config";
+import { Metric, Telemetry } from "./telemetry";
+
+export class StatsdTelemetry extends Telemetry {
+  private instance: StatsD;
+
+  constructor(private config: Configuration) {
+    super();
+    const { host, port } = this.config.statsd;
+    this.instance = new StatsD({ host, port });
+  }
+
+  public emitMetric(metric: Metric): void {
+    const stat = JSON.stringify({
+      Metric: metric,
+      Namespace: this.config.serviceName,
+      Dims: {
+        ...metric.dimensions,
+        env: this.config.env,
+      },
+    });
+
+    this.instance.gauge(stat, metric.value);
+  }
+}

src/core/telemetry/telemetry.ts

@@ -0,0 +1,9 @@
+export interface Metric {
+  name: string;
+  value: number;
+  dimensions?: StringMap<unknown>;
+}
+
+export abstract class Telemetry {
+  public abstract emitMetric(metric: Metric): void;
+}

src/middlewares/logger-interceptor.ts

@@ -1,22 +1,23 @@
-import { CallHandler, ExecutionContext, HttpException, Injectable, NestInterceptor } from "@nestjs/common";
+import { CallHandler, ExecutionContext, HttpException, HttpStatus, Injectable, NestInterceptor } from "@nestjs/common";
+import { Request } from "express";
 import { Observable, throwError } from "rxjs";
 import { catchError, tap } from "rxjs/operators";
 
 import { Configuration } from "../config";
-import { LogMetadata, Logger } from "../core";
+import { LogMetadata, Logger, Telemetry } from "../core";
 
 @Injectable()
 export class LoggingInterceptor implements NestInterceptor {
   private logger = new Logger("Request");
 
-  constructor(private config: Configuration) {}
+  constructor(private config: Configuration, private telemetry: Telemetry) {}
 
   public intercept(context: ExecutionContext, next: CallHandler): Observable<any> {
     const now = Date.now();
-    const req = context.switchToHttp().getRequest();
+    const req: Request = context.switchToHttp().getRequest();
 
     const commonProperties = {
-      url: req.originalUrl,
+      path: req.route.path,
       method: req.method,
     };
 
@@ -28,26 +29,36 @@ export class LoggingInterceptor implements NestInterceptor {
         const properties = {
           ...commonProperties,
           duration,
-          statusCode: response.statusCode,
+          status: response.statusCode,
         };
 
-        this.logger.info(
-          `${req.method} ${response.statusCode} ${req.originalUrl} (${duration}ms)`,
-          this.clean(properties),
-        );
+        const message = `${req.method} ${response.statusCode} ${req.originalUrl} (${duration}ms)`;
+        this.logger.info(message, this.clean(properties));
+        this.trackRequest(properties);
       }),
       catchError((error: Error | HttpException) => {
-        const statusCode = error instanceof HttpException ? error.getStatus() : 500;
+        const statusCode = error instanceof HttpException ? error.getStatus() : HttpStatus.INTERNAL_SERVER_ERROR;
         const duration = Date.now() - now;
         const message = `${req.method} ${statusCode} ${req.originalUrl} (${duration}ms)`;
 
         const properties = {
+          ...commonProperties,
           duration,
-          statusCode,
+          status: statusCode,
         };
 
+        this.trackRequest(properties);
         if (statusCode >= 500) {
           this.logger.error(message, this.clean(properties));
+          this.telemetry.emitMetric({
+            name: "EXCEPTIONS",
+            value: 1,
+            dimensions: {
+              type: error.name,
+              path: properties.path,
+              method: properties.method,
+            },
+          });
         } else {
           this.logger.info(message, this.clean(properties));
         }
@@ -56,6 +67,24 @@ export class LoggingInterceptor implements NestInterceptor {
     );
   }
 
+  private trackRequest(properties: { duration: number; path: string; method: string; status: number }) {
+    const { duration, ...otherProperties } = properties;
+    this.telemetry.emitMetric({
+      name: "INCOMING_REQUEST",
+      value: 1,
+      dimensions: {
+        ...otherProperties,
+      },
+    });
+    this.telemetry.emitMetric({
+      name: "INCOMING_REQUEST_DURATION",
+      value: duration,
+      dimensions: {
+        ...otherProperties,
+      },
+    });
+  }
+
   private clean(meta: LogMetadata) {
     return this.config.env === "development" ? {} : meta;
   }