Feature: Add ability for the testserver to append coverage to previous runs (#349)

CHANGELOG.md

@@ -0,0 +1,19 @@
+# Changelog
+
+Patch version should only be changes to the specs and mock api implementations.
+
+## 3.3.0
+
+- Added `--appendCoverage` flag to append to previous code coverage result [PR 349](https://github.com/Azure/autorest.testserver/pull/349)
+
+## 3.2.0
+
+- Renamed `LLC` category to `DPG`
+
+## 3.1.0
+
+- Removed wiremock
+
+## 3.0.0
+
+- Test server v2 using a simplified api optimized for a mock api [PR 251](https://github.com/Azure/autorest.testserver/pull/251)

README.md

@@ -20,6 +20,9 @@ autorest-testserver run
 # Start testserver at given port
 autorest-testserver run --port=<port>
 
+# Start testserver without reseting the coverage. This can be used when you are running the test server multiple times to get the full coverage.
+autorest-testserver run --appendCoverage
+
 # Stop testserver
 autorest-testserver stop
 
@@ -32,6 +35,8 @@ autorest-testserver run --coverageDirectory=<path>
 
 ### Coverage upload
 
+Upload the coverage produce by the autorest testserver.
+
 ```bash
 autorest-testserver-coverage publish \
   --coverageDirectory=<path> \
@@ -43,6 +48,14 @@ autorest-testserver-coverage publish \
 
 ```
 
+### Clear coverage folder
+
+Clear the coverage folder. `--coverageDirectory` is optional. It defaults to `./coverage`
+
+```bash
+autorest-testserver-coverage clear [--coverageDirectory=<path>]
+```
+
 ## Developping
 
 ```bash

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@microsoft.azure/autorest.testserver",
-  "version": "3.2.4",
+  "version": "3.3.0",
   "description": "Autorest test server.",
   "main": "dist/cli/cli.js",
   "bin": {

src/api/request-processor.ts

@@ -1,68 +1,69 @@
-import { Response } from "express";
-import { logger } from "../logger";
-import { RequestExt } from "../server";
-import { coverageService } from "../services";
-import { MockRequest } from "./mock-request";
-import { MockResponse } from "./mock-response";
-import { ValidationError } from "./validation-error";
-
-export type MockRequestHandler = (req: MockRequest) => MockResponse | Promise<MockResponse>;
-
-export const processRequest = async (
-  category: string,
-  name: string | undefined,
-  request: RequestExt,
-  response: Response,
-  func: MockRequestHandler,
-): Promise<void> => {
-  const mockRequest = new MockRequest(request);
-  const mockResponse = await callHandler(mockRequest, response, func);
-  if (mockResponse === undefined) {
-    return;
-  }
-
-  if (mockResponse.status >= 200 && mockResponse.status < 300) {
-    if (name) {
-      await coverageService.track(category, name);
-    }
-  }
-  processResponse(response, mockResponse);
-};
-
-const processResponse = (response: Response, mockResponse: MockResponse) => {
-  response.status(mockResponse.status);
-
-  if (mockResponse.headers) {
-    response.set(mockResponse.headers);
-  }
-
-  if (mockResponse.body) {
-    response.contentType(mockResponse.body.contentType).send(mockResponse.body.rawContent);
-  }
-
-  response.end();
-};
-
-const callHandler = async (
-  mockRequest: MockRequest,
-  response: Response,
-  func: MockRequestHandler,
-): Promise<MockResponse | undefined> => {
-  try {
-    return func(mockRequest);
-  } catch (e) {
-    if (!(e instanceof ValidationError)) {
-      throw e;
-    }
-
-    logger.warn(
-      [`Request validation failed: ${e.message}:`, ` Expected:\n ${e.expected}`, ` Actual: \n${e.actual}`].join("\n"),
-    );
-    response
-      .status(400)
-      .contentType("application/json")
-      .send(e.toJSON ? e.toJSON() : JSON.stringify(e.message))
-      .end();
-    return undefined;
-  }
-};
+import { Response } from "express";
+import { logger } from "../logger";
+import { RequestExt } from "../server";
+import { coverageService } from "../services";
+import { Category } from "./mock-api-router";
+import { MockRequest } from "./mock-request";
+import { MockResponse } from "./mock-response";
+import { ValidationError } from "./validation-error";
+
+export type MockRequestHandler = (req: MockRequest) => MockResponse | Promise<MockResponse>;
+
+export const processRequest = async (
+  category: Category,
+  name: string | undefined,
+  request: RequestExt,
+  response: Response,
+  func: MockRequestHandler,
+): Promise<void> => {
+  const mockRequest = new MockRequest(request);
+  const mockResponse = await callHandler(mockRequest, response, func);
+  if (mockResponse === undefined) {
+    return;
+  }
+
+  if (mockResponse.status >= 200 && mockResponse.status < 300) {
+    if (name) {
+      await coverageService.track(category, name);
+    }
+  }
+  processResponse(response, mockResponse);
+};
+
+const processResponse = (response: Response, mockResponse: MockResponse) => {
+  response.status(mockResponse.status);
+
+  if (mockResponse.headers) {
+    response.set(mockResponse.headers);
+  }
+
+  if (mockResponse.body) {
+    response.contentType(mockResponse.body.contentType).send(mockResponse.body.rawContent);
+  }
+
+  response.end();
+};
+
+const callHandler = async (
+  mockRequest: MockRequest,
+  response: Response,
+  func: MockRequestHandler,
+): Promise<MockResponse | undefined> => {
+  try {
+    return func(mockRequest);
+  } catch (e) {
+    if (!(e instanceof ValidationError)) {
+      throw e;
+    }
+
+    logger.warn(
+      [`Request validation failed: ${e.message}:`, ` Expected:\n ${e.expected}`, ` Actual: \n${e.actual}`].join("\n"),
+    );
+    response
+      .status(400)
+      .contentType("application/json")
+      .send(e.toJSON ? e.toJSON() : JSON.stringify(e.message))
+      .end();
+    return undefined;
+  }
+};

src/app/app.ts

@@ -1,39 +1,40 @@
-import path from "path";
-import { app } from "../api";
-import { registerLegacyRoutes } from "../legacy";
-import { logger } from "../logger";
-import { internalRouter } from "../routes";
-import { MockApiServer } from "../server";
-import { coverageService } from "../services";
-import { findFilesFromPattern } from "../utils";
-import { ApiMockAppConfig } from "./config";
-
-export const ROUTE_FOLDER = path.join(__dirname, "../test-routes");
-
-export class ApiMockApp {
-  private server: MockApiServer;
-
-  constructor(private config: ApiMockAppConfig) {
-    coverageService.coverageDirectory = config.coverageDirectory;
-    this.server = new MockApiServer({ port: config.port });
-  }
-
-  public async start(): Promise<void> {
-    this.server.use("/", internalRouter);
-
-    await requireMockRoutes(ROUTE_FOLDER);
-    registerLegacyRoutes(this.server);
-
-    const apiRouter = app;
-    this.server.use("/", apiRouter.router);
-    this.server.start();
-  }
-}
-
-export const requireMockRoutes = async (routesFolder: string): Promise<void> => {
-  const files = await findFilesFromPattern(path.join(routesFolder, "/**/*.js"));
-  logger.debug("Detected routes:", files);
-  for (const file of files) {
-    require(path.resolve(file));
-  }
-};
+import path from "path";
+import { app } from "../api";
+import { registerLegacyRoutes } from "../legacy";
+import { logger } from "../logger";
+import { internalRouter } from "../routes";
+import { MockApiServer } from "../server";
+import { coverageService } from "../services";
+import { findFilesFromPattern } from "../utils";
+import { ApiMockAppConfig } from "./config";
+
+export const ROUTE_FOLDER = path.join(__dirname, "../test-routes");
+
+export class ApiMockApp {
+  private server: MockApiServer;
+
+  constructor(private config: ApiMockAppConfig) {
+    this.server = new MockApiServer({ port: config.port });
+  }
+
+  public async start(): Promise<void> {
+    this.server.use("/", internalRouter);
+
+    await requireMockRoutes(ROUTE_FOLDER);
+
+    // Need to init after registering the new routes but before the legacy routes.
+    coverageService.init(this.config.coverageDirectory, this.config.appendCoverage);
+    registerLegacyRoutes(this.server);
+    const apiRouter = app;
+    this.server.use("/", apiRouter.router);
+    this.server.start();
+  }
+}
+
+export const requireMockRoutes = async (routesFolder: string): Promise<void> => {
+  const files = await findFilesFromPattern(path.join(routesFolder, "/**/*.js"));
+  logger.debug("Detected routes:", files);
+  for (const file of files) {
+    require(path.resolve(file));
+  }
+};

src/app/config.ts

@@ -1,4 +1,5 @@
-export interface ApiMockAppConfig {
-  port: number;
-  coverageDirectory: string;
-}
+export interface ApiMockAppConfig {
+  port: number;
+  coverageDirectory: string;
+  appendCoverage?: boolean;
+}

src/cli/cli-config.ts

@@ -13,4 +13,9 @@ export interface CliConfig {
    * Directory where the coverage reports should be saved.
    */
   coverageDirectory: string;
+
+  /**
+   * Append coverage
+   */
+  appendCoverage?: boolean;
 }

src/cli/cli.ts

@@ -47,7 +47,11 @@ const run = async () => {
     .command(
       ["$0", "run"],
       "Run the autorest test server.",
-      () => null,
+      (cmd) =>
+        cmd.option("appendCoverage", {
+          type: "boolean",
+          description: "Load the existing coverage reports and append to it instead of starting fresh.",
+        }),
       (args) => runCommand(args),
     )
     .command(

src/cli/commands/run-command.ts

@@ -1,14 +1,15 @@
-import { ApiMockApp, ApiMockAppConfig } from "../../app";
-import { CliConfig } from "../cli-config";
-
-const getAppConfig = (cliConfig: CliConfig): ApiMockAppConfig => {
-  return {
-    coverageDirectory: cliConfig.coverageDirectory,
-    port: cliConfig.port,
-  };
-};
-
-export const runCommand = async (cliConfig: CliConfig): Promise<void> => {
-  const app = new ApiMockApp(getAppConfig(cliConfig));
-  await app.start();
-};
+import { ApiMockApp, ApiMockAppConfig } from "../../app";
+import { CliConfig } from "../cli-config";
+
+const getAppConfig = (cliConfig: CliConfig): ApiMockAppConfig => {
+  return {
+    coverageDirectory: cliConfig.coverageDirectory,
+    port: cliConfig.port,
+    appendCoverage: cliConfig.appendCoverage,
+  };
+};
+
+export const runCommand = async (cliConfig: CliConfig): Promise<void> => {
+  const app = new ApiMockApp(getAppConfig(cliConfig));
+  await app.start();
+};

src/reporter/cli.ts

@@ -1,5 +1,6 @@
 #!/usr/bin/env node
 
+import { rm, rmdir } from "fs/promises";
 import { join } from "path";
 import yargs from "yargs";
 import { hideBin } from "yargs/helpers";
@@ -47,6 +48,14 @@ async function main() {
         );
       },
     )
+    .command(
+      "clear",
+      "Clear the code coverage directory",
+      () => null,
+      async (args) => {
+        await rm(args.coverageDirectory, { recursive: true });
+      },
+    )
     .fail(function (msg, err) {
       if (err) {
         throw err;

src/services/coverage-service.ts

@@ -1,5 +1,6 @@
 import fs from "fs";
 import { join } from "path";
+import { Category } from "../api";
 import { logger } from "../logger";
 import { ensureDir } from "../utils";
 
@@ -8,7 +9,7 @@ export interface CoverageMap {
 }
 
 export class CoverageService {
-  public coverageDirectory = "./coverage";
+  private coverageDirectory = "./coverage";
 
   private coverage: { [category: string]: CoverageMap } = {
     defaultCategoryName: {},
@@ -24,7 +25,7 @@ export class CoverageService {
    * @param name Name of the scenario.
    * @param value {Optional} For legacy test set the value of the usage.
    */
-  public async track(category: string, name: string): Promise<void> {
+  public async track(category: Category, name: string): Promise<void> {
     let map = this.coverage[category];
     if (!map) {
       map = this.coverage[category] = {};
@@ -42,7 +43,7 @@ export class CoverageService {
    * For LEGACY test only.
    * @deprecated
    */
-  public legacyTrack(category: string, name: string, value: number): void {
+  public legacyTrack(category: Category, name: string, value: number): void {
     let map = this.coverage[category];
     if (!map) {
       map = this.coverage[category] = {};
@@ -77,10 +78,17 @@ export class CoverageService {
     }
   }
 
-  private async saveCoverage(category: string) {
+  public init(coverageDirectory: string, loadExisting = false): void {
+    this.coverageDirectory = coverageDirectory;
+    if (loadExisting) {
+      this.loadExistingCoverages();
+    }
+  }
+
+  private async saveCoverage(category: Category) {
     const categoryMap = this.coverage[category];
     await ensureDir(this.coverageDirectory);
-    const path = join(this.coverageDirectory, `report-${category}.json`);
+    const path = this.getReportPath(category);
 
     try {
       await fs.promises.writeFile(path, JSON.stringify(categoryMap, null, 2));
@@ -89,16 +97,45 @@ export class CoverageService {
     }
   }
 
-  private legacySaveCoverage(category: string) {
+  private legacySaveCoverage(category: Category) {
     const categoryMap = this.coverage[category];
     fs.mkdirSync(this.coverageDirectory, { recursive: true });
-    const path = join(this.coverageDirectory, `report-${category}.json`);
+    const path = this.getReportPath(category);
     try {
       fs.writeFileSync(path, JSON.stringify(categoryMap, null, 2));
     } catch (e) {
       logger.warn("Error while saving coverage", e);
     }
   }
+
+  private loadExistingCoverages() {
+    const categories: Category[] = ["vanilla", "azure", "optional", "dpg"];
+    for (const category of categories) {
+      this.loadExistingCoverage(category);
+    }
+  }
+
+  private loadExistingCoverage(category: Category) {
+    const path = this.getReportPath(category);
+    try {
+      if (!fs.existsSync(path)) {
+        logger.warn(`Coverage for category '${category}' doesn't exists yet(File '${path}' is not found)`);
+
+        return;
+      }
+      const content = fs.readFileSync(path);
+      const data = JSON.parse(content.toString());
+      for (const [key, value] of Object.entries(data)) {
+        this.coverage[category][key] = value as number;
+      }
+    } catch (e) {
+      logger.warn("Error while loading existing coverage", e);
+    }
+  }
+
+  private getReportPath(category: Category) {
+    return join(this.coverageDirectory, `report-${category}.json`);
+  }
 }
 
 export const coverageService = new CoverageService();