Refactoring blob items models in persistency layer

2019-03-05 17:54:19 +08:00 · 2019-03-05 17:54:19 +08:00 · 8dd55133f5
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@ -0,0 +1,56 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [{
+      "type": "node",
+      "request": "launch",
+      "name": "Azurite Blob Service",
+      "cwd": "${workspaceFolder}",
+      "runtimeArgs": [
+        "-r",
+        "ts-node/register"
+      ],
+      "args": [
+        "${workspaceFolder}/src/blob/bin.ts"
+      ],
+      "outputCapture": "std"
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Current TS File",
+      "cwd": "${workspaceFolder}",
+      "runtimeArgs": [
+        "-r",
+        "ts-node/register"
+      ],
+      "args": [
+        "${workspaceFolder}/${relativeFile}"
+      ],
+      "outputCapture": "std"
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Current Mocha TS File",
+      "cwd": "${workspaceFolder}",
+      "runtimeArgs": [
+        "-r",
+        "ts-node/register"
+      ],
+      "args": [
+        "${workspaceFolder}/node_modules/mocha/bin/_mocha",
+        "-u",
+        "tdd",
+        "--timeout",
+        "999999",
+        "--colors",
+        "${workspaceFolder}/${relativeFile}"
+      ],
+      "internalConsoleOptions": "openOnSessionStart",
+      "outputCapture": "std"
+    },
+  ]
+}
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@ -0,0 +1,3 @@
+{
+    "editor.tabSize": 2,
+}
--- a/src/blob/handlers/ContainerHandler.ts
+++ b/src/blob/handlers/ContainerHandler.ts
@ -39,8 +39,8 @@ export default class ContainerHandler extends BaseHandler
      name: containerName,
      properties: {
        etag,
-        lastModified,
-      },
+        lastModified
+      }
    });

    const response: Models.ContainerCreateResponse = {
@ -48,7 +48,7 @@ export default class ContainerHandler extends BaseHandler
      lastModified,
      requestId: blobCtx.contextID,
      statusCode: 201,
-      version: API_VERSION,
+      version: API_VERSION
    };

    return response;
@ -59,11 +59,9 @@ export default class ContainerHandler extends BaseHandler
    context: Context
  ): Promise<Models.ContainerGetPropertiesResponse> {
    const blobCtx = new BlobStorageContext(context);
+    const containerName = blobCtx.container!;

-    const container = await this.dataStore.getContainer<Models.ContainerItem>(
-      blobCtx.container!
-    );
-
+    const container = await this.dataStore.getContainer(containerName);
    if (!container) {
      throw StorageErrorFactory.getContainerNotFoundError(blobCtx.contextID!);
    }
@ -73,7 +71,7 @@ export default class ContainerHandler extends BaseHandler
      ...container.properties,
      metadata: container.metadata,
      requestId: blobCtx.contextID,
-      statusCode: 200,
+      statusCode: 200
    };

    return response;
@ -93,7 +91,8 @@ export default class ContainerHandler extends BaseHandler
    const blobCtx = new BlobStorageContext(context);
    const containerName = blobCtx.container!;

-    if (this.dataStore.getContainer(containerName) === undefined) {
+    const container = await this.dataStore.getContainer(containerName);
+    if (container === undefined) {
      throw StorageErrorFactory.getContainerNotFoundError(blobCtx.contextID!);
    }

@ -107,7 +106,7 @@ export default class ContainerHandler extends BaseHandler
      date: new Date(),
      requestId: blobCtx.contextID,
      statusCode: 202,
-      version: API_VERSION,
+      version: API_VERSION
    };

    return response;
@ -141,7 +140,7 @@ export default class ContainerHandler extends BaseHandler
      eTag: newEtag(),
      lastModified: container.properties.lastModified,
      requestId: blobCtx.contextID,
-      statusCode: 200,
+      statusCode: 200
    };

    return response;
--- a/src/blob/persistence/IBlobDataStore.ts
+++ b/src/blob/persistence/IBlobDataStore.ts
@ -1,6 +1,19 @@
 import * as Models from "../generated/artifacts/models";
 import { IDataStore } from "./IDataStore";

+export interface IBlobPrivateProperties {
+  isCommitted: boolean;
+}
+
+export type IBlobModel = IBlobPrivateProperties & Models.BlobItem;
+
+/**
+ * Persistency layer data store interface.
+ *
+ * @export
+ * @interface IBlobDataStore
+ * @extends {IDataStore}
+ */
 export interface IBlobDataStore extends IDataStore {
  setServiceProperties<T extends Models.StorageServiceProperties>(
    serviceProperties: T
@ -20,9 +33,9 @@ export interface IBlobDataStore extends IDataStore {
    marker?: number
  ): Promise<[T[], number | undefined]>;

-  updateBlob<T extends Models.BlobItem>(container: string, blob: T): Promise<T>;
+  updateBlob<T extends IBlobModel>(container: string, blob: T): Promise<T>;

-  getBlob<T extends Models.BlobItem>(container: string, blob: string): Promise<T | undefined>;
+  getBlob<T extends IBlobModel>(container: string, blob: string): Promise<T | undefined>;

  deleteBlob(container: string, blob: string): Promise<void>;

--- a/src/blob/persistence/LokiBlobDataStore.ts
+++ b/src/blob/persistence/LokiBlobDataStore.ts
@ -1,8 +1,8 @@
 import { createReadStream, createWriteStream, mkdir, stat } from "fs";
 import Loki from "lokijs";
 import { join } from "path";
-
 import { promisify } from "util";
+
 import * as Models from "../generated/artifacts/models";
 import { API_VERSION } from "../utils/constants";
 import { IBlobDataStore } from "./IBlobDataStore";
@ -19,16 +19,28 @@ import { IBlobDataStore } from "./IBlobDataStore";
 /**
 * This is a persistency layer data source implementation based on loki DB.
 *
- * Loki DB includes following collections and documents
+ * Notice that, following design is for emulator purpose only,
+ * and doesn't design for best performance. We may want to optimize the collection design according to different
+ * access frequency and data size.
+ *
+ * Loki DB includes following collections and documents:
+ *
 * -- SERVICE_PROPERTIES_COLLECTION // Collection contains service properties
- *                                  // Only 1 document
+ *                                  // Only 1 document will be kept
+ *                                  // Default collection name is $SERVICE_PROPERTIES_COLLECTION$
 * -- CONTAINERS_COLLECTION // Collection contains all container items
+ *                          // Default collection name is $CONTAINERS_COLLECTION$
 *                          // Each document maps to 1 container
- *                          // Unique name
- * -- CONTAINER_COLLECTION // Every container maps to a container collection
- *                         // Container collection contains all blobs under a container
- *                         // Each document maps to 1 blob
- *                         // Unique name
+ *                          // Unique document properties: name
+ * -- <CONTAINER_COLLECTION> // Every container collection 1:1 maps to a container
+ *                           // Container collection contains all blobs under a container
+ *                           // Collection name equals to a container name
+ *                           // Each document 1:1 maps to a blob
+ *                           // Unique document properties: name
+ * -- <BLOCK_BLOB_BLOCKS_COLLECTION>    // Block blob blocks collection includes all blocks
+ *                                      // Unique document properties: (blob)name, blockID
+ * -- <PAGE_BLOB_PAGES_COLLECTION>      // Page blob pages collection includes all pages
+ * -- <APPEND_BLOB_BLOCKS_COLLECTION>   // Append blob blocks collection includes all append blob blocks
 *
 * @export
 * @class LokiBlobDataStore
@ -36,8 +48,9 @@ import { IBlobDataStore } from "./IBlobDataStore";
 export default class LokiBlobDataStore implements IBlobDataStore {
  private readonly db: Loki;

-  private readonly CONTAINERS_COLLECTION = "$containers$";
-  private readonly SERVICE_PROPERTIES_COLLECTION = "$serviceproperties$";
+  private readonly CONTAINERS_COLLECTION = "$CONTAINERS_COLLECTION$";
+  private readonly SERVICE_PROPERTIES_COLLECTION =
+    "$SERVICE_PROPERTIES_COLLECTION$";

  private SERVICE_PROPERTIES_DOCUMENT_LOKI_ID?: number;

@ -47,29 +60,29 @@ export default class LokiBlobDataStore implements IBlobDataStore {
    hourMetrics: {
      enabled: false,
      retentionPolicy: {
-        enabled: false,
+        enabled: false
      },
-      version: "1.0",
+      version: "1.0"
    },
    logging: {
      deleteProperty: true,
      read: true,
      retentionPolicy: {
-        enabled: false,
+        enabled: false
      },
      version: "1.0",
-      write: true,
+      write: true
    },
    minuteMetrics: {
      enabled: false,
      retentionPolicy: {
-        enabled: false,
+        enabled: false
      },
-      version: "1.0",
+      version: "1.0"
    },
    staticWebsite: {
-      enabled: false,
-    },
+      enabled: false
+    }
  };

  public constructor(
@ -78,7 +91,7 @@ export default class LokiBlobDataStore implements IBlobDataStore {
  ) {
    this.db = new Loki(lokiDBPath, {
      autosave: true,
-      autosaveInterval: 5000,
+      autosaveInterval: 5000
    });
  }

@ -87,7 +100,7 @@ export default class LokiBlobDataStore implements IBlobDataStore {
    await new Promise<void>((resolve, reject) => {
      stat(this.lokiDBPath, (statError, stats) => {
        if (!statError) {
-          this.db.loadDatabase({}, (dbError) => {
+          this.db.loadDatabase({}, dbError => {
            if (dbError) {
              reject(dbError);
            } else {
@ -139,7 +152,7 @@ export default class LokiBlobDataStore implements IBlobDataStore {
    }

    await new Promise((resolve, reject) => {
-      this.db.saveDatabase((err) => {
+      this.db.saveDatabase(err => {
        if (err) {
          reject(err);
        } else {
@ -234,7 +247,10 @@ export default class LokiBlobDataStore implements IBlobDataStore {
  public async deleteContainer(container: string): Promise<void> {
    const coll = this.db.getCollection(this.CONTAINERS_COLLECTION);
    const doc = coll.by("name", container);
-    coll.remove(doc);
+
+    if (doc) {
+      coll.remove(doc);
+    }

    // Following line will remove all blobs documents under that container
    this.db.removeCollection(container);
@ -395,7 +411,7 @@ export default class LokiBlobDataStore implements IBlobDataStore {
   */
  public async close(): Promise<void> {
    return new Promise<void>((resolve, reject) => {
-      this.db.close((err) => {
+      this.db.close(err => {
        if (err) {
          reject(err);
        } else {