[Tables] Support bigint (#15275)

* First stab at supporting bigint

* Address PR comments

* Fix lint issues

* expose pipeline

* Update CI and API Surface

* Try excluding node8

* TestType filter

* Display name filter

* Remove variables

* Keep jobMatrixFilter

sdk/tables/ci.yml

@@ -23,6 +23,9 @@ pr:
     include:
       - sdk/tables/
 
+variables:
+    jobMatrixFilter: ^((?!8x_node).)*$
+
 extends:
   template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml
   parameters:

sdk/tables/data-tables/review/data-tables.api.md

@@ -7,6 +7,7 @@
 import { CommonClientOptions } from '@azure/core-client';
 import { OperationOptions } from '@azure/core-client';
 import { PagedAsyncIterableIterator } from '@azure/core-paging';
+import { Pipeline } from '@azure/core-rest-pipeline';
 import { PipelinePolicy } from '@azure/core-rest-pipeline';
 
 // @public
@@ -69,6 +70,7 @@ export type GetStatisticsResponse = ServiceGetStatisticsHeaders & TableServiceSt
 // @public
 export type GetTableEntityOptions = OperationOptions & {
     queryOptions?: TableEntityQueryOptions;
+    disableTypeConversion?: boolean;
 };
 
 // @public
@@ -87,6 +89,7 @@ export const enum KnownGeoReplicationStatusType {
 // @public
 export type ListTableEntitiesOptions = OperationOptions & {
     queryOptions?: TableEntityQueryOptions;
+    disableTypeConversion?: boolean;
 };
 
 // @public
@@ -180,6 +183,7 @@ export class TableClient {
     getAccessPolicy(options?: OperationOptions): Promise<GetAccessPolicyResponse>;
     getEntity<T extends object = Record<string, unknown>>(partitionKey: string, rowKey: string, options?: GetTableEntityOptions): Promise<GetTableEntityResponse<TableEntityResult<T>>>;
     listEntities<T extends object = Record<string, unknown>>(options?: ListTableEntitiesOptions): PagedAsyncIterableIterator<TableEntityResult<T>, TableEntityResult<T>[]>;
+    pipeline: Pipeline;
     setAccessPolicy(tableAcl: SignedIdentifier[], options?: OperationOptions): Promise<SetAccessPolicyResponse>;
     submitTransaction(actions: TransactionAction[]): Promise<TableTransactionResponse>;
     readonly tableName: string;
@@ -286,6 +290,7 @@ export class TableServiceClient {
     getProperties(options?: OperationOptions): Promise<GetPropertiesResponse>;
     getStatistics(options?: OperationOptions): Promise<GetStatisticsResponse>;
     listTables(options?: ListTableItemsOptions): PagedAsyncIterableIterator<TableItem, TableItem[]>;
+    pipeline: Pipeline;
     setProperties(properties: ServiceProperties, options?: SetPropertiesOptions): Promise<SetPropertiesResponse>;
     url: string;
 }

sdk/tables/data-tables/samples-dev/workingWithBigInt.ts

@@ -0,0 +1,52 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+/**
+ * This sample demonstrates how to create and consume Int64 values using bigint
+ *
+ * @summary creates and works with an entity containing a bigint
+ * @azsdk-weight 70
+ */
+
+import { TableClient, TablesSharedKeyCredential } from "@azure/data-tables";
+// Load the .env file if it exists
+import * as dotenv from "dotenv";
+dotenv.config();
+
+const tablesUrl = process.env["TABLES_URL"] || "";
+const accountName = process.env["ACCOUNT_NAME"] || "";
+const accountKey = process.env["ACCOUNT_KEY"] || "";
+
+async function workingWithBigint() {
+  console.log("Working with bigint sample");
+  const client = new TableClient(
+    tablesUrl,
+    "testbigint",
+    new TablesSharedKeyCredential(accountName, accountKey)
+  );
+
+  await client.createTable();
+
+  type FooEntity = {
+    foo: bigint;
+  };
+
+  await client.createEntity<FooEntity>({ partitionKey: "p1", rowKey: "1", foo: BigInt("12345") });
+
+  const entity = await client.getEntity<FooEntity>("p1", "1");
+
+  // Do arithmetic operations with bigint
+  const resultPlusOne = entity.foo + BigInt(1);
+
+  console.log(resultPlusOne);
+
+  await client.deleteTable();
+}
+
+export async function main() {
+  await workingWithBigint();
+}
+
+main().catch((err) => {
+  console.error("The sample encountered an error:", err);
+});

sdk/tables/data-tables/samples-dev/workingWithInt64.ts

@@ -0,0 +1,57 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+/**
+ * This sample demonstrates how to create and consume Int64 values
+ *
+ * @summary creates and works with an entity containing an Int64 value
+ * @azsdk-weight 70
+ */
+
+import { Edm, TableClient, TablesSharedKeyCredential } from "@azure/data-tables";
+// Load the .env file if it exists
+import * as dotenv from "dotenv";
+dotenv.config();
+
+const tablesUrl = process.env["TABLES_URL"] || "";
+const accountName = process.env["ACCOUNT_NAME"] || "";
+const accountKey = process.env["ACCOUNT_KEY"] || "";
+
+async function workingWithInt64() {
+  console.log("working with Int64 sample");
+  const client = new TableClient(
+    tablesUrl,
+    "testInt64",
+    new TablesSharedKeyCredential(accountName, accountKey)
+  );
+
+  await client.createTable();
+
+  type FooEntity = {
+    foo: Edm<"Int64">;
+  };
+
+  await client.createEntity<FooEntity>({
+    partitionKey: "p1",
+    rowKey: "1",
+    // To work with Int64 we need to use an object that includes
+    // the value as a string and a notation of the type, in this case Int64
+    foo: { value: "12345", type: "Int64" }
+  });
+
+  const entity = await client.getEntity<FooEntity>("p1", "1", { disableTypeConversion: true });
+
+  // In order to do arithmetic operations with Int64 you need to use
+  // bigint or a third party library such as 'long'
+  console.log(entity);
+
+  await client.deleteTable();
+}
+
+export async function main() {
+  await workingWithInt64();
+}
+
+main().catch((err) => {
+  console.error("The sample encountered an error:", err);
+});

sdk/tables/data-tables/src/TableClient.ts

@@ -52,6 +52,7 @@ import {
 } from "./utils/internalModels";
 import { Uuid } from "./utils/uuid";
 import { parseXML, stringifyXML } from "@azure/core-xml";
+import { Pipeline } from "@azure/core-rest-pipeline";
 
 /**
  * A TableClient represents a Client to the Azure Tables service allowing you
@@ -62,6 +63,11 @@ export class TableClient {
    * Table Account URL
    */
   public url: string;
+  /**
+   * Represents a pipeline for making a HTTP request to a URL.
+   * Pipelines can have multiple policies to manage manipulating each request before and after it is made to the server.
+   */
+  public pipeline: Pipeline;
   private table: Table;
   private credential: TablesSharedKeyCredentialLike | undefined;
   private interceptClient: TableClientLike | undefined;
@@ -183,6 +189,7 @@ export class TableClient {
       generatedClient.pipeline.addPolicy(tablesSharedKeyCredentialPolicy(credential));
     }
     this.table = generatedClient.table;
+    this.pipeline = generatedClient.pipeline;
   }
 
   /**
@@ -250,13 +257,16 @@ export class TableClient {
     }
 
     try {
-      const { queryOptions, ...getEntityOptions } = updatedOptions || {};
+      const { disableTypeConversion, queryOptions, ...getEntityOptions } = updatedOptions || {};
       await this.table.queryEntitiesWithPartitionAndRowKey(this.tableName, partitionKey, rowKey, {
         ...getEntityOptions,
         queryOptions: this.convertQueryOptions(queryOptions || {}),
         onResponse
       });
-      const tableEntity = deserialize<TableEntityResult<T>>(parsedBody);
+      const tableEntity = deserialize<TableEntityResult<T>>(
+        parsedBody,
+        disableTypeConversion ?? false
+      );
 
       return tableEntity;
     } catch (e) {
@@ -348,9 +358,10 @@ export class TableClient {
 
   private async _listEntities<T extends object>(
     tableName: string,
-    options?: InternalListTableEntitiesOptions
+    options: InternalListTableEntitiesOptions = {}
   ): Promise<ListEntitiesResponse<TableEntityResult<T>>> {
-    const queryOptions = this.convertQueryOptions(options?.queryOptions || {});
+    const { disableTypeConversion = false } = options;
+    const queryOptions = this.convertQueryOptions(options.queryOptions || {});
     const {
       xMsContinuationNextPartitionKey: nextPartitionKey,
       xMsContinuationNextRowKey: nextRowKey,
@@ -360,7 +371,10 @@ export class TableClient {
       queryOptions
     });
 
-    const tableEntities = deserializeObjectsArray<TableEntityResult<T>>(value || []);
+    const tableEntities = deserializeObjectsArray<TableEntityResult<T>>(
+      value ?? [],
+      disableTypeConversion
+    );
 
     return Object.assign([...tableEntities], {
       nextPartitionKey,
@@ -678,6 +692,12 @@ interface InternalListTableEntitiesOptions extends ListTableEntitiesOptions {
    * An entity query continuation token from a previous call.
    */
   nextRowKey?: string;
+  /**
+   * If true, automatic type conversion will be disabled and entity properties will
+   * be represented by full metadata types. For example, an Int32 value will be \{value: "123", type: "Int32"\} instead of 123.
+   * This option applies for all the properties
+   */
+  disableTypeConversion?: boolean;
 }
 
 function isInternalClientOptions(options: any): options is InternalTransactionClientOptions {

sdk/tables/data-tables/src/TableServiceClient.ts

@@ -29,6 +29,7 @@ import { createSpan } from "./utils/tracing";
 import { tablesSharedKeyCredentialPolicy } from "./TablesSharedKeyCredentialPolicy";
 import { parseXML, stringifyXML } from "@azure/core-xml";
 import { ListTableItemsResponse } from "./utils/internalModels";
+import { Pipeline } from "@azure/core-rest-pipeline";
 
 /**
  * A TableServiceClient represents a Client to the Azure Tables service allowing you
@@ -39,6 +40,11 @@ export class TableServiceClient {
    * Table Account URL
    */
   public url: string;
+  /**
+   * Represents a pipeline for making a HTTP request to a URL.
+   * Pipelines can have multiple policies to manage manipulating each request before and after it is made to the server.
+   */
+  public pipeline: Pipeline;
   private table: Table;
   private service: Service;
 
@@ -132,6 +138,7 @@ export class TableServiceClient {
     if (credential) {
       client.pipeline.addPolicy(tablesSharedKeyCredentialPolicy(credential));
     }
+    this.pipeline = client.pipeline;
     this.table = client.table;
     this.service = client.service;
   }

sdk/tables/data-tables/src/models.ts

@@ -120,6 +120,12 @@ export type ListTableEntitiesOptions = OperationOptions & {
    * Query options group
    */
   queryOptions?: TableEntityQueryOptions;
+  /**
+   * If true, automatic type conversion will be disabled and entity properties will
+   * be represented by full metadata types. For example, an Int32 value will be \{value: "123", type: "Int32"\} instead of 123.
+   * This option applies for all the properties
+   */
+  disableTypeConversion?: boolean;
 };
 
 /**
@@ -130,6 +136,12 @@ export type GetTableEntityOptions = OperationOptions & {
    * Parameter group
    */
   queryOptions?: TableEntityQueryOptions;
+  /**
+   * If true, automatic type conversion will be disabled and entity properties will
+   * be represented by full metadata types. For example, an Int32 value will be \{value: "123", type: "Int32"\} instead of 123.
+   * This option applies for all the properties
+   */
+  disableTypeConversion?: boolean;
 };
 
 /**

sdk/tables/data-tables/src/serialization.ts

@@ -21,7 +21,7 @@ const Edm = {
   String: "Edm.String"
 } as const;
 
-type supportedTypes = boolean | string | number | Date | Uint8Array;
+type supportedTypes = boolean | string | number | Date | Uint8Array | bigint;
 
 type serializedType = {
   value: supportedTypes;
@@ -38,6 +38,9 @@ function serializePrimitive(value: any): serializedType {
     typeof value === "number"
   ) {
     serializedValue.value = value;
+  } else if (typeof value === "bigint") {
+    serializedValue.value = value.toString();
+    serializedValue.type = Edm.Int64;
   } else if (value instanceof Date) {
     serializedValue.value = value;
     serializedValue.type = Edm.DateTime;
@@ -105,27 +108,33 @@ export function serialize(obj: object): object {
   return serialized;
 }
 
-function getTypedObject(value: any, type: string): any {
+function getTypedObject(value: any, type: string, disableTypeConversion: boolean): any {
   switch (type) {
     case Edm.Boolean:
+      return disableTypeConversion ? { value, type: "Boolean" } : value;
     case Edm.Double:
+      return disableTypeConversion ? { value, type: "Double" } : value;
     case Edm.Int32:
+      return disableTypeConversion ? { value, type: "Int32" } : value;
     case Edm.String:
-      return value;
+      return disableTypeConversion ? { value, type: "String" } : value;
     case Edm.DateTime:
-      return { value, type: "DateTime" };
+      return disableTypeConversion ? { value, type: "DateTime" } : new Date(value);
     case Edm.Int64:
-      return { value, type: "Int64" };
+      return disableTypeConversion ? { value, type: "Int64" } : BigInt(value);
     case Edm.Guid:
       return { value, type: "Guid" };
     case Edm.Binary:
-      return base64Decode(value);
+      return disableTypeConversion ? { value, type: "Binary" } : base64Decode(value);
     default:
       throw new Error(`Unknown EDM type ${type}`);
   }
 }
 
-export function deserialize<T extends object>(obj: object): T {
+export function deserialize<T extends object = Record<string, any>>(
+  obj: object,
+  disableTypeConversion: boolean = false
+): T {
   const deserialized: any = {};
   for (const [key, value] of Object.entries(obj)) {
     if (key.indexOf("@odata.type") === -1) {
@@ -133,7 +142,7 @@ export function deserialize<T extends object>(obj: object): T {
       let typedValue = value;
       if (`${key}@odata.type` in obj) {
         const type = (obj as any)[`${key}@odata.type`];
-        typedValue = getTypedObject(value, type);
+        typedValue = getTypedObject(value, type, disableTypeConversion);
       }
       deserialized[transformedKey] = typedValue;
     }
@@ -141,6 +150,9 @@ export function deserialize<T extends object>(obj: object): T {
   return deserialized;
 }
 
-export function deserializeObjectsArray<T extends object>(objArray: object[]): T[] {
-  return objArray.map((obj) => deserialize<T>(obj));
+export function deserializeObjectsArray<T extends object>(
+  objArray: object[],
+  disableTypeConversion: boolean
+): T[] {
+  return objArray.map((obj) => deserialize<T>(obj, disableTypeConversion));
 }

sdk/tables/data-tables/test/internal/serialization.spec.ts

@@ -163,22 +163,46 @@ describe("Deserializer", () => {
     assert.strictEqual(deserialized.int32Prop, int32Value);
   });
 
-  it("should deserialize an Int64 value", () => {
+  it("should deserialize an Int64 value to bigint", () => {
     const int64Value = "12345678910";
-    const deserialized: Entity = deserialize<Entity>({
+    const deserialized = deserialize({
       int64ObjProp: int64Value,
       "int64ObjProp@odata.type": "Edm.Int64"
     });
-    assert.strictEqual(deserialized.int64ObjProp?.value, int64Value);
-    assert.strictEqual(deserialized.int64ObjProp?.type, "Int64");
+    assert.strictEqual(deserialized.int64ObjProp, BigInt(int64Value));
+  });
+
+  it("should not deserialize an Int64 when disableTypeConversion is true", () => {
+    const int64Value = "12345678910";
+    const deserialized = deserialize(
+      {
+        int64ObjProp: int64Value,
+        "int64ObjProp@odata.type": "Edm.Int64"
+      },
+      true
+    );
+    assert.strictEqual(deserialized.int64ObjProp.value, int64Value);
+    assert.strictEqual(deserialized.int64ObjProp.type, "Int64");
   });
 
   it("should deserialize a Date value", () => {
     const dateValue = new Date();
-    const deserialized = deserialize<{ dateProp: Edm<"DateTime"> }>({
+    const deserialized = deserialize({
       dateProp: dateValue.toJSON(),
       "dateProp@odata.type": "Edm.DateTime"
     });
+    assert.deepEqual(deserialized.dateProp, dateValue);
+  });
+
+  it("should not deserialize a Date value", () => {
+    const dateValue = new Date();
+    const deserialized = deserialize<{ dateProp: Edm<"DateTime"> }>(
+      {
+        dateProp: dateValue.toJSON(),
+        "dateProp@odata.type": "Edm.DateTime"
+      },
+      true
+    );
     assert.deepEqual(deserialized.dateProp, { type: "DateTime", value: dateValue.toISOString() });
   });
 

sdk/tables/data-tables/test/public/tableclient.spec.ts

@@ -172,10 +172,6 @@ describe("TableClient", () => {
     });
 
     it("should createEntity with Date", async () => {
-      type TestType = {
-        testField: Edm<"DateTime">;
-      };
-
       const testDate = "2020-09-17T00:00:00.111Z";
       const testEntity = {
         partitionKey: `P2_${suffix}`,
@@ -185,7 +181,7 @@ describe("TableClient", () => {
       let createResult: FullOperationResponse | undefined;
       let deleteResult: FullOperationResponse | undefined;
       await client.createEntity(testEntity, { onResponse: (res) => (createResult = res) });
-      const result = await client.getEntity<TestType>(testEntity.partitionKey, testEntity.rowKey);
+      const result = await client.getEntity(testEntity.partitionKey, testEntity.rowKey);
       await client.deleteEntity(testEntity.partitionKey, testEntity.rowKey, {
         onResponse: (res) => (deleteResult = res)
       });
@@ -194,7 +190,7 @@ describe("TableClient", () => {
       assert.equal(createResult?.status, 204);
       assert.equal(result.partitionKey, testEntity.partitionKey);
       assert.equal(result.rowKey, testEntity.rowKey);
-      assert.deepEqual(result.testField.value, testDate);
+      assert.deepEqual(result.testField, new Date(testDate));
     });
 
     it("should createEntity with Guid", async () => {
@@ -242,7 +238,7 @@ describe("TableClient", () => {
       let createResult: FullOperationResponse | undefined;
       let deleteResult: FullOperationResponse | undefined;
       await client.createEntity(testEntity, { onResponse: (res) => (createResult = res) });
-      const result = await client.getEntity<TestType>(testEntity.partitionKey, testEntity.rowKey);
+      const result = await client.getEntity(testEntity.partitionKey, testEntity.rowKey);
       await client.deleteEntity(testEntity.partitionKey, testEntity.rowKey, {
         onResponse: (res) => (deleteResult = res)
       });
@@ -250,7 +246,8 @@ describe("TableClient", () => {
       assert.equal(deleteResult?.status, 204);
       assert.equal(createResult?.status, 204);
       assert.equal(result.rowKey, testEntity.rowKey);
-      assert.deepEqual(result.testField, testInt64);
+      assert.equal(typeof result.testField, "bigint");
+      assert.deepEqual(result.testField, BigInt(testInt64.value));
     });
 
     it("should createEntity with Int32", async () => {
@@ -344,7 +341,9 @@ describe("TableClient", () => {
       let createResult: FullOperationResponse | undefined;
       let deleteResult: FullOperationResponse | undefined;
       await client.createEntity(testEntity, { onResponse: (res) => (createResult = res) });
-      const result = await client.getEntity<TestType>(testEntity.partitionKey, testEntity.rowKey);
+      const result = await client.getEntity<TestType>(testEntity.partitionKey, testEntity.rowKey, {
+        disableTypeConversion: true
+      });
       await client.deleteEntity(testEntity.partitionKey, testEntity.rowKey, {
         onResponse: (res) => (deleteResult = res)
       });