Small mapping fixes (#166)

* small mapping fixes

* Added IngestionMappingKind alias

* -Added back ingestionMappingType as deprecated
-Deprecate IngestionPropertiesEnums for direct imports

* -Added back ingestionMapping field, as deprecated

azure-kusto-ingest/README.md

@@ -10,7 +10,7 @@
 const IngestClient = require("azure-kusto-ingest").IngestClient;
 const IngestionProps = require("azure-kusto-ingest").IngestionProperties;
 const KustoConnectionStringBuilder = require("azure-kusto-data").KustoConnectionStringBuilder;
-const { DataFormat, JsonColumnMapping } = require("azure-kusto-ingest").IngestionPropertiesEnums;
+const { DataFormat, JsonColumnMapping } = require("azure-kusto-ingest");
 
 const kcsb = KustoConnectionStringBuilder.withAadApplicationKeyAuthentication(`https://ingest-${cluster}.kusto.windows.net`, appId, appKey, authorityId);
 
@@ -179,9 +179,9 @@ For Example:
 const IngestClient = require("azure-kusto-ingest").IngestClient;
 const IngestStatusQueues = require("azure-kusto-ingest").IngestStatusQueues;
 const IngestionProps = require("azure-kusto-ingest").IngestionProperties;
-const { ReportLevel, ReportMethod } = require("azure-kusto-ingest").IngestionPropertiesEnums;
+const { ReportLevel, ReportMethod } = require("azure-kusto-ingest");
 const KustoConnectionStringBuilder = require("azure-kusto-data").KustoConnectionStringBuilder;
-const { DataFormat, JsonColumnMapping } = require("azure-kusto-ingest").IngestionPropertiesEnums;
+const { DataFormat, JsonColumnMapping } = require("azure-kusto-ingest");
 const fs = require("fs");
 
 

azure-kusto-ingest/example.js

@@ -4,9 +4,8 @@
 const IngestClient = require("azure-kusto-ingest").IngestClient;
 const IngestStatusQueues = require("azure-kusto-ingest").IngestStatusQueues;
 const IngestionProps = require("azure-kusto-ingest").IngestionProperties;
-const { ReportLevel, ReportMethod } = require("azure-kusto-ingest").IngestionPropertiesEnums;
 const KustoConnectionStringBuilder = require("azure-kusto-data").KustoConnectionStringBuilder;
-const { DataFormat, JsonColumnMapping, IngestionMappingType, CompressionType } = require("azure-kusto-ingest").IngestionPropertiesEnums;
+const { DataFormat, JsonColumnMapping, IngestionMappingKind, CompressionType, ReportLevel, ReportMethod  } = require("azure-kusto-ingest");
 const { BlobDescriptor, StreamDescriptor } = require("azure-kusto-ingest").IngestionDescriptors;
 const StreamingIngestClient = require("azure-kusto-ingest").StreamingIngestClient;
 const fs = require('fs');
@@ -25,7 +24,7 @@ const props = new IngestionProps({
         new JsonColumnMapping("TargetColumn2", "$.sourceProp2"),
         new JsonColumnMapping("TargetColumn3", "$.sourceProp3")
     ],
-    ingestionMappingType: IngestionMappingType.JSON,
+    ingestionMappingType: IngestionMappingKind.JSON,
     reportLevel: ReportLevel.FailuresAndSuccesses,
     reportMethod: ReportMethod.Queue
 
@@ -42,12 +41,12 @@ const statusQueues = new IngestStatusQueues(ingestClient);
 
 startIngestion();
 
-// Streaming ingest client 
+// Streaming ingest client
 const props2 = new IngestionProps({
     database: "Database",
     table: "Table",
     format: DataFormat.JSON,
-    ingestionMappingReference: "Pre-defiend mapping name" // For json format mapping is required
+    ingestionMappingReference: "Pre-defined mapping name"
 });
 
 // Init with engine endpoint

azure-kusto-ingest/index.ts

@@ -10,7 +10,8 @@ import managedStreamingIngestClient from "./source/managedStreamingIngestClient"
 import KustoIngestStatusQueues from "./source/status";
 
 import {
-    DataFormat, IngestionMappingKind,
+    DataFormat,
+    IngestionMappingKind,
     ReportLevel,
     ReportMethod,
     ValidationImplications,
@@ -19,13 +20,22 @@ import {
 } from "./source/ingestionProperties";
 
 
+import { BlobDescriptor, CompressionType, FileDescriptor, StreamDescriptor } from "./source/descriptors";
 import {
-    BlobDescriptor,
-    CompressionType,
-    FileDescriptor,
-    StreamDescriptor
-} from "./source/descriptors";
-import { AvroColumnMapping, CsvColumnMapping, JsonColumnMapping, OrcColumnMapping, ParquetColumnMapping, W3CLogFileMapping } from "./source/columnMappings";
+    ApacheAvroColumnMapping,
+    AvroColumnMapping,
+    ColumnMapping,
+    ConstantTransformation,
+    CsvColumnMapping,
+    FieldTransformation,
+    JsonColumnMapping,
+    OrcColumnMapping,
+    ParquetColumnMapping,
+    SStreamColumnMapping,
+    W3CLogFileMapping,
+} from "./source/columnMappings";
+
+export { Transformation as ColumnMappingTransformation } from "./source/columnMappings";
 
 export const IngestClient = client;
 export const StreamingIngestClient = streamingIngestClient;
@@ -35,8 +45,34 @@ export { IngestionProperties } from "./source/ingestionProperties"
 export const IngestionDescriptors = {
     BlobDescriptor,
     FileDescriptor,
-    StreamDescriptor
+    StreamDescriptor,
 };
+
+export {
+    JsonColumnMapping,
+    CsvColumnMapping,
+    AvroColumnMapping,
+    ParquetColumnMapping,
+    OrcColumnMapping,
+    W3CLogFileMapping,
+    ValidationPolicy,
+    ReportLevel,
+    ReportMethod,
+    ValidationImplications,
+    ValidationOptions,
+    DataFormat,
+    IngestionMappingKind,
+    CompressionType,
+    ApacheAvroColumnMapping,
+    SStreamColumnMapping,
+    ConstantTransformation,
+    FieldTransformation,
+    ColumnMapping
+};
+
+/**
+ * @deprecated - import directly instead
+ */
 export const IngestionPropertiesEnums = {
     JsonColumnMapping,
     CsvColumnMapping,
@@ -50,8 +86,17 @@ export const IngestionPropertiesEnums = {
     ValidationImplications,
     ValidationOptions,
     DataFormat,
+    /**
+     * @deprecated - use IngestionMappingKind instead
+     */
     IngestionMappingType: IngestionMappingKind,
-    CompressionType
+    IngestionMappingKind,
+    CompressionType,
+    ApacheAvroColumnMapping,
+    SStreamColumnMapping,
+    ConstantTransformation,
+    FieldTransformation,
+    ColumnMapping,
 };
 
-export { IngestionPropertiesValidationError} from "./source/errors"
+export { IngestionPropertiesValidationError } from "./source/errors"

azure-kusto-ingest/source/ingestionProperties.ts

@@ -81,8 +81,6 @@ export enum DataFormat {
     W3CLogFile = "w3clogfile",
 }
 
-export const MappingRequiredFormats = Object.freeze([DataFormat.JSON, DataFormat.SINGLEJSON, DataFormat.AVRO, DataFormat.ORC])
-
 export enum IngestionMappingKind {
     CSV = "Csv",
     JSON = "Json",
@@ -149,8 +147,12 @@ export enum ValidationImplications {
 
 
 export class ValidationPolicy {
-    constructor(readonly validationOptions: ValidationOptions = ValidationOptions.DoNotValidate, readonly validationImplications: ValidationImplications = ValidationImplications.BestEffort) {
+    constructor(
+        readonly validationOptions: ValidationOptions = ValidationOptions.DoNotValidate,
+        readonly validationImplications: ValidationImplications = ValidationImplications.BestEffort
+    ) {
     }
+
     toJSON(): Record<string, number> {
         return {
             ValidationOptions: this.validationOptions,
@@ -169,12 +171,20 @@ export enum ReportMethod {
     Queue = 0
 }
 
-export class IngestionProperties{
+export class IngestionProperties {
     database?: string;
     table?: string;
     format: DataFormat = DataFormat.CSV;
+    /**
+     * @deprecated. Use ingestionMappingColumns instead.
+     */
+    ingestionMapping?: ColumnMapping[];
     ingestionMappingColumns?: ColumnMapping[];
     ingestionMappingReference?: string;
+    /**
+     * @deprecated. Use ingestionMappingKind instead.
+     */
+    ingestionMappingType?: IngestionMappingKind;
     ingestionMappingKind?: IngestionMappingKind;
     additionalTags?: string;
     ingestIfNotExists?: string;
@@ -195,6 +205,13 @@ export class IngestionProperties{
         if (!this.table) throw new IngestionPropertiesValidationError("Must define a target table");
         if (!this.format) throw new IngestionPropertiesValidationError("Must define a data format");
 
+        if (this.ingestionMappingType && !this.ingestionMappingKind) {
+            this.ingestionMappingKind = this.ingestionMappingType;
+        }
+
+        if (this.ingestionMapping && !this.ingestionMappingColumns) {
+            this.ingestionMappingColumns = this.ingestionMapping;
+        }
 
         if (!this.ingestionMappingColumns && !this.ingestionMappingReference) {
             if (this.ingestionMappingKind) {
@@ -202,9 +219,6 @@ export class IngestionProperties{
                     " ingestionMappingReference");
             }
 
-            if (MappingRequiredFormats.includes(this.format as DataFormat)) {
-                throw new IngestionPropertiesValidationError(`Mapping reference required for format '${this.format}'.`);
-            }
         } else {
             const mappingKind = dataFormatMappingKind(this.format);
             if (this.ingestionMappingKind && this.ingestionMappingKind !== mappingKind) {
@@ -231,8 +245,10 @@ export class IngestionProperties{
         const merged = new IngestionProperties(this);
 
         for (const key of Object.keys(extraProps) as (keyof IngestionProperties)[]) {
-            if ( extraProps[key]) {
-                (<K extends keyof IngestionProperties>(k: K) => { merged[k] = extraProps[k]; })(key);
+            if (extraProps[key]) {
+                (<K extends keyof IngestionProperties>(k: K) => {
+                    merged[k] = extraProps[k];
+                })(key);
             }
         }
 

azure-kusto-ingest/test/ingestionPropsTest.ts

@@ -51,11 +51,7 @@ describe("IngestionProperties", () => {
         it("valid input", () => {
             const props = new IngestionProperties({database: "db", table: "table", format: DataFormat.CSV, ingestionMappingReference: "CsvMappingRef"});
 
-            try {
-                props.validate();
-            } catch (ex: any) {
-                assert.fail(ex);
-            }
+            props.validate();
         });
 
         it("invalid input", () => {
@@ -65,17 +61,15 @@ describe("IngestionProperties", () => {
                 props.validate();
             } catch (ex: any) {
                 assert.strictEqual(ex.message, "Must define a target database");
+                return;
             }
+            assert.fail("Expected an exception")
         });
 
-        it("invalid input json", () => {
+        it("json without mapping should succeed", () => {
             const props = new IngestionProperties({database: "db", table: "table", format: DataFormat.JSON});
 
-            try {
-                props.validate();
-            } catch (ex: any) {
-                assert.strictEqual(ex.message, "Mapping reference required for format 'json'.");
-            }
+            props.validate();
         });
 
         it("Should error when mapping object doesn't match mapping type", () => {
@@ -85,7 +79,9 @@ describe("IngestionProperties", () => {
                 props.validate();
             } catch (ex: any) {
                 assert.strictEqual(ex.message, "Invalid columns:\nMapping kind mismatch for column 'a' - expected data format kind -  'Csv', but was 'Json'");
+                return;
             }
+            assert.fail("Expected an exception")
         })
 
         it("Should error when mapping object doesn't match mapping type multiple objects", () => {
@@ -104,7 +100,10 @@ describe("IngestionProperties", () => {
             } catch (ex: any) {
                 assert.strictEqual(ex.message, "Invalid columns:\nMapping kind mismatch for column 'b' - expected data format kind -  'Csv', but was" +
                     " 'Json'\nMapping kind mismatch for column 'c' - expected data format kind -  'Csv', but was 'Avro'");
+                return;
             }
+            assert.fail("Expected an exception")
+
         })
 
 
@@ -115,7 +114,9 @@ describe("IngestionProperties", () => {
                 props.validate();
             } catch (ex: any) {
                 assert.strictEqual(ex.message, "Mapping kind 'Json' does not match format 'csv' (should be 'Csv')");
+                return;
             }
+            assert.fail("Expected an exception")
         })
 
         it("Should error when format doesn't match implicit mapping type", () => {
@@ -125,7 +126,9 @@ describe("IngestionProperties", () => {
                 props.validate();
             } catch (ex: any) {
                 assert.strictEqual(ex.message, "Invalid columns:\nMapping kind mismatch for column 'a' - expected data format kind -  'Csv', but was 'Json'");
+                return;
             }
+            assert.fail("Expected an exception")
         })
 
         describe("Should return the correct mapping when passing Ordinal", () => {