Merge pull request #10 from Azure/convert_types

Enable implicit Parquet to Kusto types conversion

Package.nuspec

@@ -2,14 +2,14 @@
 <package >
   <metadata>
     <id>pq2json</id>
-    <version>0.1.7</version>
+    <version>0.1.8</version>
     <authors>Evgeney Ryzhyk</authors>
     <owners>Evgeney Ryzhyk</owners>
     <license type="expression">MIT</license>
     <projectUrl>https://github.com/Azure/azure-kusto-parquet-conv</projectUrl>
     <requireLicenseAcceptance>false</requireLicenseAcceptance>
     <description>Parquet to JSON (line delimited) converter tool.</description>
-    <releaseNotes>Support numeric keys in MAP type</releaseNotes>
+    <releaseNotes>Added configuration option that enables implicit Parquet to Kusto types conversion.</releaseNotes>
     <copyright>Copyright 2020</copyright>
     <tags></tags>
     <dependencies></dependencies>

pq2json/src/converter.rs

@@ -97,7 +97,7 @@ macro_rules! element_to_value {
             FieldType::UByte => Value::Number($obj.get_ubyte($i)?.into()),
             FieldType::UShort => Value::Number($obj.get_ushort($i)?.into()),
             FieldType::UInt => Value::Number($obj.get_uint($i)?.into()),
-            FieldType::ULong => Value::Number($obj.get_ulong($i)?.into()),
+            FieldType::ULong => ulong_to_value($obj.get_ulong($i)?, &$settings),
             FieldType::Float => float_to_value($obj.get_float($i)? as f64),
             FieldType::Double => float_to_value($obj.get_double($i)?),
             FieldType::Decimal => Value::String(decimal_to_string($obj.get_decimal($i)?)),
@@ -258,6 +258,14 @@ fn float_to_value(f: f64) -> Value {
         .unwrap_or_else(|| Value::Null)
 }
 
+fn ulong_to_value(l: u64, settings: &Settings) -> Value {
+    if settings.convert_types {
+        Value::Number((l as i64).into())
+    } else {
+        Value::Number(l.into())
+    }
+}
+
 const TICKS_TILL_UNIX_TIME: u64 = 621355968000000000u64;
 
 fn timestamp_to_value(settings: &Settings, ts: u64) -> Result<Value, Box<dyn Error>> {

pq2json/src/main.rs

@@ -30,6 +30,16 @@ fn main() {
                 .takes_value(false)
                 .required(false),
         )
+        .arg(
+            Arg::with_name("convert-types")
+                .short("r")
+                .long("convert-types")
+                .help(
+                    "Implicit Parquet to Kusto types conversion (e.g. U64 into long)"
+                )
+                .takes_value(false)
+                .required(false),
+        )
         .arg(
             Arg::with_name("prune")
                 .short("p")
@@ -123,6 +133,7 @@ fn main() {
         omit_empty_bags: matches.is_present("omit-empty-bags") || matches.is_present("prune"),
         timestamp_rendering,
         omit_empty_lists: matches.is_present("omit-empty-lists") || matches.is_present("prune"),
+        convert_types: matches.is_present("convert-types"),
         columns: matches
             .value_of("columns")
             .map(|columns| columns.split(",").map(|s| s.to_string()).collect()),

pq2json/src/settings.rs

@@ -3,6 +3,7 @@ pub struct Settings {
     pub omit_nulls: bool,
     pub omit_empty_bags: bool,
     pub omit_empty_lists: bool,
+    pub convert_types: bool,
     pub timestamp_rendering: TimestampRendering,
     pub columns: Option<Vec<String>>,
     pub csv: bool,