ReadME update

CosmosClone/CloneConsoleRun/App.config

@@ -7,17 +7,17 @@
   </configSections>
   
    <SourceCosmosDBSettings>
-    <add key="EndpointUrl" value="__EndPoint__" />
-    <add key="AccessKey" value="__Access__" />
+     <add key="EndpointUrl" value="__EndPoint__" />
+     <add key="AccessKey" value="__Access_" />
     <add key="DatabaseName" value="TestDB" />
-    <add key="CollectionName" value="TestSuperColl1" />
+    <add key="CollectionName" value="Sanity1" />
     <add key="ReadDelaybetweenRequestsInMs" value="2000" />
   </SourceCosmosDBSettings>
   <TargetCosmosDBSettings>
     <add key="EndpointUrl" value="__EndPoint__" />
-    <add key="AccessKey" value="__Access__" />
+    <add key="AccessKey" value="__Access_" />
     <add key="DatabaseName" value="TestDB" />
-    <add key="CollectionName" value="TestSuperColl1" />
+    <add key="CollectionName" value="Sanity5" />
     <add key="OfferThroughputRUs" value="10000" />
   </TargetCosmosDBSettings>
 
@@ -38,7 +38,7 @@
     <add key="CopyStoredProcedures" value="true" />
     <add key="CopyUDFs" value="true" />
     <add key="CopyTriggers" value="true" />
-    <add key="CopyDocuments" value="true" />
+    <add key="CopyDocuments" value="false" />
     <add key="CopyIndexingPolicy" value="true" />
     <add key="ReadBatchSize" value="3500" />
     <add key="CopyPartitionKey" value="true" />

CosmosClone/CloneConsoleRun/Program.cs

@@ -29,6 +29,7 @@ namespace CloneConsoleRun
                 //Update the app.config settings in the console project to run the below directly
                 //var documentMigrator = new CosmosCloneCommon.Migrator.DocumentMigrator();
                 //documentMigrator.StartCopy().Wait();
+                TestCosmosScrubbing();
 
                 //logger.LogInfo("Begin Code migration");
                 //var codeMigrator = new CosmosCloneCommon.Migrator.CodeMigrator();
@@ -67,8 +68,8 @@ namespace CloneConsoleRun
             //scrubRules.Add(rule3);
             //scrubRules.Add(rule4);            
             //ScrubRule rule5 = new ScrubRule("c.EntityType=\"External\"", "c.EmailAddress", RuleType.Singlevalue, "unknown@unknown.com", 4);
-            ScrubRule rule6 = new ScrubRule("c.id=\"51b6b28d-1c5f-4385-af4b-8dbb3dc45f65\"", "c.EmailAddress", RuleType.SingleValue, "unknown@unknown.com", 4);
-            //scrubRules.Add(rule6);
+            ScrubRule rule6 = new ScrubRule("c.id=\"2826d281-3a8b-4408-b064-efff26e26119\"", "c.EmailAddress", RuleType.SingleValue, "unknown@unknown.com", 4);
+            scrubRules.Add(rule6);
             var documentMigrator = new CosmosCloneCommon.Migrator.DocumentMigrator();
             documentMigrator.StartCopy(scrubRules).Wait();
             //var result = tcs.StartScrub(scrubRules);

CosmosClone/CosmicCloneUI/MainWindow.xaml.cs

@@ -279,9 +279,12 @@ namespace CosmicCloneUI
                 }
                 else
                 {
-
-                    readPercentProgress = (DocumentMigrator.TotalRecordsRetrieved * 100) / DocumentMigrator.TotalRecordsInSource;
-                    writePercentProgress = (DocumentMigrator.TotalRecordsSent * 100) / DocumentMigrator.TotalRecordsInSource;
+                    if(CloneSettings.CopyDocuments)
+                    {
+                        readPercentProgress = (DocumentMigrator.TotalRecordsRetrieved * 100) / DocumentMigrator.TotalRecordsInSource;
+                        writePercentProgress = (DocumentMigrator.TotalRecordsSent * 100) / DocumentMigrator.TotalRecordsInSource;
+                    }
+                    
                     if(CloneSettings.ScrubbingRequired && DocumentMigrator.scrubRules!=null && DocumentMigrator.scrubRules.Count>0)
                     {
                         scrubPercentProgress = DocumentMigrator.ScrubPercentProgress;

CosmosClone/CosmosCloneCommon/Utility/ObjectScrubber.cs

@@ -99,10 +99,11 @@ namespace CosmosCloneCommon.Utility
             if (token == null || token.Type == JTokenType.Null) return jTokenList;
 
             bool isLeaflevel = false;
-            for (int i = 1; i < propNames.Count; i++)
+
+            if(propNames.Count > 1)
             {
-                if (i == propNames.Count - 1) isLeaflevel = true;
-                var currentProperty = propNames[i];
+                if (propNames.Count == 2) isLeaflevel = true;
+                var currentProperty = propNames[1];
 
                 if (token.Type == JTokenType.Array)
                 {
@@ -123,7 +124,7 @@ namespace CosmosCloneCommon.Utility
                         }
                         else
                         {
-                            getPropertyValues(jArray[k], propNames.GetRange(i, propNames.Count - i), ref jTokenList);
+                            getPropertyValues(jArray[k], propNames.GetRange(1, propNames.Count - 1), ref jTokenList);
                             continue;
                         }
                     }
@@ -133,8 +134,8 @@ namespace CosmosCloneCommon.Utility
                     var jObj = (JObject)token;
                     if (isLeaflevel == true)
                     {
-                        if (jObj[currentProperty] != null )
-                        {                            
+                        if (jObj[currentProperty] != null)
+                        {
                             jTokenList.Add(jObj[currentProperty]);
                         }
                         else
@@ -143,12 +144,12 @@ namespace CosmosCloneCommon.Utility
                         }
                     }
                     else
-                    {                        
-                        getPropertyValues((JToken)jObj[currentProperty], propNames.GetRange(i, propNames.Count - i), ref jTokenList);
+                    {
+                        getPropertyValues((JToken)jObj[currentProperty], propNames.GetRange(1, propNames.Count - 1), ref jTokenList);
                     }
                 }
-                break;
-            }
+
+            } 
             return jTokenList;
         }
         public JToken getDocumentShuffledToken(JToken token, List<string> propNames, ref Queue<JToken> tokenQ)
@@ -157,10 +158,11 @@ namespace CosmosCloneCommon.Utility
 
             JToken jTokenResult = token;//just to initialize
             bool isLeaflevel = false;
-            for (int i = 1; i < propNames.Count; i++)
+            if (propNames.Count > 1)
             {
-                if (i == propNames.Count - 1) isLeaflevel = true;
-                var currentProperty = propNames[i];
+                if (propNames.Count == 2) isLeaflevel = true;
+
+                var currentProperty = propNames[1];
 
                 if (token.Type == JTokenType.Array)
                 {
@@ -177,7 +179,7 @@ namespace CosmosCloneCommon.Utility
                         }
                         else
                         {
-                            jArray[k] = getDocumentShuffledToken(jArray[k], propNames.GetRange(i, propNames.Count - i), ref tokenQ);
+                            jArray[k] = getDocumentShuffledToken(jArray[k], propNames.GetRange(1, propNames.Count - 1), ref tokenQ);
                             continue;
                         }
                     }
@@ -196,13 +198,12 @@ namespace CosmosCloneCommon.Utility
                     }
                     else
                     {
-                        jObj[currentProperty] = getDocumentShuffledToken((JToken)jObj[currentProperty], propNames.GetRange(i, propNames.Count - i), ref tokenQ);
+                        jObj[currentProperty] = getDocumentShuffledToken((JToken)jObj[currentProperty], propNames.GetRange(1, propNames.Count - 1), ref tokenQ);
                     }
                     var str3 = jObj.ToString();
                     jTokenResult = (JToken)jObj;
                 }
-                break;
-            }
+            }               
             if (jTokenResult == null)
             {
                 jTokenResult = token;
@@ -216,10 +217,12 @@ namespace CosmosCloneCommon.Utility
 
             JToken jTokenResult=token;//just to initialize
             bool isLeaflevel = false;
-            for (int i = 1; i < propNames.Count; i++)
+
+            if (propNames.Count > 1)
             {
-                if (i == propNames.Count - 1) isLeaflevel = true;
-                var currentProperty = propNames[i];
+                if (propNames.Count == 2) isLeaflevel = true;
+
+                var currentProperty = propNames[1];
 
                 if (token.Type == JTokenType.Array)
                 {
@@ -238,7 +241,7 @@ namespace CosmosCloneCommon.Utility
                         {
                             if (jArray[k] != null && jArray[k][currentProperty].Type != JTokenType.Null)
                             {
-                                jArray[k] = getUpdatedJsonArrayValue(jArray[k], propNames.GetRange(i, propNames.Count - i), overwritevalue);
+                                jArray[k] = getUpdatedJsonArrayValue(jArray[k], propNames.GetRange(1, propNames.Count - 1), overwritevalue);
                                 continue;
                             }
                             //else return null;
@@ -252,7 +255,7 @@ namespace CosmosCloneCommon.Utility
                     var jObj = (JObject)token;
                     if (isLeaflevel == true)
                     {
-                        if(jObj[currentProperty] != null && jObj[currentProperty].Type != JTokenType.Null)
+                        if (jObj[currentProperty] != null && jObj[currentProperty].Type != JTokenType.Null)
                         {
                             jObj[currentProperty] = overwritevalue;
                         }
@@ -261,15 +264,15 @@ namespace CosmosCloneCommon.Utility
                     {
                         if (jObj[currentProperty] != null && jObj[currentProperty].Type != JTokenType.Null)
                         {
-                            jObj[currentProperty] = getUpdatedJsonArrayValue((JToken)jObj[currentProperty], propNames.GetRange(i, propNames.Count - i), overwritevalue);
+                            jObj[currentProperty] = getUpdatedJsonArrayValue((JToken)jObj[currentProperty], propNames.GetRange(1, propNames.Count - 1), overwritevalue);
                         }
                         //else return null;
                     }
                     var str3 = jObj.ToString();
                     jTokenResult = (JToken)jObj;
                 }
-                break;
             }
+
             if(jTokenResult == null)
             {
                 jTokenResult = token;

README.md

@@ -1,14 +1,93 @@
+# Cosmic Clone
+1. [Overview](#overview)
+1. [Prerequisites](#prerequisites)
+1. [Deployment Steps](#deployment-steps)
+1. [Screens](#screens)
+1. [Todos](#todos)
+1. [References](#references)
+1. [Contributing](#contributing)
 
-# Contributing
 
-This project welcomes contributions and suggestions.  Most contributions require you to agree to a
-Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
-the rights to use your contribution. For details, visit https://cla.microsoft.com.
+## Overview
+Cosmic Clone is a tool to clone\backup\restore and anonymize data in an azure Cosmos Collection.
+As more applications begin to use Cosmos database, self serve capabilities such as backup, restore collection have become more essential.
+Cosmos Clone is an attempt to create a simple utility that allows to clone a Cosmos Collection.
+The utility helps in below
+*	Clone collections for QA, testing and other non production env.
+*	Backup data of a collection.
+*	Create collections with similar settings(indexes, partition, TTL etc)
+*	Anonymize data through scrubbing or shuffling of sensitive data in documents.
 
-When you submit a pull request, a CLA-bot will automatically determine whether you need to provide
-a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions
-provided by the bot. You will only need to do this once across all repos using our CLA.
+## Prerequisites
+- Azure Cosmos SQL API DB Source Collection with Data(read connection keys)
+- Azure Cosmos Destination account(read write connection keys)
 
-This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
-For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
-contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
+
+## Deployment Steps
+1. Just Compile and Run the Code.
+2. For Best performance you can run the compiled code in an Azure VM that is in the same region as the source and destination Cosmos Colleciton.
+
+
+## Screens
+
+**Initial screen**
+
+![screen1](/docs/images/sinitial.png)
+
+**Enter Source connection details**
+
+![screen2](/docs/images/sinitialDetails.png)
+
+
+**Set migration options**
+
+![screen3](/docs/images/soptions.png)
+
+
+**Configure anonymization rules**
+
+![screen4](/docs/images/sAnonymize.png)
+
+**Sample rule1**
+
+![screen5](/docs/images/sRule1.png)
+
+**Sample rule2**
+
+![screen6](/docs/images/sRule2.png)
+Note there are options to validate, save and load these rules
+
+**Migration screen**
+
+![screen7](/docs/images/sprogress1.png)
+
+**Completion notification**
+
+![screen8](/docs/images/scomplete.png)
+
+**Before and After anonymization**
+
+![screen9](/docs/images/BeforeAfter.JPG)
+
+As can be inferred from above, documents will be sanitized based on rules.
+
+### Todos
+
+ - Adapt to other Cosmos API like Graph and Cassandra apart from SQL API
+ - Parellilze read and write to improve efficiency
+ - Add anonymization option to mask with random values (predefined patterns and regular expressions)
+ - Refactor some of the UI and utility code
+ - Write more tests
+
+## References
+**Static data masking**
+https://docs.microsoft.com/en-us/sql/relational-databases/security/static-data-masking?view=sql-server-2017
+
+
+## Contributing
+ [Contribution guidelines for this project](docs/CONTRIBUTING.md)
+ 
+License
+----
+
+MIT

docs/CONTRIBUTING.md

@@ -0,0 +1,14 @@
+
+# Contributing
+
+This project welcomes contributions and suggestions.  Most contributions require you to agree to a
+Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
+the rights to use your contribution. For details, visit https://cla.microsoft.com.
+
+When you submit a pull request, a CLA-bot will automatically determine whether you need to provide
+a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions
+provided by the bot. You will only need to do this once across all repos using our CLA.
+
+This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
+For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
+contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.