Generate larger records and combine build scripts (#32)

* remove tracking of data.encrypted

* Add configuration property for size and number of records

* ignore encrypted files

* Combine generation scripts into one build.sh

* Clean-up generation code

.gitignore

@@ -108,3 +108,4 @@ app/cpp/program
 docker-env
 config.json.encoded
 config.json
+*.encrypted

app/config.py

@@ -18,6 +18,8 @@ class Config(object):
     service_principal_secret = ""
     redis_port = 6379,
     redis_host = "localhost"
+    number_of_records = 100
+    size_of_record_kb = 1
     job_processing_max_time_sec = 60,
     azure_keyvault_url = ""
     azure_keyvault_key_name = ""

app/functions.py

@@ -53,7 +53,7 @@ def processing_job(encryptedRecord, redisHost, redisPort):
     aes_cipher = _create_aes_cipher()
 
     # decrypt the data to be processed
-    record = int(aes_cipher.decrypt(base64.b64decode(encryptedRecord)))
+    record = aes_cipher.decrypt(base64.b64decode(encryptedRecord))
 
     # similuate CPU intensive process for 1 second
     start = datetime.utcnow()
@@ -64,7 +64,7 @@ def processing_job(encryptedRecord, redisHost, redisPort):
 
     # write out the results
     results = Results(LOGGER, redisHost, redisPort)
-    results.write_result(job.id, str(record))
+    results.write_result(job.id, record)
 
     # update the job status record
     jobstatus = JobStatus(LOGGER, redisHost, redisPort)

build.sh

@@ -1,18 +1,35 @@
 export PYTHONPATH=.
-python samples/CopyScriptsForArmTemplate.py
 
-# Cleanup docker-env
-rm -rf docker-env
-mkdir -p docker-env
+echo "This will create new AES key (if necessary), generate data, and upload it to Azure"
+echo "Do you want to continue? [y/n]"
+read data
+if [[ $data = y ]]
+then
+    # Generate & upload encryption key and encrypt data
+    python samples/buildKeyAndData.py
+fi
 
-# copy main tar file
-mv app.tar.gz docker-env/app.tar.gz
+echo "This will create a new app package and upload it to Azure and setup the local Docker Environment"
+echo "Do you want to continue? [y/n]"
+read upload
+if [[ $upload = y ]]
+then
+    # Upload scripts
+    python samples/buildScripts.py
 
-# Copy bootstrap scripts
-cp app/processor_bootstrap.sh docker-env/processor_bootstrap.sh
-cp app/scheduler_bootstrap.sh docker-env/scheduler_bootstrap.sh
+    # Cleanup docker-env
+    rm -rf docker-env
+    mkdir -p docker-env
 
-# Copy encoded config file to CustomData file
-mkdir -p docker-env/waagent
-cp config/config.json.encoded docker-env/waagent/CustomData
-read -p "Press any key to exit..."
+    # copy main tar file
+    mv app.tar.gz docker-env/app.tar.gz
+
+    # Copy bootstrap scripts
+    cp app/processor_bootstrap.sh docker-env/processor_bootstrap.sh
+    cp app/scheduler_bootstrap.sh docker-env/scheduler_bootstrap.sh
+
+    # Copy encoded config file to CustomData file
+    mkdir -p docker-env/waagent
+    cp config/config.json.encoded docker-env/waagent/CustomData
+    read -p "Press any key to exit..."
+fi

config/config.example.json

@@ -5,6 +5,8 @@
     "service_principal_secret": "",
     "redis_port": 6379,
     "redis_host": "localhost",
+    "number_of_records": 100,
+    "size_of_record_kb": 1,
     "job_processing_max_time_sec": 60,
     "aes_key_length": 32,
     "azure_keyvault_url": "",

data/data.encrypted

@@ -1,100 +0,0 @@
-soDS/wNhmA4MMMLLVyEV0A==
-RKkT2OL/VOYCRNKeOShm4A==
-pqXJZYvOFvyVog0Vf/NzqQ==
-YMv1GMzX781VzWItASzPfg==
-U5WkxTlMmvnMquTCnWyFbg==
-f3yi3/beZ5e2LdCklmmiGw==
-GU4ivs+9hhi8UCpYTIFWFw==
-j4JoIj9GtEv2QsSLLlzSGg==
-2f12EEtnu7EKGANI16drgA==
-SJw7ypcKHYDxEaPC35KObg==
-GnMSg+sDHNM0hUh8+kO0xQ==
-PO1zDxRRGzTWlm4A8a/6Mg==
-PLIIVPP50iC+SEzsxV1RPQ==
-np+eb36F9PQcfeMRjQeUMA==
-QNNkf86Nms9INPk4uwep6w==
-tQZhXY5B20SYFku9KERRsg==
-6FEoAse6R1j3/Czfs/6BmQ==
-P4a7Bx3Srw+FMNNetjBqbA==
-vwDf1+kvoC4FtCco8CNjgQ==
-ghXhn8ROgSiTeeENShVDIw==
-xLXj3XDVCWOKydFwBTid0A==
-Vy8a6b4eTXZRQBRJ7onsVA==
-YpNjyjjUipK892+sZg0Q1w==
-SfE2VC8c0oaVHLA9Fssz6w==
-ao8guMyL6gfJ4oZKuKpuSA==
-vL9H3FjKm64EbMKGs+CeCQ==
-bY9ozILkCxxTGfqrTaK0sA==
-t1VyUg83CsXudKqI+7sixQ==
-2FYO43cZYf3VriCtPU4XBQ==
-AKoC6ohXiWv6tPeYa0Is3g==
-tY34QG0w7G8dkUT3wyCAcw==
-+O8TS3dIRimeKhT/h7Dyqg==
-8iaO3Rb3VW/ySF8+fJB15g==
-XG17ZMarr4bIgGWQ6VFE3w==
-HCfva4yeMM0PjpLys9Z0VQ==
-95bA14Dno8wzwMcwfEAVyg==
-eKvNyzh2NdjPTMIP5dEcKg==
-XtrQxfSehWxr3xxvMiZjyQ==
-BsHxo0kEDZnQbZzH70Djbg==
-ze4SXtaWwQbWGuKzINl6HQ==
-rsJinE1UoVysrdS0egszxA==
-XBOtb5Zro/LknJ1I098TJg==
-HIygGIgJt5HJAJWPHarSOA==
-suLa3YqErIL0DNhh8XLAEw==
-lKHlceFiOHZhp9zAUv0OiQ==
-h5MihMPXoP7N009qI+Pecg==
-+ZrMTgrVVjUpJl2keZIxNg==
-6VpcGkTdklyuOg+ZHsXNEw==
-TVZySnwtFhrNUmjrggYCDw==
-/Ex31wiwtDrJMEPBLVgRcw==
-6cy5CbsMqT+ifGhnMnHjgA==
-aWDJcvsSfk8dP+YAJSwdDQ==
-vHAcRBn9jLbqNzofbNvM/w==
-f+BTbLHdyMQON8/ohmp/Eg==
-Pgko3rbW/qFhRJ+eNgHZsw==
-fAioXLC6T44KyHXzdU4hpQ==
-6XG6TVntguI2I1sXvWlG7A==
-iPKU9N5bIbKPHr17ruu6AQ==
-qLnmcieFyxLgVtdihKuOfA==
-QSiRmOdLTvMwu1M32uBawg==
-M1wBPNJGh/HsVmbBsR7gdw==
-mW26S17IOs1MlrkiIOFlSA==
-+sJDdEP1z82kRT68mspnAw==
-tCsEEGI0WkJS4469YMBPrg==
-ra9bzL2OQ7y8ei5LSYTkgQ==
-n9q1aMDKStz2hCdgaLGPKg==
-8QNRjr3GaYh4z/XrsZrwjQ==
-fTPPUUToeiLm4EXOGxmvhw==
-E3OpOa7/zb+nQrEp/eH98A==
-ny2W2nqTgry1vlY13R8Azw==
-s2RKFNYrXT8M6rXCog5Blg==
-vcd2x/It3lrEWm9YaC6wPA==
-upep+uPzZtvDLY/l7HrawQ==
-grF3hl9usyui0AFo9C7H3Q==
-9Ee6ovojHMCMyCad1vkRRA==
-s4CLpOYIVG/dYrrF9WIK7Q==
-uBOa0e2MRqiLrpTjPH55EQ==
-dUlLsfWM3r8KAE+9Yy31aQ==
-5Y//Xk7zWt+rgJAiTIeYyQ==
-7g0lEFhNw/DDKoTVO2PH6w==
-4v9vPYt6LgvcPSTio+HEAQ==
-HjU2kaoWnarhoo2pt3aKgw==
-V2dCLFTnDTqVD8tLdPCIcw==
-tSObbbmOFd9ZPWTMk0h8Uw==
-GQki6mqZuPz2Aw6enz4uVA==
-mAgk9CM8FaAf86ISjr7p9Q==
-d3UM3ghmA5+IGAB7BkVTNg==
-t8gp0rdYOSxHwLpEsV14rw==
-WwBX6aHJKJUDh2tB15eyOw==
-JB032AX+JLfLJ7ig3Bg5Lw==
-vcA2ir/zmVB34csFJ9O8Jg==
-33Zvf3qzFCmSXJzr82BaNw==
-ZLhooD91phLR8kpthtCexw==
-YEm0Q8oBs30pE+DwK1vpJw==
-kMdlbw6n4/sULhGXXzpkQg==
-+FPtMwiMeWsgDFxAmQWpzA==
-6Wy0lmdLgiB3dYVB3vdmUQ==
-u2SoRlnY/d9XmFbFSpGvpQ==
-CWkxuVtMz06kB9KEW98Qvg==
-8ohN8QtR+X65zjFdU168WQ==

generate_encrypted_files.sh

@@ -1,10 +0,0 @@
-echo "This will create new AES key and update files in Azure and locally."
-echo "Do you want to continue?"
-read choice
-
-if [[ $choice = y ]]
-then
-    set PYTHONPATH=.
-    python scripts/GenerateEncryptedFiles.py
-    read -p "Press any key to exit..."
-fi

samples/buildKeyAndData.py

@@ -8,13 +8,10 @@ from app.aescipher import AESCipher
 from app.aeshelper import AESHelper
 from app.aeskeywrapper import AESKeyWrapper
 from app.config import Config
-config = Config("app/config.json")
+config = Config()
 
-script_filename = "app/scheduler.py"
-script_encrypted_filename = os.path.join(config.encrypted_files_folder, config.encrypted_scheduler_script_filename)
 aes_key_encrypted_filename = os.path.join(config.encrypted_files_folder, config.encrypted_aes_key_filename)
 encrypted_record_file = os.path.join(config.encrypted_files_folder, config.encrypted_data_filename)
-number_of_records = 100
 
 wrapper = AESKeyWrapper(vault = config.azure_keyvault_url,
                         client_id = config.service_principal_client_id,
@@ -41,15 +38,9 @@ else:
 
     print 'AES key wrapped and saved to ' + aes_key_encrypted_filename
 
-    # Encrypt script using generated keys
-    cipher = AESCipher(aes_key, aes_iv)
-
-cipher.encrypt_file_save_file(script_filename, script_encrypted_filename)
-print script_filename + " encrypted and saved to " + script_encrypted_filename
-
 # Encrypt data
 data_generator = DataGenerator(config)
-data_generator.generate_data(number_of_records, os.path.join(config.encrypted_files_folder, "data.encrypted"))
+data_generator.generate_data(config.size_of_record_kb, config.number_of_records, os.path.join(config.encrypted_files_folder, config.encrypted_data_filename))
 
 print "Generated encrypted records file stored at: " + encrypted_record_file
 
@@ -62,10 +53,6 @@ blob_service.create_blob_from_path(container_name=config.storage_container_name,
                                    blob_name=config.encrypted_aes_key_filename,
                                    file_path=aes_key_encrypted_filename)
 
-blob_service.create_blob_from_path(container_name=config.storage_container_name,
-                                   blob_name=config.encrypted_scheduler_script_filename,
-                                   file_path=script_encrypted_filename)
-
 blob_service.create_blob_from_path(container_name=config.storage_container_name,
                                    blob_name=config.encrypted_data_filename,
                                    file_path=encrypted_record_file)

samples/buildScripts.py

@@ -14,7 +14,7 @@ from app.aeshelper import AESHelper
 config = Config('config/config.json')
 
 encrypted_aes_key_filename = "data/aes.encrypted"
-encrypted_script_filename = config.encrypted_files_folder + "/" + config.encrypted_scheduler_script_filename
+encrypted_script_filename = os.path.join(config.encrypted_files_folder, config.encrypted_scheduler_script_filename)
 
 files_to_encrypt = [
     "app/scheduler.py"

samples/dataGenerator.py

@@ -1,29 +1,44 @@
 import base64
+import json
 import os
 from app.config import Config
 from app.aescipher import AESCipher
 from app.aeskeywrapper import AESKeyWrapper
 from app.aeshelper import AESHelper
 
+class Record(object):
+    id = -1
+    data = ""
+
+    def __init__(self, size):
+        self._initData(size)
+
+    def _initData(self, size):
+        self.data = "A" * size * 1024
+
 class DataGenerator(object):
 
     def __init__(self, config):
         self.config = config
         self.aes_cipher = AESHelper(config).create_aescipher_from_config()
 
-    def generate_data(self, number_of_lines, out_file_path):
-        with open(out_file_path, 'wb+') as outFile:
-            for line in range(number_of_lines):
-                encrypted_record = self.aes_cipher.encrypt(str(line))
-                outFile.writelines(base64.b64encode(encrypted_record)+'\n')
+    def generate_data(self, size_of_record_kb, number_of_records, out_file_path):
+
+        record = Record(size_of_record_kb)
+
+        with open(out_file_path, 'w+') as out_file:
+            for recordId in range(number_of_records):
+                record.id = recordId              
+                encrypted_record = self.aes_cipher.encrypt(json.dumps(record.__dict__))
+                out_file.writelines(base64.b64encode(encrypted_record)+'\n')
 
 
 if __name__ == "__main__":
     config = Config()
     data_generator = DataGenerator(config)
 
-    data_generator.generate_data(10, "data/data.10.encrypted")
-    data_generator.generate_data(100, "data/data.100.encrypted")
-    data_generator.generate_data(1000, "data/data.1000.encrypted")
-    data_generator.generate_data(10000, "data/data.10000.encrypted")
+    data_generator.generate_data(1, 10, "data/data.10.encrypted")
+    data_generator.generate_data(1, 100, "data/data.100.encrypted")
+    data_generator.generate_data(1, 1000, "data/data.1000.encrypted")
+    data_generator.generate_data(1, 10000, "data/data.10000.encrypted")
 

samples/readFinalResults.py

@@ -3,11 +3,19 @@
 """
 import base64
 import io
-import sys
+import json
 from app.aeshelper import AESHelper
 from app.config import Config
 from azure.storage.blob import BlockBlobService
 
+class Record(object):
+    def __init__(self, id, data):
+        self.id = id
+        self.data = data
+
+def as_payload(dct):
+    return Record(dct['id'], dct['data'])
+
 if __name__ == "__main__":
     config = Config()
     aes_helper = AESHelper(config)
@@ -23,8 +31,9 @@ if __name__ == "__main__":
 
         for result in blobContents.readlines():
             decoded = aes_cipher.decrypt(base64.b64decode(result))
-            print 'encoded: ' + result + ' decoded: '  + decoded
-            results.append(decoded)
+            record = json.loads(decoded, object_hook = as_payload)
+            print str(record.id) + " " + str(len(record.data) / 1024) + "KB"
+            results.append(record.id)
 
     results.sort()
     print results