From 39f682e3032a64ac3e50757d7b1c5805c6580d6a Mon Sep 17 00:00:00 2001 From: annatisch Date: Wed, 17 May 2017 10:47:06 -0700 Subject: [PATCH 01/29] Updates to pool UI --- azure_batch_maya/scripts/pools.py | 5 ++-- azure_batch_maya/scripts/ui/ui_pools.py | 36 ++++++++++++++----------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/azure_batch_maya/scripts/pools.py b/azure_batch_maya/scripts/pools.py index 7c8f7de..7a7210e 100644 --- a/azure_batch_maya/scripts/pools.py +++ b/azure_batch_maya/scripts/pools.py @@ -114,13 +114,12 @@ class AzureBatchPools(object): pool = self._call(self.batch.pool.get, pool.id) _nodes = self._call(self.batch.compute_node.list, pool.id) nodes = [n for n in _nodes] + self.selected_pool.set_id(pool.id) self.selected_pool.set_label(pool.display_name if pool.display_name else pool.id) - self.selected_pool.set_size(pool.current_dedicated_nodes) - self.selected_pool.set_target(pool.target_dedicated_nodes) + self.selected_pool.set_size(pool) self.selected_pool.set_type( "Auto" if pool.id.startswith("Maya_Auto_Pool") else "Provisioned") self.selected_pool.set_state(pool.state.value, nodes) - self.selected_pool.set_tasks(pool.max_tasks_per_node) self.selected_pool.set_allocation(pool.allocation_state.value) self.selected_pool.set_created(pool.creation_time) self.selected_pool.set_licenses(pool.application_licenses) diff --git a/azure_batch_maya/scripts/ui/ui_pools.py b/azure_batch_maya/scripts/ui/ui_pools.py index 35bb155..b0f95a8 100644 --- a/azure_batch_maya/scripts/ui/ui_pools.py +++ b/azure_batch_maya/scripts/ui/ui_pools.py @@ -170,17 +170,12 @@ class AzureBatchPoolInfo(object): """ maya.text(self._type, edit=True, label=" {0}".format(value)) - def set_size(self, value): - """Set the number of instances in the pool. + def set_size(self, pool): + """Set the number of instances in the pool, both current and target. :param int value: Size of the pool. """ - maya.text(self._size, edit=True, label=" {0}".format(value)) - - def set_target(self, value): - """Set the target number of instances in the pool. - :param int value: The target size of the pool. - """ - maya.text(self._target, edit=True, label=" {0}".format(value)) + maya.text(self._size, edit=True, label=" target: {} current: {}".format( + pool.target_dedicated_nodes, pool.current_dedicated_nodes)) def set_created(self, value): """Set the date/time the pool was created. @@ -205,11 +200,11 @@ class AzureBatchPoolInfo(object): value += "{} nodes {} ".format(node_states[state], state.value) maya.text(self._state, edit=True, label=" {0}".format(value)) - def set_tasks(self, value): - """Set the tasks per TVM allowed in the pool. - :param int value: Tasks per TVM. + def set_id(self, value): + """Set the pool ID field. + :param str value: Pool ID. """ - maya.text(self._tasks, edit=True, label=" {0}".format(value)) + maya.text_field(self._id, edit=True, text=value) def set_allocation(self, value): """Set the allocation state of the pool. @@ -240,12 +235,11 @@ class AzureBatchPoolInfo(object): """Command for the expanding of the pool reference frame layout. Loads latest details for the specified pool and populates UI. """ + self._id = self.display_data("ID: ") self._type = self.display_info("Type: ") - self._size = self.display_info("Current Size: ") - self._target = self.display_info("Target Size: ") + self._size = self.display_info("Size: ") self._created = self.display_info("Created: ") self._state = self.display_info("State: ") - self._tasks = self.display_info("Tasks per VM: ") self._image = self.display_info("Image: ") self._allocation = self.display_info("Allocation State: ") self._licenses = self.display_info("Licenses: ") @@ -305,6 +299,16 @@ class AzureBatchPoolInfo(object): self.content.append(input) return input + def display_data(self, label): + """Display text data as a non-editable text field with heading. + :param str label: The text for the data heading. + """ + self.content.append( + maya.text(label=label, parent=self.listbox, align="right")) + input = maya.text_field(text="", parent=self.listbox, editable=False) + self.content.append(input) + return input + def delete_pool(self, *args): """Delete the specified pool.""" self.delete_button.start() From 67665372cdcfb1de934d6d5e02c882aa1a678479 Mon Sep 17 00:00:00 2001 From: annatisch Date: Mon, 22 May 2017 09:16:12 -0700 Subject: [PATCH 02/29] Added Arnold logging configuration --- azure_batch_maya/modules/arnold_renderer.py | 9 + azure_batch_maya/modules/default.py | 3 +- .../templates/arnold-basic-linux.json | 13 +- .../templates/arnold-basic-windows.json | 190 +++++++++--------- 4 files changed, 117 insertions(+), 98 deletions(-) diff --git a/azure_batch_maya/modules/arnold_renderer.py b/azure_batch_maya/modules/arnold_renderer.py index 7ad4f8d..5ca81a1 100644 --- a/azure_batch_maya/modules/arnold_renderer.py +++ b/azure_batch_maya/modules/arnold_renderer.py @@ -44,6 +44,11 @@ class ArnoldRenderJob(AzureBatchRenderJob): def __init__(self): self._renderer = "arnold" self.label = "Arnold" + self.log_levels = [ + "0 - Errors", + "1 - Warnings + Info", + "2 - Debug" + ] def settings(self): if self.scene_name == "": @@ -61,6 +66,9 @@ class ArnoldRenderJob(AzureBatchRenderJob): self.end = self.display_int("End frame: ", self.end_frame, edit=True) self.step = self.display_int("Frame step: ", self.frame_step, edit=True) + log_level = mel.eval("getAttr defaultArnoldRenderOptions.log_verbosity") + self.logging = self.display_menu("Logging: ", self.log_levels, log_level+1) + def get_title(self): return str(cmds.textField(self.job_name, query=True, text=True)) @@ -95,6 +103,7 @@ class ArnoldRenderJob(AzureBatchRenderJob): params["frameEnd"] = cmds.intField(self.end, query=True, value=True) params["frameStep"] = cmds.intField(self.step, query=True, value=True) params["renderer"] = "arnold" + params["logLevel"] = int(cmds.optionMenu(self.logging, query=True, select=True)) - 1 return params diff --git a/azure_batch_maya/modules/default.py b/azure_batch_maya/modules/default.py index d7a1128..3a67c96 100644 --- a/azure_batch_maya/modules/default.py +++ b/azure_batch_maya/modules/default.py @@ -94,12 +94,13 @@ class AzureBatchRenderJob(object): else: return cmds.text(label=value, align='left') - def display_menu(self, label, options): + def display_menu(self, label, options, selected): cmds.text(label=label, align='right') menu = cmds.optionMenu() for opt in options: cmds.menuItem(label=opt) cmds.setParent('..') + cmds.optionMenu(menu, edit=True, select=selected) return menu def display_button(self, label, cmd): diff --git a/azure_batch_maya/templates/arnold-basic-linux.json b/azure_batch_maya/templates/arnold-basic-linux.json index 7df5e19..822c098 100644 --- a/azure_batch_maya/templates/arnold-basic-linux.json +++ b/azure_batch_maya/templates/arnold-basic-linux.json @@ -71,6 +71,13 @@ "metadata": { "description": "The file group where outputs will be stored" } + }, + "logLevel": { + "type": "int", + "defaultValue": 1, + "metadata": { + "description": "Arnold logging verbosity" + } } }, "jobPreparationTask": { @@ -113,7 +120,7 @@ "elevationLevel": "admin" } }, - "commandLine": "sudo mkdir -m a=rwx -p \"/X\";sudo mount --rbind $AZ_BATCH_JOB_PREP_WORKING_DIR/assets /X;Render -renderer [parameters('renderer')] -proj \"$AZ_BATCH_JOB_PREP_WORKING_DIR\" -verb -preRender renderPrep -rd \"$AZ_BATCH_TASK_WORKING_DIR/images\" -s {0} -e {0} \"[parameters('sceneFile')]\";err=$?;python /mnt/resource/batch/tasks/workitems/[parameters('outputs')]/job-1/jobpreparation/wd/thumbnail.py $err;sudo umount \"/X\";exit $err", + "commandLine": "sudo mkdir -m a=rwx -p \"/X\";sudo mount --rbind $AZ_BATCH_JOB_PREP_WORKING_DIR/assets /X;Render -renderer [parameters('renderer')] -proj \"$AZ_BATCH_JOB_PREP_WORKING_DIR\" -ai:ltc 1 -ai:lve [parameters('logLevel')] -verb -preRender renderPrep -rd \"$AZ_BATCH_TASK_WORKING_DIR/images\" -s {0} -e {0} \"[parameters('sceneFile')]\";err=$?;python /mnt/resource/batch/tasks/workitems/[parameters('outputs')]/job-1/jobpreparation/wd/thumbnail.py $err;sudo umount \"/X\";exit $err", "environmentSettings": [ { "name": "MAYA_SCRIPT_PATH", @@ -122,10 +129,6 @@ { "name": "FLEXLM_TIMEOUT", "value": "5000000" - }, - { - "name": "MAYA_RENDER_DESC_PATH", - "value": "/opt/solidangle/mtoa/2017/" } ], "outputFiles": [ diff --git a/azure_batch_maya/templates/arnold-basic-windows.json b/azure_batch_maya/templates/arnold-basic-windows.json index ed983e3..838a0ca 100644 --- a/azure_batch_maya/templates/arnold-basic-windows.json +++ b/azure_batch_maya/templates/arnold-basic-windows.json @@ -71,103 +71,109 @@ "metadata": { "description": "The file group where outputs will be stored" } - } - }, - "jobPreparationTask": { - "resourceFiles": [ - { - "source": { - "fileGroup": "[parameters('projectData')]" - }, - "filePath": "assets\\" - }, - { - "blobSource": "[parameters('assetScript')]", - "filePath": "scripts\\renderPrep.mel" - }, - { - "blobSource": "[parameters('thumbScript')]", - "filePath": "thumbnail.py" - }, - { - "blobSource": "[parameters('workspace')]", - "filePath": "workspace.mel" + }, + "logLevel": { + "type": "int", + "defaultValue": 1, + "metadata": { + "description": "Arnold logging verbosity" } - ], - "commandLine": "dir" - }, - "taskFactory": { - "type": "parametricSweep", - "parameterSets": [ - { - "start": "[parameters('frameStart')]", - "end": "[parameters('frameEnd')]", - "step": "[parameters('frameStep')]" - } - ], - "repeatTask": { - "displayName": "Frame {0}", - "commandLine": "subst X: %AZ_BATCH_JOB_PREP_WORKING_DIR%\\assets & render -renderer [parameters('renderer')] -proj \"%AZ_BATCH_JOB_PREP_WORKING_DIR%\" -verb -preRender renderPrep -rd \"%AZ_BATCH_TASK_WORKING_DIR%\\images\" -s {0} -e {0} \"[parameters('sceneFile')]\" & call mayapy %AZ_BATCH_JOB_PREP_WORKING_DIR%\\thumbnail.py %^errorlevel%", - "environmentSettings": [ + }, + "jobPreparationTask": { + "resourceFiles": [ { - "name": "MAYA_SCRIPT_PATH", - "value": "%AZ_BATCH_JOB_PREP_WORKING_DIR%\\scripts" + "source": { + "fileGroup": "[parameters('projectData')]" + }, + "filePath": "assets\\" }, { - "name": "FLEXLM_TIMEOUT", - "value": "5000000" + "blobSource": "[parameters('assetScript')]", + "filePath": "scripts\\renderPrep.mel" + }, + { + "blobSource": "[parameters('thumbScript')]", + "filePath": "thumbnail.py" + }, + { + "blobSource": "[parameters('workspace')]", + "filePath": "workspace.mel" } ], - "outputFiles": [ + "commandLine": "dir" + }, + "taskFactory": { + "type": "parametricSweep", + "parameterSets": [ { - "filePattern": "images/**/*", - "destination": { - "autoStorage": { - "fileGroup": "[parameters('outputs')]" - } - }, - "uploadOptions": { - "uploadCondition": "taskSuccess" - } - }, - { - "filePattern": "thumbs/*.png", - "destination": { - "autoStorage": { - "fileGroup": "[parameters('outputs')]", - "path": "thumbs" - } - }, - "uploadOptions": { - "uploadCondition": "taskSuccess" - } - }, - { - "filePattern": "../stdout.txt", - "destination": { - "autoStorage": { - "fileGroup": "[parameters('outputs')]", - "path": "logs/frame_{0}.log" - } - }, - "uploadOptions": { - "uploadCondition": "taskCompletion" - } - }, - { - "filePattern": "../stderr.txt", - "destination": { - "autoStorage": { - "fileGroup": "[parameters('outputs')]", - "path": "logs/frame_{0}_error.log" - } - }, - "uploadOptions": { - "uploadCondition": "taskCompletion" - } + "start": "[parameters('frameStart')]", + "end": "[parameters('frameEnd')]", + "step": "[parameters('frameStep')]" } - ] - } - }, - "onAllTasksComplete": "terminateJob" -} + ], + "repeatTask": { + "displayName": "Frame {0}", + "commandLine": "subst X: %AZ_BATCH_JOB_PREP_WORKING_DIR%\\assets & render -renderer [parameters('renderer')] -proj \"%AZ_BATCH_JOB_PREP_WORKING_DIR%\" -ai:ltc 1 -ai:lve [parameters('logLevel')] -verb -preRender renderPrep -rd \"%AZ_BATCH_TASK_WORKING_DIR%\\images\" -s {0} -e {0} \"[parameters('sceneFile')]\" & call mayapy %AZ_BATCH_JOB_PREP_WORKING_DIR%\\thumbnail.py %^errorlevel%", + "environmentSettings": [ + { + "name": "MAYA_SCRIPT_PATH", + "value": "%AZ_BATCH_JOB_PREP_WORKING_DIR%\\scripts" + }, + { + "name": "FLEXLM_TIMEOUT", + "value": "5000000" + } + ], + "outputFiles": [ + { + "filePattern": "images/**/*", + "destination": { + "autoStorage": { + "fileGroup": "[parameters('outputs')]" + } + }, + "uploadOptions": { + "uploadCondition": "taskSuccess" + } + }, + { + "filePattern": "thumbs/*.png", + "destination": { + "autoStorage": { + "fileGroup": "[parameters('outputs')]", + "path": "thumbs" + } + }, + "uploadOptions": { + "uploadCondition": "taskSuccess" + } + }, + { + "filePattern": "../stdout.txt", + "destination": { + "autoStorage": { + "fileGroup": "[parameters('outputs')]", + "path": "logs/frame_{0}.log" + } + }, + "uploadOptions": { + "uploadCondition": "taskCompletion" + } + }, + { + "filePattern": "../stderr.txt", + "destination": { + "autoStorage": { + "fileGroup": "[parameters('outputs')]", + "path": "logs/frame_{0}_error.log" + } + }, + "uploadOptions": { + "uploadCondition": "taskCompletion" + } + } + ] + } + }, + "onAllTasksComplete": "terminateJob" + } From e7dde603eafcdd9bbaaeccadc8e15f40f9e437b9 Mon Sep 17 00:00:00 2001 From: annatisch Date: Mon, 22 May 2017 14:04:11 -0700 Subject: [PATCH 03/29] Review feedback --- README.md | 2 +- azure_batch_maya/mel/create_shelf.mel | 2 +- azure_batch_maya/modules/arnold_renderer.py | 28 ++++---- azure_batch_maya/modules/default.py | 1 - azure_batch_maya/modules/maya_software.py | 28 ++++---- azure_batch_maya/plug-in/AzureBatch.py | 43 ++++++------ azure_batch_maya/scripts/api.py | 1 - azure_batch_maya/scripts/assets.py | 63 +++++++++-------- azure_batch_maya/scripts/config.py | 28 +++++--- azure_batch_maya/scripts/environment.py | 4 +- azure_batch_maya/scripts/history.py | 61 ++++++++++------- azure_batch_maya/scripts/pools.py | 4 +- azure_batch_maya/scripts/shared.py | 29 +++++--- azure_batch_maya/scripts/submission.py | 15 ++++- azure_batch_maya/scripts/tools/install_pip.py | 7 +- azure_batch_maya/scripts/tools/job_watcher.py | 15 ++--- azure_batch_maya/scripts/ui/ui_assets.py | 31 ++------- azure_batch_maya/scripts/ui/ui_config.py | 31 ++------- azure_batch_maya/scripts/ui/ui_environment.py | 31 ++------- azure_batch_maya/scripts/ui/ui_history.py | 34 ++-------- azure_batch_maya/scripts/ui/ui_pools.py | 31 ++------- azure_batch_maya/scripts/ui/ui_shared.py | 34 ++-------- azure_batch_maya/scripts/ui/ui_submission.py | 67 ++++++------------- azure_batch_maya/scripts/utils.py | 1 - .../templates/arnold-basic-linux.json | 2 +- .../templates/arnold-basic-windows.json | 2 +- docs/submitting_jobs.md | 2 +- 27 files changed, 247 insertions(+), 350 deletions(-) diff --git a/README.md b/README.md index 4b90d56..4073e0a 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ ensure the updated dependencies are loaded correctly. Before using the plug-in, it will need to be authenticated using your Azure Batch and Azure Storage account keys. In order to retrieve this information: -1. Open the Azure management portal (ms.portal.azure.com). +1. Open the Azure management portal (portal.azure.com). 2. Select Azure Batch Accounts in the left-hand menu. This can be found under `More Services` in the `Compute` category. 3. Select your account in the list. Copy and paste the account URL into `Service` field of the plug-in UI. Paste the account name into the `Batch Account` field. 4. In the portal, select `Keys` on the left-hand menu. Copy and paste one of the access keys into the `Batch Key` field in the plug-in. diff --git a/azure_batch_maya/mel/create_shelf.mel b/azure_batch_maya/mel/create_shelf.mel index 854b330..a5eee01 100644 --- a/azure_batch_maya/mel/create_shelf.mel +++ b/azure_batch_maya/mel/create_shelf.mel @@ -9,7 +9,7 @@ global proc run_guiStarter() global proc openMissionControl() { - python("import webbrowser\nwebbrowser.open(\"https://ms.portal.azure.com\", 2, True)"); + python("import webbrowser\nwebbrowser.open(\"https://portal.azure.com\", 2, True)"); } diff --git a/azure_batch_maya/modules/arnold_renderer.py b/azure_batch_maya/modules/arnold_renderer.py index 5ca81a1..b8e39ae 100644 --- a/azure_batch_maya/modules/arnold_renderer.py +++ b/azure_batch_maya/modules/arnold_renderer.py @@ -60,8 +60,8 @@ class ArnoldRenderJob(AzureBatchRenderJob): file_prefix = os.path.split(file_prefix)[1] else: file_prefix = "" - self.job_name = self.display_string("Job Name: ", job_name) - self.output_name = self.display_string("Output Prefix: ", file_prefix) + self.job_name = self.display_string("Job name: ", job_name) + self.output_name = self.display_string("Output prefix: ", file_prefix) self.start = self.display_int("Start frame: ", self.start_frame, edit=True) self.end = self.display_int("End frame: ", self.end_frame, edit=True) self.step = self.display_int("Frame step: ", self.frame_step, edit=True) @@ -82,18 +82,20 @@ class ArnoldRenderJob(AzureBatchRenderJob): pending_changes = cmds.file(query=True, modified=True) if not pending_changes: return self.scene_name, [self.scene_name] - options = ["Save and Continue", - "Don't Save and Continue", - "Cancel"] + options = { + 'save': "Save and continue", + 'nosave': "Continue without saving", + 'cancel': "Cancel" + } answer = cmds.confirmDialog(title='Unsaved Changes', - message='There are unsaved changes. Proceed?', - button=options, - defaultButton=options[0], - cancelButton=options[2], - dismissString=options[2]) - if answer == options[2]: - raise Exception("Submission Aborted") - if answer == options[0]: + message='There are unsaved changes. Continue?', + button=options.values(), + defaultButton=options['save'], + cancelButton=options['cancel'], + dismissString=options['cancel']) + if answer == options['cancel']: + raise Exception("Submission aborted") + if answer == options['save']: cmds.SaveScene() return self.scene_name, [self.scene_name] diff --git a/azure_batch_maya/modules/default.py b/azure_batch_maya/modules/default.py index 3a67c96..b86be21 100644 --- a/azure_batch_maya/modules/default.py +++ b/azure_batch_maya/modules/default.py @@ -154,7 +154,6 @@ class AzureBatchRenderJob(object): horizontalScrollBarThickness=0, verticalScrollBarThickness=3, parent=layout, - #width=360, height=260) self.subLayout = cmds.rowColumnLayout( diff --git a/azure_batch_maya/modules/maya_software.py b/azure_batch_maya/modules/maya_software.py index e234697..bc5469d 100644 --- a/azure_batch_maya/modules/maya_software.py +++ b/azure_batch_maya/modules/maya_software.py @@ -70,21 +70,23 @@ class AzureBatchMayaJob(AzureBatchRenderJob): raise ValueError("Current Maya scene has not been saved to disk.") pending_changes = cmds.file(query=True, modified=True) if not pending_changes: - return [self.scene_name] - options = ["Save and Continue", - "Don't Save and Continue", - "Cancel"] + return self.scene_name, [self.scene_name] + options = { + 'save': "Save and continue", + 'nosave': "Continue without saving", + 'cancel': "Cancel" + } answer = cmds.confirmDialog(title='Unsaved Changes', - message='There are unsaved changes. Proceed?', - button=options, - defaultButton=options[0], - cancelButton=options[2], - dismissString=options[2]) - if answer == options[2]: - raise Exception("Submission Aborted") - if answer == options[0]: + message='There are unsaved changes. Continue?', + button=options.values(), + defaultButton=options['save'], + cancelButton=options['cancel'], + dismissString=options['cancel']) + if answer == options['cancel']: + raise Exception("Submission aborted") + if answer == options['save']: cmds.SaveScene() - return [self.scene_name] + return self.scene_name, [self.scene_name] def get_params(self): params = {} diff --git a/azure_batch_maya/plug-in/AzureBatch.py b/azure_batch_maya/plug-in/AzureBatch.py index 5889d04..2d019b5 100644 --- a/azure_batch_maya/plug-in/AzureBatch.py +++ b/azure_batch_maya/plug-in/AzureBatch.py @@ -53,44 +53,43 @@ warnings.simplefilter('ignore') INSTALL_DIR = os.path.normpath( os.path.join(cmds.internalVar(userScriptDir=True), 'azure-batch-libs')) sys.path.append(INSTALL_DIR) - + REQUIREMENTS = [ "pathlib==1.0.1", ] NAMESPACE_PACAKGES = [ - #"azure-mgmt-nspkg==2.0.0", "azure-mgmt-batch==4.0.0", - "azure-mgmt-storage==1.0.0rc1", + "azure-mgmt-storage==1.0.0", "azure-common==1.1.5", "azure-batch==3.0.0", "azure-storage==0.32.0", ] VERSION = "0.9.0" -SLA_PREF = "AzureBatch_SLA" +EULA_PREF = "AzureBatch_EULA" SHELF_FILE = "shelf_AzureBatch.mel" cmd_name = "AzureBatch" fMayaExitingCB = None os.environ["AZUREBATCH_VERSION"] = VERSION -def sla_prompt(): - """Open prompt for T's & C's agreement.""" +def eula_prompt(): + """Open prompt for terms and conditions.""" current_file = inspect.getfile(inspect.currentframe()) current_dir = os.path.dirname(os.path.abspath(current_file)) - SLA = os.path.join(current_dir, "SLA.html") + eula = os.path.join(current_dir, "EULA.html") form = cmds.setParent(q=True) cmds.formLayout(form, e=True, width=500) heading = cmds.text( - l='Maya Cloud Rendering Service Level Agreement', font="boldLabelFont") + l='Maya Cloud Rendering License Agreement', font="boldLabelFont") text = cmds.text(l="By loading this plug-in you are agreeing to " "the following terms and conditions.") - if not os.path.exists(SLA): - raise RuntimeError("SLA notice not found at {0}".format(SLA)) + if not os.path.exists(eula): + raise RuntimeError("EULA notice not found at {0}".format(eula)) - with open(SLA, "rb") as sla_text: - html = sla_text.read() + with open(eula, "rb") as eula_text: + html = eula_text.read() unicode = html.decode("windows-1252") encoded_str = unicode.encode("ascii", "xmlcharrefreplace") read = cmds.scrollField(editable=False, wordWrap=True, height=300, @@ -388,7 +387,7 @@ def install_pkg(package): def install_namespace_pkg(package, namespace): - """Azure packages have issues installing one by one as the don't + """Azure packages have issues installing one by one as they don't unpackage correctly into the namespace directory. So we have to install to a temp directory and move it to the right place. @@ -418,19 +417,19 @@ def install_namespace_pkg(package, namespace): def initializePlugin(obj): """Initialize Plug-in""" print("Initializing Azure Batch plug-in") - existing = cmds.optionVar(exists=SLA_PREF) + existing = cmds.optionVar(exists=EULA_PREF) if not existing: agree = cmds.layoutDialog(ui=sla_prompt, title="Azure Batch Maya Client") if str(agree) != 'Agree': raise RuntimeError("Plugin initialization aborted.") - cmds.optionVar(stringValue=(SLA_PREF, VERSION)) + cmds.optionVar(stringValue=(EULA_PREF, VERSION)) else: - agreed = cmds.optionVar(query=SLA_PREF) + agreed = cmds.optionVar(query=EULA_PREF) if StrictVersion(agreed) < VERSION: - agree = cmds.layoutDialog(ui=sla_prompt, title="AzureBatch Maya Client") + agree = cmds.layoutDialog(ui=sla_prompt, title="Azure Batch Maya Client") if str(agree) != 'Agree': raise RuntimeError("Plugin initialization aborted.") - cmds.optionVar(stringValue=(SLA_PREF, VERSION)) + cmds.optionVar(stringValue=(EULA_PREF, VERSION)) print("Checking for dependencies...") missing_libs = [] @@ -483,7 +482,7 @@ def initializePlugin(obj): raise ImportError("Please restart Maya. Azure Batch installed " "Python dependencies.") - print("Dependency check complete!") + print("Dependency check complete") plugin = OpenMayaMPx.MFnPlugin( obj, "Microsoft Corporation", VERSION, "Any") plugin.registerCommand(cmd_name, cmd_creator) @@ -507,7 +506,7 @@ def initializePlugin(obj): def uninitializePlugin(obj): """Remove and uninstall plugin.""" - print("Removing AzureBatch plug-in") + print("Removing Azure Batch plug-in") plugin = MFnPlugin(obj) plugin.deregisterCommand(cmd_name) try: @@ -520,7 +519,7 @@ def uninitializePlugin(obj): if cmds.window("AzureBatch", exists=1): cmds.deleteUI("AzureBatch") AzureBatchSetup.remove_environment() - print("Finished clearing up all AzureBatch components") + print("Finished clearing up all Azure Batch components") """Check for environment and set up if not found.""" @@ -528,5 +527,5 @@ try: sys.path.extend(os.environ["AZUREBATCH_SCRIPTS"].split(os.pathsep)) sys.path.append(os.environ['AZUREBATCH_MODULES']) except KeyError as e: - print("Couldn't find AzureBatch environment, setting up now...") + print("Couldn't find Azure Batch environment, setting up now...") setup_module() diff --git a/azure_batch_maya/scripts/api.py b/azure_batch_maya/scripts/api.py index 3505ecf..082b025 100644 --- a/azure_batch_maya/scripts/api.py +++ b/azure_batch_maya/scripts/api.py @@ -32,7 +32,6 @@ try: import maya.OpenMayaMPx as omp except ImportError: print("No maya module found.") - import os import logging LOG = logging.getLogger('AzureBatchMaya') diff --git a/azure_batch_maya/scripts/assets.py b/azure_batch_maya/scripts/assets.py index 55934f6..c367608 100644 --- a/azure_batch_maya/scripts/assets.py +++ b/azure_batch_maya/scripts/assets.py @@ -28,7 +28,7 @@ import logging from datetime import datetime -import threading +import multiprocessing import os import sys import glob @@ -52,15 +52,16 @@ from default import AzureBatchRenderAssets SYS_SEARCHPATHS = [] USR_SEARCHPATHS = [] -UPLOAD_THREADS = 10 +BYTES = 1024 class AzureBatchAssets(object): """Handler for asset file functionality.""" - def __init__(self, frame, call): + def __init__(self, index, frame, call): """Create new Asset Handler. + :param index: The UI tab index. :param frame: The shared plug-in UI frame. :type frame: :class:`.AzureBatchUI` :param func call: The shared REST API call wrapper. @@ -69,6 +70,7 @@ class AzureBatchAssets(object): self._call = call self._session = None self._assets = None + self._tab_index = index self.batch = None self.modules = self._collect_modules() @@ -154,6 +156,12 @@ class AzureBatchAssets(object): self.renderer = AzureBatchRenderAssets() self._log.debug("Configured renderer to {0}".format(self.renderer.render_engine)) + def _switch_tab(self): + """Make this tab the currently displayed tab. If this tab is already + open, this will do nothing. + """ + self.frame.select_tab(self._tab_index) + def _collect_assets(self): """Called on upload. If the asset tab has not yet been loaded before job submission is attempted, then the asset references have not yet @@ -213,37 +221,38 @@ class AzureBatchAssets(object): return Asset(map_file, [], self.batch, self._log) def _upload_all(self, to_upload, progress, total, project): - """Upload all selected assets in 10 threads.""" - uploads_running = [] + """Upload all selected assets in multiple processes according to available cores.""" progress_queue = Queue() - for i in range(0, len(to_upload), UPLOAD_THREADS): - for index, asset in enumerate(to_upload[i:i + UPLOAD_THREADS]): + for i in range(0, len(to_upload), multiprocessing.cpu_count()): + for index, asset in enumerate(to_upload[i:i + multiprocessing.cpu_count()]): self._log.debug("Starting thread for asset: {}".format(asset.path)) - upload = threading.Thread( + upload = multiprocessing.Process( target=asset.upload, args=(index, progress, progress_queue, project)) upload.start() - uploads_running.append(upload) - self._log.debug("Batch of asset uploads pending: {}".format(threading.active_count())) - - while any(t for t in uploads_running if t.is_alive()) or not progress_queue.empty(): + while multiprocessing.active_children() or not progress_queue.empty(): uploaded = progress_queue.get() if isinstance(uploaded, Exception): raise uploaded elif callable(uploaded): uploaded() else: - total = total - (uploaded/1024/1024) + total = total - (uploaded/BYTES/BYTES) self.ui.upload_status("Uploading {0}...".format(self._format_size(total))) progress_queue.task_done() - def _format_size(self, data): + def _format_size(self, nbytes): """Format the data size in bytes to nicely display for upload progress. """ - if data > 1024: - return "{:0.2f}GB".format(data/1014) - else: - return "{:0.2f}MB".format(data) + suffixes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] + if nbytes == 0: + return '0 B' + i = 0 + while nbytes >= BYTES and i < len(suffixes)-1: + nbytes /= BYTES + i += 1 + f = ('%.2f' % nbytes).rstrip('0').rstrip('.') + return '%s %s' % (f, suffixes[i]) def _total_data(self, files): """Format the combined size of the files to display @@ -252,7 +261,7 @@ class AzureBatchAssets(object): data = float(0) for asset in files: data += asset.size - return data/1024/1024 + return data/BYTES/BYTES def configure(self, session): """Populate the Batch client for the current sessions of the asset tab. @@ -288,14 +297,14 @@ class AzureBatchAssets(object): def add_files(self, files, column_layout, scroll_layout): """Function called by the 'Add File(s)' button in the UI for adding arbitrary - file references to the collection that included with the next job subbmission. + file references to the collection to be included with the next job subbmission. """ for f in files: self._assets.add_asset(f, self.ui, column_layout, scroll_layout) def add_dir(self, dirs, column_layout, scroll_layout): """Function called by the 'Add Directory' button in the UI for adding arbitrary - file references to the collection that included with the next job subbmission. + file references to the collection to be included with the next job subbmission. """ for folder in dirs: for root, _, files in os.walk(folder): @@ -310,8 +319,8 @@ class AzureBatchAssets(object): if the upload process is part of job submission. :param progress_bar: The progress of the current process. This is only populated if the upload process is part of job submission. - :param job_id: The ID of the job being submitted. This is only populated is the - upload process if path of job submission. + :param job_id: The ID of the job being submitted. This is only populated if the + upload process is part of job submission. :param load_plugins: A list of plugins to be added to the pre-render script for loading on the server. Only populated if part of a job submission. :param os_flavor: The OS flavor of the rendering pool. Only set as part of the job @@ -339,7 +348,7 @@ class AzureBatchAssets(object): progress_bar.is_cancelled() progress_bar.status('Uploading files...') progress_bar.max(len(asset_refs)) - self.frame.select_tab(3) + self._switch_tab() self.ui.disable(False) self.ui.upload_button.start() payload = self._total_data(asset_refs) @@ -364,7 +373,7 @@ class AzureBatchAssets(object): else: maya.error(str(exp)) finally: - # If part of job submission errors and progress bar + # If part of job submission, errors and progress bar # will be handled back in submission.py if not job_set: progress_bar.end() @@ -548,7 +557,7 @@ class Assets(object): class Asset(object): - """Representation of a single asset, managing it's file reference, + """Representation of a single asset, managing its file reference, display listing and upload of the file. """ @@ -652,7 +661,7 @@ class Asset(object): def make_visible(self, index): """Attempt to auto-scroll the asset display list so that the progress of currently uploading assets remains in view. - TODO: Thie needs some work.... + TODO: This needs some work.... """ if index == 0: while maya.scroll_layout(self.scroll_layout, query=True, scrollAreaValue=True)[0] > 0: diff --git a/azure_batch_maya/scripts/config.py b/azure_batch_maya/scripts/config.py index 08d3ea9..5a7098d 100644 --- a/azure_batch_maya/scripts/config.py +++ b/azure_batch_maya/scripts/config.py @@ -30,6 +30,7 @@ import ConfigParser import os import logging import sys +import traceback from ui_config import ConfigUI from api import MayaAPI as maya @@ -49,14 +50,17 @@ LOG_LEVELS = { class AzureBatchConfig(object): """Handler for authentication and configuration of the SDK clients.""" - def __init__(self, frame, start): + def __init__(self, index, frame, start): """Create new configuration Handler. + :param index: The UI tab index. :param frame: The shared plug-in UI frame. :type frame: :class:`.AzureBatchUI` :param func call: The shared REST API call wrapper. """ + self.ui = None self.session = start + self._tab_index = index self._data_dir = os.path.join(maya.prefs_dir(), 'AzureBatchData') self._ini_file = "azure_batch.ini" self._cfg = ConfigParser.ConfigParser() @@ -110,7 +114,10 @@ class AzureBatchConfig(object): self._log = self._configure_logging( self._cfg.get("AzureBatch", "logging")) except (ConfigParser.NoOptionError, ConfigParser.NoSectionError) as exp: - print(exp) #TODO: Better error handling + # We should only worry about this if it happens when authenticating + # using the UI, otherwise it's expected. + if self.ui: + raise ValueError("Invalid Configuration File: {}".format(exp)) def _configure_logging(self, log_level): """Configure the logger. Setup the file output and format @@ -173,10 +180,10 @@ class AzureBatchConfig(object): try: filter = batch.models.PoolListOptions(max_results=1, select="id") self._client.pool.list(filter) - self._storage.create_container("batch-maya-assets", fail_on_exist=False) + self._storage.list_containers(num_results=1) return True except Exception as exp: - self._log.info("Could not get authenticate session: {0}".format(exp)) + self._log.info("Failed to authenticate: {0}".format(exp)) return False def set_logging(self, level): @@ -205,10 +212,15 @@ class AzureBatchConfig(object): def authenticate(self): """Begin authentication - initiated by the UI button.""" - self._configure_plugin() - self._auth = self._auto_authentication() - self.ui.set_authenticate(self._auth) - self.session() + try: + self._configure_plugin() + self._auth = self._auto_authentication() + except ValueError as exp: + maya.error(exp) + self._auth = False + finally: + self.ui.set_authenticate(self._auth) + self.session() def get_cached_vm_sku(self): """Attempt to retrieve a selected VM SKU from a previous session.""" diff --git a/azure_batch_maya/scripts/environment.py b/azure_batch_maya/scripts/environment.py index a6fac2d..f355e65 100644 --- a/azure_batch_maya/scripts/environment.py +++ b/azure_batch_maya/scripts/environment.py @@ -63,9 +63,10 @@ LICENSES = [ class AzureBatchEnvironment(object): """Handler for rendering environment configuration functionality.""" - def __init__(self, frame, call): + def __init__(self, index, frame, call): """Create new Environment Handler. + :param index: The UI tab index. :param frame: The shared plug-in UI frame. :type frame: :class:`.AzureBatchUI` :param func call: The shared REST API call wrapper. @@ -73,6 +74,7 @@ class AzureBatchEnvironment(object): self._log = logging.getLogger('AzureBatchMaya') self._call = call self._session = None + self._tab_index = index self.licenses = {} self._get_plugin_licenses() diff --git a/azure_batch_maya/scripts/history.py b/azure_batch_maya/scripts/history.py index c2f483f..18a8d96 100644 --- a/azure_batch_maya/scripts/history.py +++ b/azure_batch_maya/scripts/history.py @@ -45,9 +45,10 @@ import azure.batch as batch class AzureBatchHistory(object): """Handler for job display functionality.""" - def __init__(self, frame, call): + def __init__(self, index, frame, call): """Create new job history Handler. + :param index: The UI tab index. :param frame: The shared plug-in UI frame. :type frame: :class:`.AzureBatchUI` :param func call: The shared REST API call wrapper. @@ -55,9 +56,10 @@ class AzureBatchHistory(object): self._log = logging.getLogger('AzureBatchMaya') self._call = call self._session = None + self._tab_index = index self.batch = None self.index = 0 - self.per_call = 5 + self.jobs_per_page = 5 self.count = 0 self.min = True self.max = False @@ -70,17 +72,28 @@ class AzureBatchHistory(object): """Get the pixel height of the job thumbnail to display. Note: This function only works under Python 2.7 """ + # The first 8 bytes of data are the PNG signature, the next 4 + # are the type field of the first chunk (which we don't need). + png_signature_bytes = 12 + # The IHDR header is always the first chunk in a valid PNG image + png_header = 'IHDR' + # X and Y image dimensions are the first 8 bytes of the IDHR chunk + dimensions_bytes = 8 + # 120 pixels is the default height of a thumbnail image y = 120 + # Byte order = network (!), format = unsigned long (L) + excepted_byte_format = '!LL' with open(image, 'rb') as im: - im.read(12) - if im.read(4) == 'IHDR': - x, y = struct.unpack("!LL", im.read(8)) + png_signature = im.read(png_signature_bytes) + valid_png = im.read(len(png_header)) == png_header + if valid_png: + x, y = struct.unpack(excepted_byte_format, im.read(dimensions_bytes)) return y def _download_thumbnail(self, job, thumbs): """Download a preview thumbnail for the selected job. - Only certain output formats are supported. If not thumbnail exists - then we display the default 'no preview' image. + Only certain output formats are supported. If the thumbnail doesn't + exist then we display the default 'no preview' image. TODO: Remove direct storage reference to use batch.download. :param job: The selected job object. @@ -105,7 +118,7 @@ class AzureBatchHistory(object): self._log.debug("Thumbnail path: {}".format(thumb_path)) try: if not os.path.isfile(thumb_path): - self._log.info("Downloading task thumb: {}".format(thumbs[-1])) + self._log.info("Downloading task thumbnail: {}".format(thumbs[-1])) self.storage.get_blob_to_path('fgrp-' + job.id, thumbs[-1], thumb_path) self._log.info(" thumbnail download successful.\n") except Exception as exp: @@ -120,25 +133,25 @@ class AzureBatchHistory(object): """Calculate the paging progress label, including which page is currently displayed out of how many. """ - if (self.index + self.per_call) > self.count: - extra = (self.index + self.per_call) - self.count - new_range = ((self.index + self.per_call) - extra) + if (self.index + self.jobs_per_page) > self.count: + extra = (self.index + self.jobs_per_page) - self.count + new_range = ((self.index + self.jobs_per_page) - extra) self.ui.num_jobs = "{0} - {1} of {2}".format( min((self.index + 1), new_range), new_range, self.count) else: self.ui.num_jobs = "{0} - {1} of {2}".format( - min((self.index + 1), self.count), (self.index + self.per_call), self.count) + min((self.index + 1), self.count), (self.index + self.jobs_per_page), self.count) def _set_min_max(self): """Determine whether we are currently displaying the first or - last page, for that the forward and back buttons can be disabled + last page, so that the forward and back buttons can be disabled accordingly. """ self.min = True if self.index < 1 else False - if (self.count % self.per_call) == 0: - self.max = (self.index >= (self.count - self.per_call)) or (self.per_call > self.count) + if (self.count % self.jobs_per_page) == 0: + self.max = (self.index >= (self.count - self.jobs_per_page)) or (self.jobs_per_page > self.count) else: - self.max = self.index >= (self.count - self.per_call + (self.per_call - (self.count % self.per_call))) + self.max = self.index >= (self.count - self.jobs_per_page + (self.jobs_per_page - (self.count % self.jobs_per_page))) self.ui.last_page = not self.max self.ui.first_page = not self.min @@ -172,7 +185,7 @@ class AzureBatchHistory(object): def show_jobs(self): """Display the current page of jobs.""" - self.jobs = self.all_jobs[self.index:self.index + self.per_call] + self.jobs = self.all_jobs[self.index:self.index + self.jobs_per_page] self._set_num_jobs() self._set_min_max() display_jobs = [] @@ -182,11 +195,11 @@ class AzureBatchHistory(object): def show_next_jobs(self): """Show the next page of jobs.""" - self.index = min(self.index + self.per_call, self.count) + self.index = min(self.index + self.jobs_per_page, self.count) def show_prev_jobs(self): """Show the previous page of jobs.""" - self.index = max(self.index - self.per_call, 0) + self.index = max(self.index - self.jobs_per_page, 0) def show_first_jobs(self): """Return to the first page of jobs (most recently submitted).""" @@ -194,14 +207,14 @@ class AzureBatchHistory(object): def show_last_jobs(self): """Skip to the last page of jobs (first ones submitted).""" - if (self.count % self.per_call) == 0: - self.index = self.count - self.per_call + if (self.count % self.jobs_per_page) == 0: + self.index = self.count - self.jobs_per_page else: - self.index = self.count - self.per_call + \ - (self.per_call - (self.count % self.per_call)) + self.index = self.count - self.jobs_per_page + \ + (self.jobs_per_page - (self.count % self.jobs_per_page)) def job_selected(self, job_ui): - """A job has been selected, so it's details need to be retrieved + """A job has been selected, so its details need to be retrieved and displayed. This is also called when a job entry has been refreshed. """ diff --git a/azure_batch_maya/scripts/pools.py b/azure_batch_maya/scripts/pools.py index 7a7210e..4329b77 100644 --- a/azure_batch_maya/scripts/pools.py +++ b/azure_batch_maya/scripts/pools.py @@ -42,9 +42,10 @@ from ui_pools import PoolsUI class AzureBatchPools(object): """Handler for pool functionality.""" - def __init__(self, frame, call): + def __init__(self, index, frame, call): """Create new Pool Handler. + :param index: The UI tab index. :param frame: The shared plug-in UI frame. :type frame: :class:`.AzureBatchUI` :param func call: The shared REST API call wrapper. @@ -52,6 +53,7 @@ class AzureBatchPools(object): self._log = logging.getLogger('AzureBatchMaya') self._call = call self._session = None + self._tab_index = index self.batch = None self.ui = PoolsUI(self, frame) diff --git a/azure_batch_maya/scripts/shared.py b/azure_batch_maya/scripts/shared.py index dc0ac29..8b8ebe1 100644 --- a/azure_batch_maya/scripts/shared.py +++ b/azure_batch_maya/scripts/shared.py @@ -27,9 +27,9 @@ #-------------------------------------------------------------------------- import logging -import webbrowser import os -import threading +import sys +import traceback from ui_shared import AzureBatchUI from config import AzureBatchConfig @@ -51,6 +51,15 @@ ACCEPTED_ERRORS = [ class AzureBatchSettings(object): + tab_index = { + 'AUTH': 1, + 'SUBMIT': 2, + 'ASSETS': 3, + 'POOLS': 4, + 'JOBHISTORY': 5, + 'ENV': 6 + } + @staticmethod def starter(): """Called by the mel script when the shelf button is clicked.""" @@ -63,12 +72,12 @@ class AzureBatchSettings(object): self._log = logging.getLogger('AzureBatchMaya') try: self.frame = AzureBatchUI(self) - self.config = AzureBatchConfig(self.frame, self.start) - self.submission = AzureBatchSubmission(self.frame, self.call) - self.assets = AzureBatchAssets(self.frame, self.call) - self.pools = AzureBatchPools(self.frame, self.call) - self.history = AzureBatchHistory(self.frame, self.call) - self.env = AzureBatchEnvironment(self.frame, self.call) + self.config = AzureBatchConfig(self.tab_index['AUTH'], self.frame, self.start) + self.submission = AzureBatchSubmission(self.tab_index['SUBMIT'], self.frame, self.call) + self.assets = AzureBatchAssets(self.tab_index['ASSETS'], self.frame, self.call) + self.pools = AzureBatchPools(self.tab_index['POOLS'], self.frame, self.call) + self.history = AzureBatchHistory(self.tab_index['JOBHISTORY'], self.frame, self.call) + self.env = AzureBatchEnvironment(self.tab_index['ENV'], self.frame, self.call) self.start() except Exception as exp: if (maya.window("AzureBatch", q=1, exists=1)): @@ -79,7 +88,7 @@ class AzureBatchSettings(object): def start(self): """Start the plugin UI. Depending on whether auto-authentication was successful, the plugin will start by displaying the submission tab. - Otherwise the UI will be disables, and the log in tab will be displayed. + Otherwise the UI will be disabled, and the login tab will be displayed. """ try: self._log.debug("Starting AzureBatchShared...") @@ -119,4 +128,6 @@ class AzureBatchSettings(object): except Exception as exp: if (maya.window("AzureBatch", q=1, exists=1)): maya.delete_ui("AzureBatch") + exc_type, exc_value, exc_traceback = sys.exc_info() + self._log.error("".join(traceback.format_exception(exc_type, exc_value, exc_traceback))) raise ValueError("Error: {0}".format(exp)) diff --git a/azure_batch_maya/scripts/submission.py b/azure_batch_maya/scripts/submission.py index aa3fce3..5915572 100644 --- a/azure_batch_maya/scripts/submission.py +++ b/azure_batch_maya/scripts/submission.py @@ -49,15 +49,17 @@ from default import AzureBatchRenderJob class AzureBatchSubmission(object): """Handler for job submission functionality.""" - def __init__(self, frame, call): + def __init__(self, index, frame, call): """Create new Submission Handler. + :param index: The UI tab index. :param frame: The shared plug-in UI frame. :type frame: :class:`.AzureBatchUI` :param func call: The shared REST API call wrapper. """ self._log = logging.getLogger('AzureBatchMaya') self._call = call + self._tab_index = index self.ui = SubmissionUI(self, frame) self.modules = self._collect_modules() @@ -67,6 +69,7 @@ class AzureBatchSubmission(object): self.pool_manager = None self.env_manager = None self.batch = None + self.max_pool_size = 1000 #callback.after_new(self.ui.refresh) #callback.after_open(self.ui.refresh) @@ -111,6 +114,12 @@ class AzureBatchSubmission(object): self.renderer = AzureBatchRenderJob() self._log.debug("Configured renderer to {0}".format(self.renderer.render_engine)) + def _switch_tab(self): + """Make this tab the currently displayed tab. If this tab is already + open, this will do nothing. + """ + self.frame.select_tab(self._tab_index) + def _check_outputs(self): """Check whether at least one of the scene cameras is marked as renderable and that at least one layer is a render layer. If not, there will be no @@ -280,7 +289,7 @@ class AzureBatchSubmission(object): application_params['assetScript'] = map_url application_params['thumbScript'] = thumb_url application_params['workspace'] = workspace_url - self.frame.select_tab(2) + self._switch_tab() self.ui.submit_status("Configuring job...") progress.status("Configuring job...") @@ -313,5 +322,5 @@ class AzureBatchSubmission(object): finally: if progress: progress.end() - self.frame.select_tab(2) + self._switch_tab() self.renderer.disable(True) diff --git a/azure_batch_maya/scripts/tools/install_pip.py b/azure_batch_maya/scripts/tools/install_pip.py index 848dacc..ddbab6a 100644 --- a/azure_batch_maya/scripts/tools/install_pip.py +++ b/azure_batch_maya/scripts/tools/install_pip.py @@ -1,9 +1,8 @@ # coding=utf-8 -# -------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os import tempfile diff --git a/azure_batch_maya/scripts/tools/job_watcher.py b/azure_batch_maya/scripts/tools/job_watcher.py index d197cf4..75d6ddf 100644 --- a/azure_batch_maya/scripts/tools/job_watcher.py +++ b/azure_batch_maya/scripts/tools/job_watcher.py @@ -1,9 +1,8 @@ # coding=utf-8 -# -------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import webbrowser import ConfigParser @@ -15,12 +14,12 @@ import re batch_client = None storage_client = None +header_line_length = 50 def header(header): header_chars = len(header) - total_len = 50 - dashes = total_len - header_chars + dashes = header_line_length - header_chars mult = int(dashes/2) padded = "\n\n" + mult*"-" + header + mult*"-" if dashes % 2 > 0: @@ -88,7 +87,7 @@ def _check_job_stopped(job): if job.state in stopped_status: print(header("Job has stopped")) print("Job status: {0}".format(job.state)) - raise RuntimeError("Job is no longer running. Status: {0}".format(job.state)) + raise RuntimeError("Job is no longer active. State: {0}".format(job.state)) elif job.state == JobState.completed: print(header("Job has completed")) return True @@ -115,7 +114,7 @@ def track_job_progress(id, container, dwnld_dir): percentage = (100 * len(completed_tasks)) / len(tasks) print("Running - {}%".format(percentage)) if errored_tasks: - print(" - Warning: some tasks have completed with a non-zero exit code.") + print(" - Warning: some tasks have failed.") _track_completed_outputs(container, dwnld_dir) if _check_job_stopped(job): diff --git a/azure_batch_maya/scripts/ui/ui_assets.py b/azure_batch_maya/scripts/ui/ui_assets.py index 7df7008..d4ed9b3 100644 --- a/azure_batch_maya/scripts/ui/ui_assets.py +++ b/azure_batch_maya/scripts/ui/ui_assets.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os import utils diff --git a/azure_batch_maya/scripts/ui/ui_config.py b/azure_batch_maya/scripts/ui/ui_config.py index fc22123..13e5aa1 100644 --- a/azure_batch_maya/scripts/ui/ui_config.py +++ b/azure_batch_maya/scripts/ui/ui_config.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import utils diff --git a/azure_batch_maya/scripts/ui/ui_environment.py b/azure_batch_maya/scripts/ui/ui_environment.py index 519a7ff..dbd6b90 100644 --- a/azure_batch_maya/scripts/ui/ui_environment.py +++ b/azure_batch_maya/scripts/ui/ui_environment.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os import utils diff --git a/azure_batch_maya/scripts/ui/ui_history.py b/azure_batch_maya/scripts/ui/ui_history.py index aad6a41..5c86bb1 100644 --- a/azure_batch_maya/scripts/ui/ui_history.py +++ b/azure_batch_maya/scripts/ui/ui_history.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os import webbrowser @@ -328,10 +305,11 @@ class AzureBatchJobInfo(object): def set_job(self, value): """Set the label for the job ID, and format the portal URL reference for the job with this ID. + Except that with the current portal we have no way to directly link to a job. :param str value: The job ID. """ maya.text_field(self._job, edit=True, text=value) - self.url = "https://ms.portal.azure.com" + self.url = "https://portal.azure.com" def set_pool(self, value): """Set the label for the pool ID that this job ran on. diff --git a/azure_batch_maya/scripts/ui/ui_pools.py b/azure_batch_maya/scripts/ui/ui_pools.py index b0f95a8..e4f361d 100644 --- a/azure_batch_maya/scripts/ui/ui_pools.py +++ b/azure_batch_maya/scripts/ui/ui_pools.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os diff --git a/azure_batch_maya/scripts/ui/ui_shared.py b/azure_batch_maya/scripts/ui/ui_shared.py index f6c2af7..79728c6 100644 --- a/azure_batch_maya/scripts/ui/ui_shared.py +++ b/azure_batch_maya/scripts/ui/ui_shared.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os @@ -100,7 +77,8 @@ class AzureBatchUI(object): """Select a specific tab for display. :param int tab: The index of the tab to display. """ - maya.tab_layout(self.tab_display, edit=True, selectTabIndex=tab) + index = self.base.tab_index[tab] + maya.tab_layout(self.tab_display, edit=True, selectTabIndex=index) def selected_tab(self): """Get the index of the currently selected tab. diff --git a/azure_batch_maya/scripts/ui/ui_submission.py b/azure_batch_maya/scripts/ui/ui_submission.py index 2e47836..53b91cb 100644 --- a/azure_batch_maya/scripts/ui/ui_submission.py +++ b/azure_batch_maya/scripts/ui/ui_submission.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os import utils @@ -35,6 +12,11 @@ from api import MayaAPI as maya class SubmissionUI(object): """Class to create the 'Submit' tab in the plug-in UI""" + AUTO_POOL = 1 + EXISTING_POOL = 2 + NEW_POOL = 3 + + def __init__(self, base, frame): """Create 'Submit' tab and add to UI frame. @@ -46,7 +28,7 @@ class SubmissionUI(object): self.base = base self.label = "Submit" self.page = maya.form_layout(enableBackground=True) - self.select_pool_type = 1 + self.select_pool_type = self.AUTO_POOL self.select_instances = 1 with utils.ScrollLayout(height=475, parent=self.page) as scroll: @@ -60,8 +42,8 @@ class SubmissionUI(object): maya.text(label="Pools: ", align="right") maya.radio_group( labelArray3=("Auto provision a pool for this job", - "Reuse an existing persistent pool", - "Create a new persistent pool"), + "Reuse an existing persistent pool", + "Create a new persistent pool"), numberOfRadioButtons=3, select=self.select_pool_type, vertical=True, @@ -73,9 +55,9 @@ class SubmissionUI(object): self.control = maya.int_slider( field=True, value=self.select_instances, minValue=1, - maxValue=1000, + maxValue=self.base.max_pool_size, fieldMinValue=1, - fieldMaxValue=1000, + fieldMaxValue=self.base.max_pool_size, changeCommand=self.set_pool_instances, annotation="Number of instances in pool") maya.parent() @@ -207,7 +189,7 @@ class SubmissionUI(object): :returns: A dictionary with selected pool type as key and pool specification as value. """ - if self.select_pool_type == 2: + if self.select_pool_type == self.EXISTING_POOL: details = str(maya.menu(self.control, query=True, value=True)) else: details = self.select_instances @@ -224,16 +206,16 @@ class SubmissionUI(object): Displays the pool size UI control. Command for select_pool_type radio buttons. """ - self.select_pool_type = 3 + self.select_pool_type = self.NEW_POOL maya.delete_ui(self.control) maya.text(self.pool_text, edit=True, label="Instances: ") self.control = maya.int_slider( field=True, value=self.select_instances, minValue=1, - maxValue=1000, + maxValue=self.base.max_pool_size, fieldMinValue=1, - fieldMaxValue=1000, + fieldMaxValue=self.base.max_pool_size, parent=self.pool_settings, changeCommand=self.set_pool_instances, annotation="Number of instances in pool") @@ -243,16 +225,16 @@ class SubmissionUI(object): Displays the pool size UI control. Command for select_pool_type radio buttons. """ - self.select_pool_type = 1 + self.select_pool_type = self.AUTO_POOL maya.delete_ui(self.control) maya.text(self.pool_text, edit=True, label="Instances: ") self.control = maya.int_slider( field=True, value=self.select_instances, minValue=1, - maxValue=1000, + maxValue=self.base.max_pool_size, fieldMinValue=1, - fieldMaxValue=1000, + fieldMaxValue=self.base.max_pool_size, parent=self.pool_settings, changeCommand=self.set_pool_instances, annotation="Number of instances in pool") @@ -263,7 +245,7 @@ class SubmissionUI(object): in a dropdown menu. Command for select_pool_type radio buttons. """ - self.select_pool_type = 2 + self.select_pool_type = self.EXISTING_POOL maya.delete_ui(self.control) maya.text(self.pool_text, edit=True, label="loading...") maya.refresh() @@ -274,8 +256,3 @@ class SubmissionUI(object): annotation="Use an existing persistent pool ID") for pool_id in pool_options: maya.menu_option(pool_id) - #with utils.Dropdown(None, parent=self.pool_settings, annotation="Use an existing persistent pool ID") as pools: - # self.control = pools - # for pool_id in pool_options: - # self.control.add_item(pool_id) - diff --git a/azure_batch_maya/scripts/utils.py b/azure_batch_maya/scripts/utils.py index 4eae703..968eb43 100644 --- a/azure_batch_maya/scripts/utils.py +++ b/azure_batch_maya/scripts/utils.py @@ -177,7 +177,6 @@ class Layout(object): return self.layout def __exit__(self, type, value, traceback): - #TODO: Exception handling should go in here maya.parent() diff --git a/azure_batch_maya/templates/arnold-basic-linux.json b/azure_batch_maya/templates/arnold-basic-linux.json index 822c098..bc36f98 100644 --- a/azure_batch_maya/templates/arnold-basic-linux.json +++ b/azure_batch_maya/templates/arnold-basic-linux.json @@ -1,6 +1,6 @@ { "templateMetadata": { - "description": "Sample application template for working with Blender." + "description": "Application template for working with Maya and Arnold on CentOS." }, "parameters": { "sceneFile": { diff --git a/azure_batch_maya/templates/arnold-basic-windows.json b/azure_batch_maya/templates/arnold-basic-windows.json index 838a0ca..79ac383 100644 --- a/azure_batch_maya/templates/arnold-basic-windows.json +++ b/azure_batch_maya/templates/arnold-basic-windows.json @@ -1,6 +1,6 @@ { "templateMetadata": { - "description": "Sample application template for working with Blender." + "description": "Application template for working with Maya and Arnold on Windows." }, "parameters": { "sceneFile": { diff --git a/docs/submitting_jobs.md b/docs/submitting_jobs.md index 7cbae19..8855f51 100644 --- a/docs/submitting_jobs.md +++ b/docs/submitting_jobs.md @@ -76,7 +76,7 @@ You can also delete the pool and resize the number of VMs in the pool. A pool ca ## Monitoring jobs -Once you have submitted a job, you can monitor it both via the [Azure Management Portal](http://ms.portal.azure.com/) and the `Jobs` tab of the plug-in. +Once you have submitted a job, you can monitor it both via the [Azure Management Portal](http://portal.azure.com/) and the `Jobs` tab of the plug-in. Selecting a listed job will display the current state of the job. You can also use this tab to cancel and delete jobs, as well as download the outputs and rendering logs. To download outputs, use the `Outputs` field to set the desired destination directory, and click the center button (with the gear icon) to start a background process that will watch the job and download outputs as it progresses. You can close Maya without disrupting the download. From 998d464de527b7716fd44cc6c7edb8f2abf55e7e Mon Sep 17 00:00:00 2001 From: annatisch Date: Mon, 22 May 2017 14:24:50 -0700 Subject: [PATCH 04/29] operating system enum --- azure_batch_maya/scripts/assets.py | 4 ++-- azure_batch_maya/scripts/environment.py | 12 ++++++------ azure_batch_maya/scripts/utils.py | 20 ++++++++++++++------ 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/azure_batch_maya/scripts/assets.py b/azure_batch_maya/scripts/assets.py index c367608..61f8f2d 100644 --- a/azure_batch_maya/scripts/assets.py +++ b/azure_batch_maya/scripts/assets.py @@ -182,7 +182,7 @@ class AzureBatchAssets(object): for rule in maya.workspace(fileRuleList=True): project_dir = maya.workspace(fileRuleEntry=rule) remote_path = utils.get_remote_directory(maya.workspace(en=project_dir), os_flavor) - if os_flavor == 'Windows': + if os_flavor == utils.OperatingSystem.windows: full_remote_path = "X:\\\\" + remote_path else: full_remote_path = "/X/" + remote_path @@ -211,7 +211,7 @@ class AzureBatchAssets(object): handle.write("loadPlugin \"{}\";\n".format(plugin)) handle.write("dirmap -en true;\n") for local, remote in pathmap.items(): - if os_flavor == 'Windows': + if os_flavor == utils.OperatingSystem.windows: full_remote_path = "X:\\\\" + remote(os_flavor) else: full_remote_path = "/X/" + remote(os_flavor) diff --git a/azure_batch_maya/scripts/environment.py b/azure_batch_maya/scripts/environment.py index f355e65..9ef3afc 100644 --- a/azure_batch_maya/scripts/environment.py +++ b/azure_batch_maya/scripts/environment.py @@ -32,7 +32,7 @@ import json from api import MayaAPI as maya from api import MayaCallbacks as callback - +import utils from ui_environment import EnvironmentUI @@ -154,16 +154,16 @@ class AzureBatchEnvironment(object): windows_offers = [value['offer'] for value in MAYA_IMAGES.values() if 'windows' in value['node_sku_id']] linux_offers = [value['offer'] for value in MAYA_IMAGES.values() if value['offer'] not in windows_offers] if pool_image.offer in windows_offers: - return 'Windows' + return utils.OperatingSystem.windows elif pool_image.offer in linux_offers: - return 'Linux' + return utils.OperatingSystem.linux else: raise ValueError('Selected pool is not using a valid Maya image.') - if 'Windows' in self.ui.get_image(): - return 'Windows' + if utils.OperatingSystem.windows.value in self.ui.get_image(): + return utils.OperatingSystem.windows else: - return 'Linux' + return utils.OperatingSystem.linux def get_environment_settings(self): env_vars = self.ui.get_env_vars() diff --git a/azure_batch_maya/scripts/utils.py b/azure_batch_maya/scripts/utils.py index 968eb43..ccae961 100644 --- a/azure_batch_maya/scripts/utils.py +++ b/azure_batch_maya/scripts/utils.py @@ -26,12 +26,14 @@ # #-------------------------------------------------------------------------- -from api import MayaAPI as maya +from enum import Enum import os import logging import platform import pathlib +from api import MayaAPI as maya + from batch_extensions import _file_utils as file_utils from exception import CancellationException, FileUploadException @@ -67,7 +69,7 @@ def get_remote_file_path(assetpath): """ def generate_path(os_flavor, fullpath=assetpath): local_sep = os.sep - remote_sep = '\\' if os_flavor == 'Windows' else '/' + remote_sep = '\\' if os_flavor == OperatingSystem.windows else '/' path = shorten_path(*os.path.split(fullpath)) if ':' in path: drive_letter, path = path.split(':', 1) @@ -82,7 +84,7 @@ def get_remote_directory(dir_path, os_flavor): path according to the remote OS. """ local_sep = os.sep - remote_sep = '\\' if os_flavor == 'Windows' else '/' + remote_sep = '\\' if os_flavor == OperatingSystem.windows else '/' if ':' in dir_path: drive_letter, dir_path = dir_path.split(':', 1) dir_path = drive_letter + local_sep + dir_path[1:] @@ -95,7 +97,7 @@ def format_scene_path(scene_file, os_flavor): be on the render node. """ scene_path = get_remote_file_path(scene_file)(os_flavor) - if os_flavor == 'Windows': + if os_flavor == OperatingSystem.windows: return "X:\\\\" + scene_path + '\\\\' + os.path.basename(scene_file) else: return "/X/" + scene_path + '/' + os.path.basename(scene_file) @@ -119,6 +121,12 @@ def get_os(): return platform.system() +class OperatingSystem(Enum): + windows = 'Windows' + linux = 'Linux' + darwin = 'Darwin' + + class Row(object): """UI row class.""" @@ -446,12 +454,12 @@ class JobWatcher(object): self.job_watcher = os.path.join( os.path.dirname(__file__), "tools", "job_watcher.py") platform = get_os() - if platform == "Windows": + if platform == OperatingSystem.windows: self.proc_cmd = 'system("WMIC PROCESS where (Name=\'mayapy.exe\') get Commandline")' self.start_cmd = 'system("start mayapy {0}")' self.quotes = '\\"' self.splitter = 'mayapy' - elif platform == "Darwin": + elif platform == OperatingSystem.darwin: self.proc_cmd = 'system("ps -ef")' self.start_cmd = 'system("osascript -e \'tell application \\"Terminal\\" to do script \\"python {0}\\"\'")' self.quotes = '\\\\\\"' From 717d7751bce2fbdb8ff99cd2d6e1f1c9d65a918b Mon Sep 17 00:00:00 2001 From: annatisch Date: Mon, 22 May 2017 14:25:08 -0700 Subject: [PATCH 05/29] Renamed job history and SLA --- azure_batch_maya/plug-in/{SLA.html => EULA.html} | 0 azure_batch_maya/scripts/{history.py => jobhistory.py} | 6 +++--- azure_batch_maya/scripts/shared.py | 6 +++--- .../scripts/ui/{ui_history.py => ui_jobhistory.py} | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) rename azure_batch_maya/plug-in/{SLA.html => EULA.html} (100%) rename azure_batch_maya/scripts/{history.py => jobhistory.py} (98%) rename azure_batch_maya/scripts/ui/{ui_history.py => ui_jobhistory.py} (99%) diff --git a/azure_batch_maya/plug-in/SLA.html b/azure_batch_maya/plug-in/EULA.html similarity index 100% rename from azure_batch_maya/plug-in/SLA.html rename to azure_batch_maya/plug-in/EULA.html diff --git a/azure_batch_maya/scripts/history.py b/azure_batch_maya/scripts/jobhistory.py similarity index 98% rename from azure_batch_maya/scripts/history.py rename to azure_batch_maya/scripts/jobhistory.py index 18a8d96..28e48fb 100644 --- a/azure_batch_maya/scripts/history.py +++ b/azure_batch_maya/scripts/jobhistory.py @@ -37,12 +37,12 @@ import shutil import re from api import MayaAPI as maya -from ui_history import HistoryUI +from ui_jobhistory import JobHistoryUI import azure.batch as batch -class AzureBatchHistory(object): +class AzureBatchJobHistory(object): """Handler for job display functionality.""" def __init__(self, index, frame, call): @@ -63,7 +63,7 @@ class AzureBatchHistory(object): self.count = 0 self.min = True self.max = False - self.ui = HistoryUI(self, frame) + self.ui = JobHistoryUI(self, frame) self.all_jobs = [] self.jobs = [] self.selected_job = None diff --git a/azure_batch_maya/scripts/shared.py b/azure_batch_maya/scripts/shared.py index 8b8ebe1..770029d 100644 --- a/azure_batch_maya/scripts/shared.py +++ b/azure_batch_maya/scripts/shared.py @@ -34,7 +34,7 @@ import traceback from ui_shared import AzureBatchUI from config import AzureBatchConfig from submission import AzureBatchSubmission -from history import AzureBatchHistory +from jobhistory import AzureBatchJobHistory from assets import AzureBatchAssets from pools import AzureBatchPools from environment import AzureBatchEnvironment @@ -76,7 +76,7 @@ class AzureBatchSettings(object): self.submission = AzureBatchSubmission(self.tab_index['SUBMIT'], self.frame, self.call) self.assets = AzureBatchAssets(self.tab_index['ASSETS'], self.frame, self.call) self.pools = AzureBatchPools(self.tab_index['POOLS'], self.frame, self.call) - self.history = AzureBatchHistory(self.tab_index['JOBHISTORY'], self.frame, self.call) + self.jobhistory = AzureBatchJobHistory(self.tab_index['JOBHISTORY'], self.frame, self.call) self.env = AzureBatchEnvironment(self.tab_index['ENV'], self.frame, self.call) self.start() except Exception as exp: @@ -95,7 +95,7 @@ class AzureBatchSettings(object): if self.config.auth: self.frame.is_logged_in() self.env.configure(self.config) - self.history.configure(self.config) + self.jobhistory.configure(self.config) self.assets.configure(self.config) self.pools.configure(self.config, self.env) self.submission.start(self.config, self.assets, self.pools, self.env) diff --git a/azure_batch_maya/scripts/ui/ui_history.py b/azure_batch_maya/scripts/ui/ui_jobhistory.py similarity index 99% rename from azure_batch_maya/scripts/ui/ui_history.py rename to azure_batch_maya/scripts/ui/ui_jobhistory.py index 5c86bb1..ff1ae27 100644 --- a/azure_batch_maya/scripts/ui/ui_history.py +++ b/azure_batch_maya/scripts/ui/ui_jobhistory.py @@ -10,14 +10,14 @@ from api import MayaAPI as maya import utils -class HistoryUI(object): +class JobHistoryUI(object): """Class to create the 'Jobs' tab in the plug-in UI""" def __init__(self, base, frame): """Create 'Jobs' tab and add to UI frame. :param base: The base class for handling jobs monitoring functionality. - :type base: :class:`.AzureBatchHistory` + :type base: :class:`.AzureBatchJobHistory` :param frame: The shared plug-in UI frame. :type frame: :class:`.AzureBatchUI` """ @@ -237,7 +237,7 @@ class AzureBatchJobInfo(object): """Create a new job reference. :param base: The base class for handling jobs monitoring functionality. - :type base: :class:`.AzureBatchHistory` + :type base: :class:`.AzureBatchJobHistory` :param int index: The index of where this reference is displayed on the current page. :param layout: The layout on which the job details will be displayed. From 7f59ba92143618c108f440d6d95f647e1fd5c5af Mon Sep 17 00:00:00 2001 From: annatisch Date: Mon, 22 May 2017 14:56:45 -0700 Subject: [PATCH 06/29] Cleaned up maya renderer --- azure_batch_maya/modules/arnold_renderer.py | 36 ++-- azure_batch_maya/modules/maya_software.py | 23 ++- azure_batch_maya/scripts/config.py | 58 +++--- azure_batch_maya/scripts/submission.py | 13 +- .../templates/arnold-basic-linux.json | 9 +- .../templates/arnold-basic-windows.json | 9 +- .../templates/mayaSoftware-basic-linux.json | 174 ++++++++++++++++++ .../templates/mayaSoftware-basic-windows.json | 167 +++++++++++++++++ 8 files changed, 412 insertions(+), 77 deletions(-) create mode 100644 azure_batch_maya/templates/mayaSoftware-basic-linux.json create mode 100644 azure_batch_maya/templates/mayaSoftware-basic-windows.json diff --git a/azure_batch_maya/modules/arnold_renderer.py b/azure_batch_maya/modules/arnold_renderer.py index b8e39ae..027019d 100644 --- a/azure_batch_maya/modules/arnold_renderer.py +++ b/azure_batch_maya/modules/arnold_renderer.py @@ -41,9 +41,11 @@ from default import AzureBatchRenderJob, AzureBatchRenderAssets class ArnoldRenderJob(AzureBatchRenderJob): + render_engine = 'arnold' + def __init__(self): - self._renderer = "arnold" - self.label = "Arnold" + self._renderer = 'arnold' + self.label = 'Arnold' self.log_levels = [ "0 - Errors", "1 - Warnings + Info", @@ -51,7 +53,7 @@ class ArnoldRenderJob(AzureBatchRenderJob): ] def settings(self): - if self.scene_name == "": + if self.scene_name == '': job_name = "Untitled" else: job_name = str(os.path.splitext(os.path.basename(self.scene_name))[0]) @@ -76,7 +78,7 @@ class ArnoldRenderJob(AzureBatchRenderJob): return True def get_jobdata(self): - if self.scene_name == "": + if self.scene_name == '': raise ValueError("Current Maya scene has not been saved to disk.") pending_changes = cmds.file(query=True, modified=True) @@ -87,8 +89,8 @@ class ArnoldRenderJob(AzureBatchRenderJob): 'nosave': "Continue without saving", 'cancel': "Cancel" } - answer = cmds.confirmDialog(title='Unsaved Changes', - message='There are unsaved changes. Continue?', + answer = cmds.confirmDialog(title="Unsaved Changes", + message="There are unsaved changes. Continue?", button=options.values(), defaultButton=options['save'], cancelButton=options['cancel'], @@ -101,23 +103,23 @@ class ArnoldRenderJob(AzureBatchRenderJob): def get_params(self): params = {} - params["frameStart"] = cmds.intField(self.start, query=True, value=True) - params["frameEnd"] = cmds.intField(self.end, query=True, value=True) - params["frameStep"] = cmds.intField(self.step, query=True, value=True) - params["renderer"] = "arnold" - params["logLevel"] = int(cmds.optionMenu(self.logging, query=True, select=True)) - 1 + params['frameStart'] = cmds.intField(self.start, query=True, value=True) + params['frameEnd'] = cmds.intField(self.end, query=True, value=True) + params['frameStep'] = cmds.intField(self.step, query=True, value=True) + params['renderer'] = self._renderer + params['logLevel'] = int(cmds.optionMenu(self.logging, query=True, select=True)) - 1 return params class ArnoldRenderAssets(AzureBatchRenderAssets): assets = [] - render_engine = "arnold" + render_engine = 'arnold' file_nodes = { - "aiStandIn": ["dso"], - "aiPhotometricLight": ["aiFilename"], - "aiVolume": ["dso", "filename"], - "aiImage": ["filename"] + 'aiStandIn': ['dso'], + 'aiPhotometricLight': ['aiFilename'], + 'aiVolume': ['dso', 'filename'], + 'aiImage': ['filename'] } def check_path(self, path): @@ -137,7 +139,7 @@ class ArnoldRenderAssets(AzureBatchRenderAssets): nodes = cmds.ls(type=node_type) for node in nodes: for attr in attributes: - collected.append(cmds.getAttr(node + "." + attr)) + collected.append(cmds.getAttr(node + '.' + attr)) for path in collected: self.assets.append(self.check_path(path)) return self.assets diff --git a/azure_batch_maya/modules/maya_software.py b/azure_batch_maya/modules/maya_software.py index bc5469d..fb14da2 100644 --- a/azure_batch_maya/modules/maya_software.py +++ b/azure_batch_maya/modules/maya_software.py @@ -38,8 +38,10 @@ from default import AzureBatchRenderJob, AzureBatchRenderAssets class AzureBatchMayaJob(AzureBatchRenderJob): + render_engine = "mayaSoftware" + def __init__(self): - self._renderer = "mayaSoftware" + self._renderer = "sw" self.label = "Maya Software" def settings(self): @@ -66,8 +68,9 @@ class AzureBatchMayaJob(AzureBatchRenderJob): return True def get_jobdata(self): - if self.scene_name == "": + if self.scene_name == '': raise ValueError("Current Maya scene has not been saved to disk.") + pending_changes = cmds.file(query=True, modified=True) if not pending_changes: return self.scene_name, [self.scene_name] @@ -76,8 +79,8 @@ class AzureBatchMayaJob(AzureBatchRenderJob): 'nosave': "Continue without saving", 'cancel': "Cancel" } - answer = cmds.confirmDialog(title='Unsaved Changes', - message='There are unsaved changes. Continue?', + answer = cmds.confirmDialog(title="Unsaved Changes", + message="There are unsaved changes. Continue?", button=options.values(), defaultButton=options['save'], cancelButton=options['cancel'], @@ -90,14 +93,10 @@ class AzureBatchMayaJob(AzureBatchRenderJob): def get_params(self): params = {} - params["StartFrame"] = cmds.intField(self.start, query=True, value=True) - params["EndFrame"] = cmds.intField(self.end, query=True, value=True) - params["Renderer"] = "sw" - params["JobFile"] = os.path.basename(self.scene_name) - filename = str(cmds.textField(self.output_name, query=True, text=True)) - if '/' in filename or '\\' in filename: - raise ValueError("Subfolders not supported in output filename.") - params["OutputName"] = filename + params["frameStart"] = cmds.intField(self.start, query=True, value=True) + params["frameEnd"] = cmds.intField(self.end, query=True, value=True) + params["frameStep"] = cmds.intField(self.step, query=True, value=True) + params["renderer"] = self._renderer return params diff --git a/azure_batch_maya/scripts/config.py b/azure_batch_maya/scripts/config.py index 5a7098d..133caa3 100644 --- a/azure_batch_maya/scripts/config.py +++ b/azure_batch_maya/scripts/config.py @@ -40,6 +40,7 @@ import batch_extensions as batch from batch_extensions.batch_auth import SharedKeyCredentials +VERSION = os.environ['AZUREBATCH_VERSION'] LOG_LEVELS = { 'debug':10, 'info':20, @@ -63,6 +64,7 @@ class AzureBatchConfig(object): self._tab_index = index self._data_dir = os.path.join(maya.prefs_dir(), 'AzureBatchData') self._ini_file = "azure_batch.ini" + self._user_agent = "batchmaya/{}".format(VERSION) self._cfg = ConfigParser.ConfigParser() self._client = None self._log = None @@ -101,18 +103,18 @@ class AzureBatchConfig(object): try: self._cfg.read(config_file) self._storage = storage.BlockBlobService( - self._cfg.get("AzureBatch", "storage_account"), - self._cfg.get("AzureBatch", "storage_key"), - endpoint_suffix="core.windows.net") + self._cfg.get('AzureBatch', 'storage_account'), + self._cfg.get('AzureBatch', 'storage_key')) self._storage.MAX_SINGLE_PUT_SIZE = 2 * 1024 * 1024 credentials = SharedKeyCredentials( - self._cfg.get("AzureBatch", "batch_account"), - self._cfg.get("AzureBatch", "batch_key")) + self._cfg.get('AzureBatch', 'batch_account'), + self._cfg.get('AzureBatch', 'batch_key')) self._client = batch.BatchExtensionsClient( - credentials, base_url=self._cfg.get("AzureBatch", "batch_url"), + credentials, base_url=self._cfg.get('AzureBatch', 'batch_url'), storage_client=self._storage) + self._client._config.add_user_agent(self._user_agent) self._log = self._configure_logging( - self._cfg.get("AzureBatch", "logging")) + self._cfg.get('AzureBatch', 'logging')) except (ConfigParser.NoOptionError, ConfigParser.NoSectionError) as exp: # We should only worry about this if it happens when authenticating # using the UI, otherwise it's expected. @@ -144,31 +146,31 @@ class AzureBatchConfig(object): configuration file. """ try: - self._cfg.add_section("AzureBatch") + self._cfg.add_section('AzureBatch') except ConfigParser.DuplicateSectionError: pass try: - self.ui.endpoint = self._cfg.get("AzureBatch", "batch_url") + self.ui.endpoint = self._cfg.get('AzureBatch', 'batch_url') except ConfigParser.NoOptionError: self.ui.endpoint = "" try: - self.ui.account = self._cfg.get("AzureBatch", "batch_account") + self.ui.account = self._cfg.get('AzureBatch', 'batch_account') except ConfigParser.NoOptionError: self.ui.account = "" try: - self.ui.key = self._cfg.get("AzureBatch", "batch_key") + self.ui.key = self._cfg.get('AzureBatch', 'batch_key') except ConfigParser.NoOptionError: self.ui.key = "" try: - self.ui.storage = self._cfg.get("AzureBatch", "storage_account") + self.ui.storage = self._cfg.get('AzureBatch', 'storage_account') except ConfigParser.NoOptionError: self.ui.storage = "" try: - self.ui.storage_key = self._cfg.get("AzureBatch", "storage_key") + self.ui.storage_key = self._cfg.get('AzureBatch', 'storage_key') except ConfigParser.NoOptionError: self.ui.storage_key = "" try: - self.ui.logging = int(self._cfg.get("AzureBatch", "logging")) + self.ui.logging = int(self._cfg.get('AzureBatch', 'logging')) except ConfigParser.NoOptionError: self.ui.logging = 10 self.ui.set_authenticate(self._auth) @@ -192,20 +194,20 @@ class AzureBatchConfig(object): """ log_level = int(LOG_LEVELS[level]) self._log.setLevel(log_level) - self._cfg.set("AzureBatch", "logging", str(level)) + self._cfg.set('AzureBatch', 'logging', str(level)) def save_changes(self): """Persist configuration changes to file for future sessions.""" try: - self._cfg.add_section("AzureBatch") + self._cfg.add_section('AzureBatch') except ConfigParser.DuplicateSectionError: pass - self._cfg.set("AzureBatch", "batch_url", self.ui.endpoint) - self._cfg.set("AzureBatch", "batch_account", self.ui.account) - self._cfg.set("AzureBatch", "batch_key", self.ui.key) - self._cfg.set("AzureBatch", "storage_account", self.ui.storage) - self._cfg.set("AzureBatch", "storage_key", self.ui.storage_key) - self._cfg.set("AzureBatch", "logging", self.ui.logging) + self._cfg.set('AzureBatch', 'batch_url', self.ui.endpoint) + self._cfg.set('AzureBatch', 'batch_account', self.ui.account) + self._cfg.set('AzureBatch', 'batch_key', self.ui.key) + self._cfg.set('AzureBatch', 'storage_account', self.ui.storage) + self._cfg.set('AzureBatch', 'storage_key', self.ui.storage_key) + self._cfg.set('AzureBatch', 'logging', self.ui.logging) config_file = os.path.join(self._data_dir, self._ini_file) with open(config_file, 'w') as handle: self._cfg.write(handle) @@ -225,13 +227,13 @@ class AzureBatchConfig(object): def get_cached_vm_sku(self): """Attempt to retrieve a selected VM SKU from a previous session.""" try: - return self._cfg.get("AzureBatch", "vm_sku") + return self._cfg.get('AzureBatch', 'vm_sku') except ConfigParser.NoOptionError: return None def store_vm_sku(self, sku): """Cache selected VM SKU for later sessions.""" - self._cfg.set("AzureBatch", "vm_sku", sku) + self._cfg.set('AzureBatch', 'vm_sku', sku) config_file = os.path.join(self._data_dir, self._ini_file) with open(config_file, 'w') as handle: self._cfg.write(handle) @@ -239,13 +241,13 @@ class AzureBatchConfig(object): def get_cached_image(self): """Attempt to retrieve a selected image a previous session.""" try: - return self._cfg.get("AzureBatch", "image") + return self._cfg.get('AzureBatch', 'image') except ConfigParser.NoOptionError: return None def store_image(self, image): """Cache selected image for later sessions.""" - self._cfg.set("AzureBatch", "image", image) + self._cfg.set('AzureBatch', 'image', image) config_file = os.path.join(self._data_dir, self._ini_file) with open(config_file, 'w') as handle: self._cfg.write(handle) @@ -253,13 +255,13 @@ class AzureBatchConfig(object): def get_cached_autoscale_formula(self): """Attempt to retrieve an autoscale forumla from a previous session.""" try: - return self._cfg.get("AzureBatch", "autoscale") + return self._cfg.get('AzureBatch', 'autoscale') except ConfigParser.NoOptionError: return None def store_autoscale_formula(self, formula): """Cache selected VM SKU for later sessions.""" - self._cfg.set("AzureBatch", "autoscale", formula) + self._cfg.set('AzureBatch', 'autoscale', formula) config_file = os.path.join(self._data_dir, self._ini_file) with open(config_file, 'w') as handle: self._cfg.write(handle) diff --git a/azure_batch_maya/scripts/submission.py b/azure_batch_maya/scripts/submission.py index 5915572..2bdbb84 100644 --- a/azure_batch_maya/scripts/submission.py +++ b/azure_batch_maya/scripts/submission.py @@ -146,10 +146,10 @@ class AzureBatchSubmission(object): culprits to the ignore list if necessary. """ try: - with open(os.path.join(os.environ["AZUREBATCH_TOOLS"], + with open(os.path.join(os.environ['AZUREBATCH_TOOLS'], "supported_plugins.json"), 'r') as plugins: supported_plugins = json.load(plugins) - with open(os.path.join(os.environ["AZUREBATCH_TOOLS"], + with open(os.path.join(os.environ['AZUREBATCH_TOOLS'], "ignored_plugins.json"), 'r') as plugins: ignored_plugins = json.load(plugins) except EnvironmentError: @@ -163,7 +163,7 @@ class AzureBatchSubmission(object): "yet supported.\nRendering may be affected.\n") for plugin in unsupported_plugins: warning += plugin + "\n" - options = ["Continue", "Cancel"] + options = ['Continue', 'Cancel'] answer = maya.confirm(warning, options) if answer == options[-1]: raise CancellationException("Submission Aborted") @@ -203,7 +203,7 @@ class AzureBatchSubmission(object): pool_id = str(pool_spec[2]) if pool_id == "None": raise PoolException("No pool selected.") - return {"poolId" : pool_id} + return {'poolId' : pool_id} if pool_spec.get(3): self._log.info("Creating new pool.") return self.pool_manager.create_pool(int(pool_spec[3]), job_name) @@ -271,7 +271,8 @@ class AzureBatchSubmission(object): batch_parameters['displayName'] = self.renderer.get_title() batch_parameters['metadata'] = [{"name": "JobType", "value": "Maya"}] template_file = os.path.join( - os.environ['AZUREBATCH_TEMPLATES'], 'arnold-basic-{}.json'.format(pool_os.lower())) + os.environ['AZUREBATCH_TEMPLATES'], + "{}-basic-{}.json".format(self.renderer.render_engine, pool_os.lower())) batch_parameters['applicationTemplateInfo'] = {'filePath': template_file} application_params = {} batch_parameters['applicationTemplateInfo']['parameters'] = application_params @@ -317,7 +318,7 @@ class AzureBatchSubmission(object): except Exception as exp: self._log.error(str(exp)) exc_type, exc_value, exc_traceback = sys.exc_info() - self._log.debug("".join(traceback.format_exception(exc_type, exc_value, exc_traceback))) + self._log.debug(''.join(traceback.format_exception(exc_type, exc_value, exc_traceback))) maya.error(str(exp)) finally: if progress: diff --git a/azure_batch_maya/templates/arnold-basic-linux.json b/azure_batch_maya/templates/arnold-basic-linux.json index bc36f98..f2525ff 100644 --- a/azure_batch_maya/templates/arnold-basic-linux.json +++ b/azure_batch_maya/templates/arnold-basic-linux.json @@ -11,17 +11,12 @@ }, "renderer": { "type": "string", - "defaultValue": "file", + "defaultValue": "arnold", "metadata": { "description": "The Maya renderer to be used for the render" }, "allowedValues": [ - "arnold", - "default", - "sw", - "turtlebake", - "turtle", - "vr" + "arnold" ] }, "projectData": { diff --git a/azure_batch_maya/templates/arnold-basic-windows.json b/azure_batch_maya/templates/arnold-basic-windows.json index 79ac383..62b6ea4 100644 --- a/azure_batch_maya/templates/arnold-basic-windows.json +++ b/azure_batch_maya/templates/arnold-basic-windows.json @@ -11,17 +11,12 @@ }, "renderer": { "type": "string", - "defaultValue": "file", + "defaultValue": "arnold", "metadata": { "description": "The Maya renderer to be used for the render" }, "allowedValues": [ - "arnold", - "default", - "sw", - "turtlebake", - "turtle", - "vr" + "arnold" ] }, "projectData": { diff --git a/azure_batch_maya/templates/mayaSoftware-basic-linux.json b/azure_batch_maya/templates/mayaSoftware-basic-linux.json new file mode 100644 index 0000000..f6b8787 --- /dev/null +++ b/azure_batch_maya/templates/mayaSoftware-basic-linux.json @@ -0,0 +1,174 @@ +{ + "templateMetadata": { + "description": "Application template for working with Maya on CentOS." + }, + "parameters": { + "sceneFile": { + "type": "string", + "metadata": { + "description": "The Maya scene file to be rendered" + } + }, + "renderer": { + "type": "string", + "defaultValue": "sw", + "metadata": { + "description": "The Maya renderer to be used for the render" + }, + "allowedValues": [ + "sw" + ] + }, + "projectData": { + "type": "string", + "metadata": { + "description": "The file group where the input data is stored" + } + }, + "assetScript": { + "type": "string", + "metadata": { + "description": "The SAS URL to a pre-render asset path redirection script" + } + }, + "thumbScript": { + "type": "string", + "metadata": { + "description": "The SAS URL to the thumbnail generation script" + } + }, + "frameStart": { + "type": "int", + "metadata": { + "description": "Index of the first frame to render" + } + }, + "workspace": { + "type": "string", + "metadata": { + "description": "The SAS URL to the project workspace" + } + }, + "frameStep": { + "type": "int", + "metadata": { + "description": "Incremental step in frame sequeunce" + } + }, + "frameEnd": { + "type": "int", + "metadata": { + "description": "Index of the last frame to render" + } + }, + "outputs": { + "type": "string", + "metadata": { + "description": "The file group where outputs will be stored" + } + } + }, + "jobPreparationTask": { + "resourceFiles": [ + { + "source": { + "fileGroup": "[parameters('projectData')]" + }, + "filePath": "assets/" + }, + { + "blobSource": "[parameters('assetScript')]", + "filePath": "scripts/renderPrep.mel" + }, + { + "blobSource": "[parameters('thumbScript')]", + "filePath": "thumbnail.py" + }, + { + "blobSource": "[parameters('workspace')]", + "filePath": "workspace.mel" + } + ], + "commandLine": "dir" + }, + "taskFactory": { + "type": "parametricSweep", + "parameterSets": [ + { + "start": "[parameters('frameStart')]", + "end": "[parameters('frameEnd')]", + "step": "[parameters('frameStep')]" + } + ], + "repeatTask": { + "displayName": "Frame {0}", + "userIdentity": { + "autoUser": { + "scope": "task", + "elevationLevel": "admin" + } + }, + "commandLine": "sudo mkdir -m a=rwx -p \"/X\";sudo mount --rbind $AZ_BATCH_JOB_PREP_WORKING_DIR/assets /X;Render -renderer [parameters('renderer')] -proj \"$AZ_BATCH_JOB_PREP_WORKING_DIR\" -verb -preRender renderPrep -rd \"$AZ_BATCH_TASK_WORKING_DIR/images\" -s {0} -e {0} \"[parameters('sceneFile')]\";err=$?;python /mnt/resource/batch/tasks/workitems/[parameters('outputs')]/job-1/jobpreparation/wd/thumbnail.py $err;sudo umount \"/X\";exit $err", + "environmentSettings": [ + { + "name": "MAYA_SCRIPT_PATH", + "value": "/mnt/resource/batch/tasks/workitems/[parameters('outputs')]/job-1/jobpreparation/wd/scripts" + }, + { + "name": "FLEXLM_TIMEOUT", + "value": "5000000" + } + ], + "outputFiles": [ + { + "filePattern": "images/**/*", + "destination": { + "autoStorage": { + "fileGroup": "[parameters('outputs')]" + } + }, + "uploadOptions": { + "uploadCondition": "taskSuccess" + } + }, + { + "filePattern": "thumbs/*.png", + "destination": { + "autoStorage": { + "fileGroup": "[parameters('outputs')]", + "path": "thumbs" + } + }, + "uploadOptions": { + "uploadCondition": "taskSuccess" + } + }, + { + "filePattern": "../stdout.txt", + "destination": { + "autoStorage": { + "fileGroup": "[parameters('outputs')]", + "path": "logs/frame_{0}.log" + } + }, + "uploadOptions": { + "uploadCondition": "taskCompletion" + } + }, + { + "filePattern": "../stderr.txt", + "destination": { + "autoStorage": { + "fileGroup": "[parameters('outputs')]", + "path": "logs/frame_{0}_error.log" + } + }, + "uploadOptions": { + "uploadCondition": "taskCompletion" + } + } + ] + } + }, + "onAllTasksComplete": "terminateJob" +} diff --git a/azure_batch_maya/templates/mayaSoftware-basic-windows.json b/azure_batch_maya/templates/mayaSoftware-basic-windows.json new file mode 100644 index 0000000..5ac1265 --- /dev/null +++ b/azure_batch_maya/templates/mayaSoftware-basic-windows.json @@ -0,0 +1,167 @@ +{ + "templateMetadata": { + }, + "parameters": { + "sceneFile": { + "type": "string", + "metadata": { + "description": "The Maya scene file to be rendered" + } + }, + "renderer": { + "type": "string", + "defaultValue": "sw", + "metadata": { + "description": "The Maya renderer to be used for the render" + }, + "allowedValues": [ + "sw" + ] + }, + "projectData": { + "type": "string", + "metadata": { + "description": "The file group where the input data is stored" + } + }, + "assetScript": { + "type": "string", + "metadata": { + "description": "The SAS URL to a pre-render asset path redirection script" + } + }, + "thumbScript": { + "type": "string", + "metadata": { + "description": "The SAS URL to the thumbnail generation script" + } + }, + "workspace": { + "type": "string", + "metadata": { + "description": "The SAS URL to the project workspace" + } + }, + "frameStart": { + "type": "int", + "metadata": { + "description": "Index of the first frame to render" + } + }, + "frameStep": { + "type": "int", + "metadata": { + "description": "Incremental step in frame sequeunce" + } + }, + "frameEnd": { + "type": "int", + "metadata": { + "description": "Index of the last frame to render" + } + }, + "outputs": { + "type": "string", + "metadata": { + "description": "The file group where outputs will be stored" + } + } + }, + "jobPreparationTask": { + "resourceFiles": [ + { + "source": { + "fileGroup": "[parameters('projectData')]" + }, + "filePath": "assets\\" + }, + { + "blobSource": "[parameters('assetScript')]", + "filePath": "scripts\\renderPrep.mel" + }, + { + "blobSource": "[parameters('thumbScript')]", + "filePath": "thumbnail.py" + }, + { + "blobSource": "[parameters('workspace')]", + "filePath": "workspace.mel" + } + ], + "commandLine": "dir" + }, + "taskFactory": { + "type": "parametricSweep", + "parameterSets": [ + { + "start": "[parameters('frameStart')]", + "end": "[parameters('frameEnd')]", + "step": "[parameters('frameStep')]" + } + ], + "repeatTask": { + "displayName": "Frame {0}", + "commandLine": "subst X: %AZ_BATCH_JOB_PREP_WORKING_DIR%\\assets & render -renderer [parameters('renderer')] -proj \"%AZ_BATCH_JOB_PREP_WORKING_DIR%\" -verb -preRender renderPrep -rd \"%AZ_BATCH_TASK_WORKING_DIR%\\images\" -s {0} -e {0} \"[parameters('sceneFile')]\" & call mayapy %AZ_BATCH_JOB_PREP_WORKING_DIR%\\thumbnail.py %^errorlevel%", + "environmentSettings": [ + { + "name": "MAYA_SCRIPT_PATH", + "value": "%AZ_BATCH_JOB_PREP_WORKING_DIR%\\scripts" + }, + { + "name": "FLEXLM_TIMEOUT", + "value": "5000000" + } + ], + "outputFiles": [ + { + "filePattern": "images/**/*", + "destination": { + "autoStorage": { + "fileGroup": "[parameters('outputs')]" + } + }, + "uploadOptions": { + "uploadCondition": "taskSuccess" + } + }, + { + "filePattern": "thumbs/*.png", + "destination": { + "autoStorage": { + "fileGroup": "[parameters('outputs')]", + "path": "thumbs" + } + }, + "uploadOptions": { + "uploadCondition": "taskSuccess" + } + }, + { + "filePattern": "../stdout.txt", + "destination": { + "autoStorage": { + "fileGroup": "[parameters('outputs')]", + "path": "logs/frame_{0}.log" + } + }, + "uploadOptions": { + "uploadCondition": "taskCompletion" + } + }, + { + "filePattern": "../stderr.txt", + "destination": { + "autoStorage": { + "fileGroup": "[parameters('outputs')]", + "path": "logs/frame_{0}_error.log" + } + }, + "uploadOptions": { + "uploadCondition": "taskCompletion" + } + } + ] + } + }, + "onAllTasksComplete": "terminateJob" +} From 62ce77eb766ebfba75f53e2d35c916657dcf92e5 Mon Sep 17 00:00:00 2001 From: annatisch Date: Mon, 22 May 2017 16:12:13 -0700 Subject: [PATCH 07/29] Some bug fixes --- azure_batch_maya/plug-in/AzureBatch.py | 4 ++-- azure_batch_maya/scripts/config.py | 16 +++++++++------- azure_batch_maya/scripts/submission.py | 4 ++-- azure_batch_maya/scripts/ui/ui_shared.py | 3 +-- .../templates/mayaSoftware-basic-windows.json | 1 + tests/test_assets.py | 3 ++- tests/test_pools.py | 2 +- tests/test_submission.py | 7 ++++--- 8 files changed, 22 insertions(+), 18 deletions(-) diff --git a/azure_batch_maya/plug-in/AzureBatch.py b/azure_batch_maya/plug-in/AzureBatch.py index 2d019b5..cfac6a6 100644 --- a/azure_batch_maya/plug-in/AzureBatch.py +++ b/azure_batch_maya/plug-in/AzureBatch.py @@ -419,14 +419,14 @@ def initializePlugin(obj): print("Initializing Azure Batch plug-in") existing = cmds.optionVar(exists=EULA_PREF) if not existing: - agree = cmds.layoutDialog(ui=sla_prompt, title="Azure Batch Maya Client") + agree = cmds.layoutDialog(ui=eula_prompt, title="Azure Batch Maya Client") if str(agree) != 'Agree': raise RuntimeError("Plugin initialization aborted.") cmds.optionVar(stringValue=(EULA_PREF, VERSION)) else: agreed = cmds.optionVar(query=EULA_PREF) if StrictVersion(agreed) < VERSION: - agree = cmds.layoutDialog(ui=sla_prompt, title="Azure Batch Maya Client") + agree = cmds.layoutDialog(ui=eula_prompt, title="Azure Batch Maya Client") if str(agree) != 'Agree': raise RuntimeError("Plugin initialization aborted.") cmds.optionVar(stringValue=(EULA_PREF, VERSION)) diff --git a/azure_batch_maya/scripts/config.py b/azure_batch_maya/scripts/config.py index 133caa3..560f33c 100644 --- a/azure_batch_maya/scripts/config.py +++ b/azure_batch_maya/scripts/config.py @@ -40,7 +40,6 @@ import batch_extensions as batch from batch_extensions.batch_auth import SharedKeyCredentials -VERSION = os.environ['AZUREBATCH_VERSION'] LOG_LEVELS = { 'debug':10, 'info':20, @@ -64,7 +63,7 @@ class AzureBatchConfig(object): self._tab_index = index self._data_dir = os.path.join(maya.prefs_dir(), 'AzureBatchData') self._ini_file = "azure_batch.ini" - self._user_agent = "batchmaya/{}".format(VERSION) + self._user_agent = "batchmaya/{}".format(os.environ.get('AZUREBATCH_VERSION')) self._cfg = ConfigParser.ConfigParser() self._client = None self._log = None @@ -98,7 +97,7 @@ class AzureBatchConfig(object): os.makedirs(self._data_dir) config_file = os.path.join(self._data_dir, self._ini_file) if not os.path.exists(config_file): - self._log = self._configure_logging(10) + self._log = self._configure_logging(LOG_LEVELS['debug']) return try: self._cfg.read(config_file) @@ -112,14 +111,17 @@ class AzureBatchConfig(object): self._client = batch.BatchExtensionsClient( credentials, base_url=self._cfg.get('AzureBatch', 'batch_url'), storage_client=self._storage) - self._client._config.add_user_agent(self._user_agent) + self._client.config.add_user_agent(self._user_agent) self._log = self._configure_logging( self._cfg.get('AzureBatch', 'logging')) - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError) as exp: + except Exception as exp: # We should only worry about this if it happens when authenticating # using the UI, otherwise it's expected. if self.ui: - raise ValueError("Invalid Configuration File: {}".format(exp)) + raise ValueError("Invalid Configuration: {}".format(exp)) + else: + # We'll need a place holder logger + self._log = self._configure_logging(LOG_LEVELS['debug']) def _configure_logging(self, log_level): """Configure the logger. Setup the file output and format @@ -218,7 +220,7 @@ class AzureBatchConfig(object): self._configure_plugin() self._auth = self._auto_authentication() except ValueError as exp: - maya.error(exp) + maya.error(str(exp)) self._auth = False finally: self.ui.set_authenticate(self._auth) diff --git a/azure_batch_maya/scripts/submission.py b/azure_batch_maya/scripts/submission.py index 2bdbb84..2600ff1 100644 --- a/azure_batch_maya/scripts/submission.py +++ b/azure_batch_maya/scripts/submission.py @@ -61,6 +61,7 @@ class AzureBatchSubmission(object): self._call = call self._tab_index = index + self.max_pool_size = 1000 self.ui = SubmissionUI(self, frame) self.modules = self._collect_modules() self.renderer = None @@ -69,7 +70,6 @@ class AzureBatchSubmission(object): self.pool_manager = None self.env_manager = None self.batch = None - self.max_pool_size = 1000 #callback.after_new(self.ui.refresh) #callback.after_open(self.ui.refresh) @@ -272,7 +272,7 @@ class AzureBatchSubmission(object): batch_parameters['metadata'] = [{"name": "JobType", "value": "Maya"}] template_file = os.path.join( os.environ['AZUREBATCH_TEMPLATES'], - "{}-basic-{}.json".format(self.renderer.render_engine, pool_os.lower())) + "{}-basic-{}.json".format(self.renderer.render_engine, pool_os.value.lower())) batch_parameters['applicationTemplateInfo'] = {'filePath': template_file} application_params = {} batch_parameters['applicationTemplateInfo']['parameters'] = application_params diff --git a/azure_batch_maya/scripts/ui/ui_shared.py b/azure_batch_maya/scripts/ui/ui_shared.py index 79728c6..9e3edcd 100644 --- a/azure_batch_maya/scripts/ui/ui_shared.py +++ b/azure_batch_maya/scripts/ui/ui_shared.py @@ -77,8 +77,7 @@ class AzureBatchUI(object): """Select a specific tab for display. :param int tab: The index of the tab to display. """ - index = self.base.tab_index[tab] - maya.tab_layout(self.tab_display, edit=True, selectTabIndex=index) + maya.tab_layout(self.tab_display, edit=True, selectTabIndex=tab) def selected_tab(self): """Get the index of the currently selected tab. diff --git a/azure_batch_maya/templates/mayaSoftware-basic-windows.json b/azure_batch_maya/templates/mayaSoftware-basic-windows.json index 5ac1265..6f2be91 100644 --- a/azure_batch_maya/templates/mayaSoftware-basic-windows.json +++ b/azure_batch_maya/templates/mayaSoftware-basic-windows.json @@ -1,5 +1,6 @@ { "templateMetadata": { + "description": "Application template for working with Maya on Windows." }, "parameters": { "sceneFile": { diff --git a/tests/test_assets.py b/tests/test_assets.py index 24113d1..c560968 100644 --- a/tests/test_assets.py +++ b/tests/test_assets.py @@ -438,6 +438,7 @@ class TestAzureBatchAssets(unittest.TestCase): os.environ["AZUREBATCH_TEMPLATES"] = os.path.join(top_dir, 'azure_batch_maya', 'templates') os.environ["AZUREBATCH_MODULES"] = mod_dir os.environ["AZUREBATCH_SCRIPTS"] = "{0};{1};{2}".format(src_dir, ui_dir, tools_dir) + os.environ["AZUREBATCH_VERSION"] = "0.1" return super(TestAzureBatchAssets, self).setUp() @@ -445,7 +446,7 @@ class TestAzureBatchAssets(unittest.TestCase): @mock.patch("assets.callback") @mock.patch("assets.AssetsUI") def test_batchassets_create(self, mock_ui, mock_call, mock_collect): - assets = AzureBatchAssets("frame", "call") + assets = AzureBatchAssets(3, "frame", "call") mock_ui.assert_called_with(assets, "frame") mock_collect.assert_called_with() #mock_call.after_new.assert_called_with(assets.callback_refresh) diff --git a/tests/test_pools.py b/tests/test_pools.py index 0112655..c246c40 100644 --- a/tests/test_pools.py +++ b/tests/test_pools.py @@ -61,7 +61,7 @@ class AzureTestBatchPools(unittest.TestCase): @mock.patch("pools.PoolsUI") def test_pools_initialize(self, mock_ui): - pools = AzureBatchPools("frame", "call") + pools = AzureBatchPools(4, "frame", "call") mock_ui.assert_called_with(pools, "frame") def test_pools_configure(self): diff --git a/tests/test_submission.py b/tests/test_submission.py index e49537c..88d06ce 100644 --- a/tests/test_submission.py +++ b/tests/test_submission.py @@ -56,11 +56,11 @@ from batch_extensions import models from azure.storage.blob import BlockBlobService -LIVE = True def print_status(status): print(status) + class TestBatchSubmission(unittest.TestCase): def setUp(self): @@ -74,6 +74,7 @@ class TestBatchSubmission(unittest.TestCase): os.environ["AZUREBATCH_TEMPLATES"] = os.path.join(top_dir, 'azure_batch_maya', 'templates') os.environ["AZUREBATCH_MODULES"] = mod_dir os.environ["AZUREBATCH_SCRIPTS"] = "{0};{1};{2}".format(src_dir, ui_dir, tools_dir) + os.environ["AZUREBATCH_VERSION"] = "0.1" self.mock_self = mock.create_autospec(AzureBatchSubmission) self.mock_self.batch = mock.create_autospec(BatchExtensionsClient) self.mock_self.batch.job = mock.create_autospec(operations.ExtendedJobOperations) @@ -93,7 +94,7 @@ class TestBatchSubmission(unittest.TestCase): @mock.patch("submission.callback") @mock.patch("submission.SubmissionUI") def test_submission_create(self, mock_ui, mock_call, mock_mods): - submission = AzureBatchSubmission("frame", "call") + submission = AzureBatchSubmission(2, "frame", "call") mock_mods.assert_called_with() mock_ui.assert_called_with(submission, "frame") #mock_call.after_new.assert_called_with(mock.ANY) @@ -165,7 +166,7 @@ class TestBatchSubmission(unittest.TestCase): self.mock_self.pool_manager.create_auto_pool.return_value = {'autoPool': 'auto-pool'} self.mock_self.pool_manager.create_pool.return_value = {'poolId': 'new-pool'} self.mock_self.env_manager.get_environment_settings.return_value = [{'name':'foo', 'value':'bar'}] - self.mock_self.renderer = mock.Mock() + self.mock_self.renderer = mock.Mock(render_engine='arnold') self.mock_self.renderer.get_jobdata.return_value = ("a", "b") self.mock_self.renderer.get_params.return_value = {"foo": "bar"} self.mock_self.renderer.get_title.return_value = "job name" From de1508a7db24786bc76c7475ba466b4b3415c10a Mon Sep 17 00:00:00 2001 From: annatisch Date: Mon, 22 May 2017 16:12:23 -0700 Subject: [PATCH 08/29] reverted asset upload to threading --- azure_batch_maya/scripts/assets.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/azure_batch_maya/scripts/assets.py b/azure_batch_maya/scripts/assets.py index 61f8f2d..5de741e 100644 --- a/azure_batch_maya/scripts/assets.py +++ b/azure_batch_maya/scripts/assets.py @@ -28,7 +28,7 @@ import logging from datetime import datetime -import multiprocessing +import threading import os import sys import glob @@ -53,6 +53,7 @@ from default import AzureBatchRenderAssets SYS_SEARCHPATHS = [] USR_SEARCHPATHS = [] BYTES = 1024 +UPLOAD_THREADS = 10 class AzureBatchAssets(object): @@ -221,15 +222,18 @@ class AzureBatchAssets(object): return Asset(map_file, [], self.batch, self._log) def _upload_all(self, to_upload, progress, total, project): - """Upload all selected assets in multiple processes according to available cores.""" + """Upload all selected assets in 10 threads.""" + uploads_running = [] progress_queue = Queue() - for i in range(0, len(to_upload), multiprocessing.cpu_count()): - for index, asset in enumerate(to_upload[i:i + multiprocessing.cpu_count()]): + for i in range(0, len(to_upload), UPLOAD_THREADS): + for index, asset in enumerate(to_upload[i:i + UPLOAD_THREADS]): self._log.debug("Starting thread for asset: {}".format(asset.path)) - upload = multiprocessing.Process( + upload = threading.Thread( target=asset.upload, args=(index, progress, progress_queue, project)) upload.start() - while multiprocessing.active_children() or not progress_queue.empty(): + uploads_running.append(upload) + self._log.debug("Batch of asset uploads pending: {}".format(threading.active_count())) + while any(t for t in uploads_running if t.is_alive()) or not progress_queue.empty(): uploaded = progress_queue.get() if isinstance(uploaded, Exception): raise uploaded From e931f9e5e78677fbbd4cf37b4980ab95aceba113 Mon Sep 17 00:00:00 2001 From: annatisch Date: Tue, 23 May 2017 09:00:14 -0700 Subject: [PATCH 09/29] Fixed windows template --- .../templates/arnold-basic-windows.json | 195 +++++++++--------- 1 file changed, 98 insertions(+), 97 deletions(-) diff --git a/azure_batch_maya/templates/arnold-basic-windows.json b/azure_batch_maya/templates/arnold-basic-windows.json index 62b6ea4..9539097 100644 --- a/azure_batch_maya/templates/arnold-basic-windows.json +++ b/azure_batch_maya/templates/arnold-basic-windows.json @@ -73,102 +73,103 @@ "metadata": { "description": "Arnold logging verbosity" } - }, - "jobPreparationTask": { - "resourceFiles": [ - { - "source": { - "fileGroup": "[parameters('projectData')]" - }, - "filePath": "assets\\" + } + }, + "jobPreparationTask": { + "resourceFiles": [ + { + "source": { + "fileGroup": "[parameters('projectData')]" }, - { - "blobSource": "[parameters('assetScript')]", - "filePath": "scripts\\renderPrep.mel" - }, - { - "blobSource": "[parameters('thumbScript')]", - "filePath": "thumbnail.py" - }, - { - "blobSource": "[parameters('workspace')]", - "filePath": "workspace.mel" - } - ], - "commandLine": "dir" - }, - "taskFactory": { - "type": "parametricSweep", - "parameterSets": [ - { - "start": "[parameters('frameStart')]", - "end": "[parameters('frameEnd')]", - "step": "[parameters('frameStep')]" - } - ], - "repeatTask": { - "displayName": "Frame {0}", - "commandLine": "subst X: %AZ_BATCH_JOB_PREP_WORKING_DIR%\\assets & render -renderer [parameters('renderer')] -proj \"%AZ_BATCH_JOB_PREP_WORKING_DIR%\" -ai:ltc 1 -ai:lve [parameters('logLevel')] -verb -preRender renderPrep -rd \"%AZ_BATCH_TASK_WORKING_DIR%\\images\" -s {0} -e {0} \"[parameters('sceneFile')]\" & call mayapy %AZ_BATCH_JOB_PREP_WORKING_DIR%\\thumbnail.py %^errorlevel%", - "environmentSettings": [ - { - "name": "MAYA_SCRIPT_PATH", - "value": "%AZ_BATCH_JOB_PREP_WORKING_DIR%\\scripts" - }, - { - "name": "FLEXLM_TIMEOUT", - "value": "5000000" - } - ], - "outputFiles": [ - { - "filePattern": "images/**/*", - "destination": { - "autoStorage": { - "fileGroup": "[parameters('outputs')]" - } - }, - "uploadOptions": { - "uploadCondition": "taskSuccess" - } - }, - { - "filePattern": "thumbs/*.png", - "destination": { - "autoStorage": { - "fileGroup": "[parameters('outputs')]", - "path": "thumbs" - } - }, - "uploadOptions": { - "uploadCondition": "taskSuccess" - } - }, - { - "filePattern": "../stdout.txt", - "destination": { - "autoStorage": { - "fileGroup": "[parameters('outputs')]", - "path": "logs/frame_{0}.log" - } - }, - "uploadOptions": { - "uploadCondition": "taskCompletion" - } - }, - { - "filePattern": "../stderr.txt", - "destination": { - "autoStorage": { - "fileGroup": "[parameters('outputs')]", - "path": "logs/frame_{0}_error.log" - } - }, - "uploadOptions": { - "uploadCondition": "taskCompletion" - } - } - ] + "filePath": "assets\\" + }, + { + "blobSource": "[parameters('assetScript')]", + "filePath": "scripts\\renderPrep.mel" + }, + { + "blobSource": "[parameters('thumbScript')]", + "filePath": "thumbnail.py" + }, + { + "blobSource": "[parameters('workspace')]", + "filePath": "workspace.mel" } - }, - "onAllTasksComplete": "terminateJob" - } + ], + "commandLine": "dir" + }, + "taskFactory": { + "type": "parametricSweep", + "parameterSets": [ + { + "start": "[parameters('frameStart')]", + "end": "[parameters('frameEnd')]", + "step": "[parameters('frameStep')]" + } + ], + "repeatTask": { + "displayName": "Frame {0}", + "commandLine": "subst X: %AZ_BATCH_JOB_PREP_WORKING_DIR%\\assets & render -renderer [parameters('renderer')] -proj \"%AZ_BATCH_JOB_PREP_WORKING_DIR%\" -ai:ltc 1 -ai:lve [parameters('logLevel')] -verb -preRender renderPrep -rd \"%AZ_BATCH_TASK_WORKING_DIR%\\images\" -s {0} -e {0} \"[parameters('sceneFile')]\" & call mayapy %AZ_BATCH_JOB_PREP_WORKING_DIR%\\thumbnail.py %^errorlevel%", + "environmentSettings": [ + { + "name": "MAYA_SCRIPT_PATH", + "value": "%AZ_BATCH_JOB_PREP_WORKING_DIR%\\scripts" + }, + { + "name": "FLEXLM_TIMEOUT", + "value": "5000000" + } + ], + "outputFiles": [ + { + "filePattern": "images/**/*", + "destination": { + "autoStorage": { + "fileGroup": "[parameters('outputs')]" + } + }, + "uploadOptions": { + "uploadCondition": "taskSuccess" + } + }, + { + "filePattern": "thumbs/*.png", + "destination": { + "autoStorage": { + "fileGroup": "[parameters('outputs')]", + "path": "thumbs" + } + }, + "uploadOptions": { + "uploadCondition": "taskSuccess" + } + }, + { + "filePattern": "../stdout.txt", + "destination": { + "autoStorage": { + "fileGroup": "[parameters('outputs')]", + "path": "logs/frame_{0}.log" + } + }, + "uploadOptions": { + "uploadCondition": "taskCompletion" + } + }, + { + "filePattern": "../stderr.txt", + "destination": { + "autoStorage": { + "fileGroup": "[parameters('outputs')]", + "path": "logs/frame_{0}_error.log" + } + }, + "uploadOptions": { + "uploadCondition": "taskCompletion" + } + } + ] + } + }, + "onAllTasksComplete": "terminateJob" +} From b6ee673726efc1e1f064e2333ff86e92fe075fb9 Mon Sep 17 00:00:00 2001 From: annatisch Date: Tue, 23 May 2017 09:13:11 -0700 Subject: [PATCH 10/29] Updated license headers --- AzureBatchMaya.pyproj | 30 ++++--- azure_batch_maya/modules/arnold_renderer.py | 40 +++------- azure_batch_maya/modules/default.py | 31 +------- azure_batch_maya/modules/maya_software.py | 33 ++------ azure_batch_maya/plug-in/AzureBatch.py | 31 +------- azure_batch_maya/scripts/api.py | 31 +------- azure_batch_maya/scripts/assets.py | 31 +------- .../batch_extensions/_template_utils.py | 2 +- azure_batch_maya/scripts/config.py | 31 +------- azure_batch_maya/scripts/environment.py | 31 +------- azure_batch_maya/scripts/exception.py | 31 +------- azure_batch_maya/scripts/jobhistory.py | 31 +------- azure_batch_maya/scripts/pools.py | 31 +------- azure_batch_maya/scripts/shared.py | 31 +------- azure_batch_maya/scripts/submission.py | 31 +------- azure_batch_maya/scripts/utils.py | 31 +------- tests/__init__.py | 79 ------------------- tests/test_assets.py | 33 ++------ tests/test_extensions.py | 1 - tests/test_jobwatcher.py | 5 ++ tests/test_pools.py | 32 +------- tests/test_submission.py | 31 +------- 22 files changed, 102 insertions(+), 556 deletions(-) delete mode 100644 tests/__init__.py diff --git a/AzureBatchMaya.pyproj b/AzureBatchMaya.pyproj index dfab7f3..2ec5b3e 100644 --- a/AzureBatchMaya.pyproj +++ b/AzureBatchMaya.pyproj @@ -10,10 +10,8 @@ . Maya.Client Maya.Client - - - - + {a76b797f-9426-4cd7-a078-9af9c9e4b437} + 2.7 azure_batch_maya\scripts\;azure_batch_maya\scripts\ui\;tests\data\;tests\data\modules\ @@ -89,7 +87,7 @@ Code - + Code @@ -115,7 +113,7 @@ Code - + Code @@ -149,9 +147,6 @@ Code - - Code - @@ -182,7 +177,7 @@ - + Code @@ -194,6 +189,8 @@ + + Code @@ -206,6 +203,19 @@ Code + + + {a76b797f-9426-4cd7-a078-9af9c9e4b437} + {9a7a9026-48c1-4688-9d5d-e5699d47d074} + 2.7 + env (Python 64-bit 2.7) + Scripts\python.exe + Scripts\pythonw.exe + Lib\ + PYTHONPATH + Amd64 + + 10.0 $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\Python Tools\Microsoft.PythonTools.targets diff --git a/azure_batch_maya/modules/arnold_renderer.py b/azure_batch_maya/modules/arnold_renderer.py index 027019d..36acb35 100644 --- a/azure_batch_maya/modules/arnold_renderer.py +++ b/azure_batch_maya/modules/arnold_renderer.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os import sys @@ -57,7 +34,7 @@ class ArnoldRenderJob(AzureBatchRenderJob): job_name = "Untitled" else: job_name = str(os.path.splitext(os.path.basename(self.scene_name))[0]) - file_prefix = mel.eval("getAttr defaultRenderGlobals.imageFilePrefix") + file_prefix = cmds.getAttr("defaultRenderGlobals.imageFilePrefix") if file_prefix: file_prefix = os.path.split(file_prefix)[1] else: @@ -68,7 +45,10 @@ class ArnoldRenderJob(AzureBatchRenderJob): self.end = self.display_int("End frame: ", self.end_frame, edit=True) self.step = self.display_int("Frame step: ", self.frame_step, edit=True) - log_level = mel.eval("getAttr defaultArnoldRenderOptions.log_verbosity") + try: + log_level = cmds.getAttr("defaultArnoldRenderOptions.log_verbosity") + except ValueError: + log_level = 1 self.logging = self.display_menu("Logging: ", self.log_levels, log_level+1) def get_title(self): @@ -96,7 +76,7 @@ class ArnoldRenderJob(AzureBatchRenderJob): cancelButton=options['cancel'], dismissString=options['cancel']) if answer == options['cancel']: - raise Exception("Submission aborted") + raise Exception("Submission cancelled") if answer == options['save']: cmds.SaveScene() return self.scene_name, [self.scene_name] diff --git a/azure_batch_maya/modules/default.py b/azure_batch_maya/modules/default.py index b86be21..9eed7fb 100644 --- a/azure_batch_maya/modules/default.py +++ b/azure_batch_maya/modules/default.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os diff --git a/azure_batch_maya/modules/maya_software.py b/azure_batch_maya/modules/maya_software.py index fb14da2..8a32da1 100644 --- a/azure_batch_maya/modules/maya_software.py +++ b/azure_batch_maya/modules/maya_software.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os import sys @@ -86,7 +63,7 @@ class AzureBatchMayaJob(AzureBatchRenderJob): cancelButton=options['cancel'], dismissString=options['cancel']) if answer == options['cancel']: - raise Exception("Submission aborted") + raise Exception("Submission cancelled") if answer == options['save']: cmds.SaveScene() return self.scene_name, [self.scene_name] diff --git a/azure_batch_maya/plug-in/AzureBatch.py b/azure_batch_maya/plug-in/AzureBatch.py index cfac6a6..762d202 100644 --- a/azure_batch_maya/plug-in/AzureBatch.py +++ b/azure_batch_maya/plug-in/AzureBatch.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import urllib import os diff --git a/azure_batch_maya/scripts/api.py b/azure_batch_maya/scripts/api.py index 082b025..763ba7d 100644 --- a/azure_batch_maya/scripts/api.py +++ b/azure_batch_maya/scripts/api.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- try: from maya import cmds, mel, utils diff --git a/azure_batch_maya/scripts/assets.py b/azure_batch_maya/scripts/assets.py index 5de741e..6f6696b 100644 --- a/azure_batch_maya/scripts/assets.py +++ b/azure_batch_maya/scripts/assets.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import logging from datetime import datetime diff --git a/azure_batch_maya/scripts/batch_extensions/_template_utils.py b/azure_batch_maya/scripts/batch_extensions/_template_utils.py index b23e286..8b432b0 100644 --- a/azure_batch_maya/scripts/batch_extensions/_template_utils.py +++ b/azure_batch_maya/scripts/batch_extensions/_template_utils.py @@ -436,7 +436,7 @@ def _parse_arm_parameter(name, template_obj, parameters): user_value = user_value['value'] except TypeError: pass - if not user_value: + if user_value is None: raise ValueError("No value supplied for parameter '{}' and no default value".format(name)) if isinstance(user_value, dict): # If substitute value is a complex object - it may require diff --git a/azure_batch_maya/scripts/config.py b/azure_batch_maya/scripts/config.py index 560f33c..6b22a39 100644 --- a/azure_batch_maya/scripts/config.py +++ b/azure_batch_maya/scripts/config.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import ConfigParser import os diff --git a/azure_batch_maya/scripts/environment.py b/azure_batch_maya/scripts/environment.py index 9ef3afc..c9459d9 100644 --- a/azure_batch_maya/scripts/environment.py +++ b/azure_batch_maya/scripts/environment.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os import logging diff --git a/azure_batch_maya/scripts/exception.py b/azure_batch_maya/scripts/exception.py index bf68332..93559ae 100644 --- a/azure_batch_maya/scripts/exception.py +++ b/azure_batch_maya/scripts/exception.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import traceback import logging diff --git a/azure_batch_maya/scripts/jobhistory.py b/azure_batch_maya/scripts/jobhistory.py index 28e48fb..848a6d6 100644 --- a/azure_batch_maya/scripts/jobhistory.py +++ b/azure_batch_maya/scripts/jobhistory.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os import logging diff --git a/azure_batch_maya/scripts/pools.py b/azure_batch_maya/scripts/pools.py index 4329b77..adf502e 100644 --- a/azure_batch_maya/scripts/pools.py +++ b/azure_batch_maya/scripts/pools.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os import logging diff --git a/azure_batch_maya/scripts/shared.py b/azure_batch_maya/scripts/shared.py index 770029d..c424d70 100644 --- a/azure_batch_maya/scripts/shared.py +++ b/azure_batch_maya/scripts/shared.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import logging import os diff --git a/azure_batch_maya/scripts/submission.py b/azure_batch_maya/scripts/submission.py index 2600ff1..6b46548 100644 --- a/azure_batch_maya/scripts/submission.py +++ b/azure_batch_maya/scripts/submission.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os import sys diff --git a/azure_batch_maya/scripts/utils.py b/azure_batch_maya/scripts/utils.py index ccae961..4bc1b49 100644 --- a/azure_batch_maya/scripts/utils.py +++ b/azure_batch_maya/scripts/utils.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- from enum import Enum import os diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e7b8e1e..0000000 --- a/tests/__init__.py +++ /dev/null @@ -1,79 +0,0 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- - - -import sys -import os - - -if sys.version_info[:2] < (2, 7, ): - try: - import unittest2 - from unittest2 import TestLoader, TextTestRunner - - except ImportError: - print("The Batch Maya Plugin test suite requires " - "the unittest2 package to run on Python 2.6 and " - "below.\nPlease install this package to continue.") - sys.exit() -else: - import unittest - from unittest import TestLoader, TextTestRunner - -if sys.version_info[:2] >= (3, 3, ): - from unittest import mock -else: - try: - import mock - except ImportError: - print("The Batch Maya Plugin test suite requires " - "the mock package to run on Python 3.2 and below.\n" - "Please install this package to continue.") - raise - - -if __name__ == '__main__': - - runner = TextTestRunner(verbosity=2) - - test_dir = os.path.dirname(__file__) - top_dir = os.path.dirname(test_dir) - src_dir = os.path.join(top_dir, 'azure_batch_maya', 'scripts') - mod_dir = os.path.join(test_dir, 'data', 'modules') - ui_dir = os.path.join(src_dir, 'ui') - tools_dir = os.path.join(src_dir, 'tools') - os.environ["AZUREBATCH_ICONS"] = os.path.join(top_dir, 'azure_batch_maya', 'icons') - os.environ["AZUREBATCH_MODULES"] = mod_dir - os.environ["AZUREBATCH_SCRIPTS"] = "{0};{1};{2}".format(src_dir, ui_dir, tools_dir) - sys.path.extend([src_dir, ui_dir, tools_dir, mod_dir]) - - test_loader = TestLoader() - suite = test_loader.discover(test_dir, - pattern="test_*.py", - top_level_dir=top_dir) - runner.run(suite) diff --git a/tests/test_assets.py b/tests/test_assets.py index c560968..c257900 100644 --- a/tests/test_assets.py +++ b/tests/test_assets.py @@ -1,31 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- - +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import sys import os @@ -49,6 +25,7 @@ from assets import Asset, Assets, AzureBatchAssets from exception import FileUploadException from utils import ProgressBar, ProcButton + class TestAsset(unittest.TestCase): def setUp(self): diff --git a/tests/test_extensions.py b/tests/test_extensions.py index d6de9ce..ea61c88 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -20,7 +20,6 @@ from batch_extensions import _pool_utils as pool_utils from batch_extensions import _file_utils as file_utils - class TestBatchExtensions(unittest.TestCase): # pylint: disable=attribute-defined-outside-init,no-member,too-many-public-methods diff --git a/tests/test_jobwatcher.py b/tests/test_jobwatcher.py index 81961e3..44d6dd2 100644 --- a/tests/test_jobwatcher.py +++ b/tests/test_jobwatcher.py @@ -1,3 +1,8 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + try: import unittest2 as unittest except ImportError: diff --git a/tests/test_pools.py b/tests/test_pools.py index c246c40..fa81754 100644 --- a/tests/test_pools.py +++ b/tests/test_pools.py @@ -1,31 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- - +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import sys import os diff --git a/tests/test_submission.py b/tests/test_submission.py index 88d06ce..d1b32f3 100644 --- a/tests/test_submission.py +++ b/tests/test_submission.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import sys import os From 1cc2990a9f9c7a778deb3290361a5f6215ad6aac Mon Sep 17 00:00:00 2001 From: annatisch Date: Tue, 23 May 2017 10:47:23 -0700 Subject: [PATCH 11/29] Support using pools made in the portal --- azure_batch_maya/scripts/pools.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/azure_batch_maya/scripts/pools.py b/azure_batch_maya/scripts/pools.py index adf502e..1dc24fb 100644 --- a/azure_batch_maya/scripts/pools.py +++ b/azure_batch_maya/scripts/pools.py @@ -56,10 +56,15 @@ class AzureBatchPools(object): """ #if lazy and self.pools: # return [pool.id for pool in self.pools if not pool.auto] - self.pools = [p for p in self._call(self.batch.pool.list) if p.id.startswith('Maya_')] + all_pools = self._call(self.batch.pool.list) + self.pools = [] + for pool in all_pools: + if pool.virtual_machine_configuration and \ + pool.virtual_machine_configuration.image_reference.publisher == 'batch': + self.pools.append(pool) self.pools.sort(key=lambda x: x.creation_time, reverse=True) self.count = len(self.pools) - return [pool.id for pool in self.pools if pool.id.startswith("Maya_Pool")] + return [pool.id for pool in self.pools if not pool.id.startswith("Maya_Auto_Pool")] def get_pools(self): """Retrieves the currently running pools and populates the UI From 5961e084cea07d655d155f4d5a1f26881f8bf37c Mon Sep 17 00:00:00 2001 From: annatisch Date: Wed, 24 May 2017 10:45:21 -0700 Subject: [PATCH 12/29] Fixed bug for displaying unknown pool image --- azure_batch_maya/scripts/environment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_batch_maya/scripts/environment.py b/azure_batch_maya/scripts/environment.py index c9459d9..5fd3171 100644 --- a/azure_batch_maya/scripts/environment.py +++ b/azure_batch_maya/scripts/environment.py @@ -120,8 +120,8 @@ class AzureBatchEnvironment(object): if pool_image: return pool_image[0] else: - self._log.debug("Pool using unknown image reference: {}".format(image_ref['offer'])) - return "" + self._log.debug("Pool using unknown image reference: {}".format(image_ref.offer)) + return image_ref.offer def get_vm_sku(self): return self.ui.get_sku() From c3822773a81b1b45297b40f92cef4072f664dcf9 Mon Sep 17 00:00:00 2001 From: annatisch Date: Wed, 24 May 2017 15:15:35 -0700 Subject: [PATCH 13/29] Updated tests --- tests/test_pools.py | 8 ++++++++ tests/test_submission.py | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/test_pools.py b/tests/test_pools.py index fa81754..8d78449 100644 --- a/tests/test_pools.py +++ b/tests/test_pools.py @@ -50,9 +50,15 @@ class AzureTestBatchPools(unittest.TestCase): self.mock_self.batch.pool = mock.create_autospec(batch.operations.ExtendedPoolOperations) pool1 = mock.create_autospec(models.CloudPool) pool1.id = "12345" + pool1.virtual_machine_configuration = mock.create_autospec(models.VirtualMachineConfiguration) + pool1.virtual_machine_configuration.image_reference = mock.create_autospec(models.ImageReference) + pool1.virtual_machine_configuration.image_reference.publisher = "MicrosoftWindows" pool1.creation_time = datetime.datetime.now() pool2 = mock.create_autospec(models.CloudPool) pool2.id = "67890" + pool2.virtual_machine_configuration = mock.create_autospec(models.VirtualMachineConfiguration) + pool2.virtual_machine_configuration.image_reference = mock.create_autospec(models.ImageReference) + pool2.virtual_machine_configuration.image_reference.publisher = "LinuxUbuntu" pool2.creation_time = datetime.datetime.now() self.mock_self._call = lambda x: [pool1, pool2] @@ -61,7 +67,9 @@ class AzureTestBatchPools(unittest.TestCase): self.assertEqual(len(self.mock_self.pools), 0) pool1.id = "Maya_Pool_A" + pool1.virtual_machine_configuration.image_reference.publisher = "batch" pool2.id = "Maya_Auto_Pool_B" + pool2.virtual_machine_configuration.image_reference.publisher = "batch" ids = AzureBatchPools.list_pools(self.mock_self) self.assertEqual(ids, ["Maya_Pool_A"]) self.assertEqual(len(self.mock_self.pools), 2) diff --git a/tests/test_submission.py b/tests/test_submission.py index d1b32f3..2ce9d10 100644 --- a/tests/test_submission.py +++ b/tests/test_submission.py @@ -25,6 +25,7 @@ from pools import AzureBatchPools from environment import AzureBatchEnvironment from shared import AzureBatchSettings from exception import CancellationException +from utils import OperatingSystem from batch_extensions import BatchExtensionsClient from batch_extensions.batch_auth import SharedKeyCredentials @@ -139,7 +140,7 @@ class TestBatchSubmission(unittest.TestCase): mock_utils.format_scene_path.return_value = "test_file_path" self.mock_self._configure_pool = lambda t: AzureBatchSubmission._configure_pool(self.mock_self, t) self.mock_self._check_plugins.return_value = [] - self.mock_self._get_os_flavor.return_value = 'Windows' + self.mock_self._get_os_flavor.return_value = OperatingSystem.windows self.mock_self.pool_manager.create_auto_pool.return_value = {'autoPool': 'auto-pool'} self.mock_self.pool_manager.create_pool.return_value = {'poolId': 'new-pool'} self.mock_self.env_manager.get_environment_settings.return_value = [{'name':'foo', 'value':'bar'}] From 099d7797617fac18a41d42fb51c45209be85fe47 Mon Sep 17 00:00:00 2001 From: annatisch Date: Wed, 24 May 2017 15:23:19 -0700 Subject: [PATCH 14/29] Updated to version 0.9.1 --- CHANGES.txt | 11 ++++++++++- LICENSE.txt | 2 +- azure_batch_maya/plug-in/AzureBatch.py | 2 +- package.py | 2 +- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 346acc1..99648bf 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1 +1,10 @@ -v0.9.0 - Core re-written to use latest Azure Batch Python SDK and Batch Extensions template files. \ No newline at end of file + +2017-05-24 v0.9.1 +----------------- +- Expanded Pool display UI to show pools created via the Azure portal +- Fixed bug where Pool couldn't be displayed if created with a non-current image + + +2017-05-23 v0.9.0 +----------------- +- Core re-written to use latest Azure Batch Python SDK and Batch Extensions template files. \ No newline at end of file diff --git a/LICENSE.txt b/LICENSE.txt index ff0015b..f6fba2d 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,5 +1,5 @@ -The Azure Batch Maya Sample ver. 0.9.0 +The Azure Batch Maya Sample ver. 0.9.1 Copyright (c) Microsoft Corporation All rights reserved. diff --git a/azure_batch_maya/plug-in/AzureBatch.py b/azure_batch_maya/plug-in/AzureBatch.py index 762d202..ba512c0 100644 --- a/azure_batch_maya/plug-in/AzureBatch.py +++ b/azure_batch_maya/plug-in/AzureBatch.py @@ -43,7 +43,7 @@ NAMESPACE_PACAKGES = [ "azure-storage==0.32.0", ] -VERSION = "0.9.0" +VERSION = "0.9.1" EULA_PREF = "AzureBatch_EULA" SHELF_FILE = "shelf_AzureBatch.mel" cmd_name = "AzureBatch" diff --git a/package.py b/package.py index 0918f81..d5deda6 100644 --- a/package.py +++ b/package.py @@ -32,7 +32,7 @@ import subprocess import shutil import zipfile -VERSION = "0.9.0" +VERSION = "0.9.1" def main(): """Build Maya Plug-in package""" From 0a73b5757b576fc594d67bc03ccf985966db2836 Mon Sep 17 00:00:00 2001 From: DavidKydd Date: Mon, 12 Jun 2017 16:18:15 +1200 Subject: [PATCH 15/29] Update Maya images as old ones were broken by a a validation change in a dependency (azure marketplace) --- azure_batch_maya/scripts/environment.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azure_batch_maya/scripts/environment.py b/azure_batch_maya/scripts/environment.py index 5fd3171..456bf8e 100644 --- a/azure_batch_maya/scripts/environment.py +++ b/azure_batch_maya/scripts/environment.py @@ -18,16 +18,16 @@ MAYA_IMAGES = { { 'node_sku_id': 'batch.node.windows amd64', 'publisher': 'batch', - 'offer': 'autodesk-maya-arnold-win2016-preview', - 'sku': 'maya2017', + 'offer': 'rendering-windows2016', + 'sku': 'rendering', 'version': 'latest' }, 'Batch CentOS Preview': { 'node_sku_id': 'batch.node.centos 7', 'publisher': 'batch', - 'offer': 'autodesk-maya-arnold-centos73-preview', - 'sku': 'maya2017', + 'offer': 'autodesk-maya-arnold-centos73', + 'sku': 'maya-arnold-2017', 'version': 'latest' }, } From 1a349ac4a078d4eefdc5d92919ce026908ab09b3 Mon Sep 17 00:00:00 2001 From: annatisch Date: Mon, 12 Jun 2017 14:50:49 -0700 Subject: [PATCH 16/29] Fix for job watcher OS --- azure_batch_maya/scripts/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_batch_maya/scripts/utils.py b/azure_batch_maya/scripts/utils.py index 4bc1b49..e99c759 100644 --- a/azure_batch_maya/scripts/utils.py +++ b/azure_batch_maya/scripts/utils.py @@ -431,12 +431,12 @@ class JobWatcher(object): self.job_watcher = os.path.join( os.path.dirname(__file__), "tools", "job_watcher.py") platform = get_os() - if platform == OperatingSystem.windows: + if platform == OperatingSystem.windows.value: self.proc_cmd = 'system("WMIC PROCESS where (Name=\'mayapy.exe\') get Commandline")' self.start_cmd = 'system("start mayapy {0}")' self.quotes = '\\"' self.splitter = 'mayapy' - elif platform == OperatingSystem.darwin: + elif platform == OperatingSystem.darwin.value: self.proc_cmd = 'system("ps -ef")' self.start_cmd = 'system("osascript -e \'tell application \\"Terminal\\" to do script \\"python {0}\\"\'")' self.quotes = '\\\\\\"' From 9760bdc773c6d6b968bfc56dd110145d0cffe821 Mon Sep 17 00:00:00 2001 From: annatisch Date: Mon, 12 Jun 2017 14:52:59 -0700 Subject: [PATCH 17/29] Bumped version --- CHANGES.txt | 6 ++++++ LICENSE.txt | 2 +- azure_batch_maya/plug-in/AzureBatch.py | 2 +- package.py | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 99648bf..a2435de 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,10 @@ +2017-06-12 v0.9.2 +----------------- +- Fixed bug in OS detection of Job Watcher +- Updated VM Image references + + 2017-05-24 v0.9.1 ----------------- - Expanded Pool display UI to show pools created via the Azure portal diff --git a/LICENSE.txt b/LICENSE.txt index f6fba2d..a37a50a 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,5 +1,5 @@ -The Azure Batch Maya Sample ver. 0.9.1 +The Azure Batch Maya Sample ver. 0.9.2 Copyright (c) Microsoft Corporation All rights reserved. diff --git a/azure_batch_maya/plug-in/AzureBatch.py b/azure_batch_maya/plug-in/AzureBatch.py index ba512c0..6c8c876 100644 --- a/azure_batch_maya/plug-in/AzureBatch.py +++ b/azure_batch_maya/plug-in/AzureBatch.py @@ -43,7 +43,7 @@ NAMESPACE_PACAKGES = [ "azure-storage==0.32.0", ] -VERSION = "0.9.1" +VERSION = "0.9.2" EULA_PREF = "AzureBatch_EULA" SHELF_FILE = "shelf_AzureBatch.mel" cmd_name = "AzureBatch" diff --git a/package.py b/package.py index d5deda6..41dd003 100644 --- a/package.py +++ b/package.py @@ -32,7 +32,7 @@ import subprocess import shutil import zipfile -VERSION = "0.9.1" +VERSION = "0.9.2" def main(): """Build Maya Plug-in package""" From 07322142810084cd8a273b9264864a2ec3d29295 Mon Sep 17 00:00:00 2001 From: annatisch Date: Fri, 23 Jun 2017 07:15:03 -0700 Subject: [PATCH 18/29] Added output container sas caching --- azure_batch_maya/scripts/batch_extensions/_file_utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/azure_batch_maya/scripts/batch_extensions/_file_utils.py b/azure_batch_maya/scripts/batch_extensions/_file_utils.py index 4edf860..5ee2201 100644 --- a/azure_batch_maya/scripts/batch_extensions/_file_utils.py +++ b/azure_batch_maya/scripts/batch_extensions/_file_utils.py @@ -226,6 +226,7 @@ class FileUtils(object): def __init__(self, get_storage_client): self.resource_file_cache = {} + self.container_sas_cache = {} self.resolve_storage_account = get_storage_client def filter_resource_cache(self, container, prefix): @@ -264,7 +265,11 @@ class FileUtils(object): def get_container_sas(self, file_group_name): storage_client = self.resolve_storage_account() container = _get_container_name(file_group_name) - return _generate_container_sas_token(container, storage_client) + try: + return self.container_sas_cache[container] + except KeyError: + self.container_sas_cache[container] = _generate_container_sas_token(container, storage_client) + return self.container_sas_cache[container] def get_container_list(self, source): """List blob references in container.""" From 8b55785d9a31a2e240152071dad627149e3ad4c7 Mon Sep 17 00:00:00 2001 From: annatisch Date: Fri, 23 Jun 2017 07:15:26 -0700 Subject: [PATCH 19/29] Improved storage account resolution --- .../batch_extensions/batch_extensions_client.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/azure_batch_maya/scripts/batch_extensions/batch_extensions_client.py b/azure_batch_maya/scripts/batch_extensions/batch_extensions_client.py index 7f73314..6da3ffc 100644 --- a/azure_batch_maya/scripts/batch_extensions/batch_extensions_client.py +++ b/azure_batch_maya/scripts/batch_extensions/batch_extensions_client.py @@ -106,11 +106,11 @@ class BatchExtensionsClient(BatchServiceClient): # the Batch account in the subscription # Example URL: https://batchaccount.westus.batch.azure.com region = urlsplit(self.config.base_url).netloc.split('.', 2)[1] - accounts = [x for x in client.batch_account.list() - if x.name == self._account and x.location == region] + accounts = (x for x in client.batch_account.list() + if x.name == self._account and x.location == region) try: - account = accounts[0] - except IndexError: + account = next(accounts) + except StopIteration: raise ValueError('Couldn\'t find the account named {} in subscription {} ' 'in region {}'.format( self._account, self._subscription, region)) @@ -125,7 +125,7 @@ class BatchExtensionsClient(BatchServiceClient): keys = storage_client.storage_accounts.list_keys(storage_resource_group, storage_account) storage_key = keys.keys[0].value # pylint: disable=no-member - self.resolved_storage_client = CloudStorageAccount(storage_account, storage_key)\ + self._resolved_storage_client = CloudStorageAccount(storage_account, storage_key)\ .create_block_blob_service() - return self.resolved_storage_client + return self._resolved_storage_client From 4c07ae8e33f4e9b3c155a6d89c3870f75720f901 Mon Sep 17 00:00:00 2001 From: annatisch Date: Fri, 23 Jun 2017 07:17:55 -0700 Subject: [PATCH 20/29] Added threading to task submission and output download --- .../scripts/batch_extensions/_job_utils.py | 31 ++++++++++++++++--- azure_batch_maya/scripts/tools/job_watcher.py | 29 +++++++++-------- 2 files changed, 43 insertions(+), 17 deletions(-) diff --git a/azure_batch_maya/scripts/batch_extensions/_job_utils.py b/azure_batch_maya/scripts/batch_extensions/_job_utils.py index 94bd3b7..2ebe057 100644 --- a/azure_batch_maya/scripts/batch_extensions/_job_utils.py +++ b/azure_batch_maya/scripts/batch_extensions/_job_utils.py @@ -5,7 +5,12 @@ from msrest.exceptions import ValidationError, ClientRequestError from azure.batch.models import BatchErrorException - +import threading +try: + from queue import Queue +except ImportError: + from Queue import Queue + # pylint: disable=too-few-public-methods @@ -25,16 +30,34 @@ def _handle_batch_exception(action): raise Exception(ex) +def _bulk_add_tasks(client, job_id, tasks, queue): + added_tasks = client.add_collection(job_id, tasks) + for task in added_tasks.value: + queue.put(task) + def deploy_tasks(client, job_id, tasks): MAX_TASKS_COUNT_IN_BATCH = 100 + MAX_SUBMIT_THREADS = 10 def add_task(): start = 0 - while start < len(tasks): + progress_queue = Queue() + submitting_tasks = [] + submitted_tasks = [] + while True: end = min(start + MAX_TASKS_COUNT_IN_BATCH, len(tasks)) - client.add_collection(job_id, tasks[start:end]) + submit = threading.Thread(target=_bulk_add_tasks, args=(client, job_id, tasks[start:end], progress_queue)) + submit.start() + submitting_tasks.append(submit) start = end - + if start >= len(tasks) or len(submitting_tasks) >= MAX_SUBMIT_THREADS: + while any(s for s in submitting_tasks if s.is_alive()) or not progress_queue.empty(): + submitted_tasks.append(progress_queue.get()) + progress_queue.task_done() + submitting_tasks = [] + if start >= len(tasks): + break + return submitted_tasks _handle_batch_exception(add_task) diff --git a/azure_batch_maya/scripts/tools/job_watcher.py b/azure_batch_maya/scripts/tools/job_watcher.py index 75d6ddf..63dabd8 100644 --- a/azure_batch_maya/scripts/tools/job_watcher.py +++ b/azure_batch_maya/scripts/tools/job_watcher.py @@ -10,11 +10,12 @@ import time import sys import os import re - +import threading batch_client = None storage_client = None header_line_length = 50 +concurrent_downloads = 20 def header(header): @@ -39,22 +40,14 @@ def _check_valid_dir(directory): def _download_output(container, blob_name, output_path, size): - def progress(data, total): - try: - percent = float(data)*100/float(size) - sys.stdout.write(' Downloading... {0}%\r'.format(int(percent))) - except: - sys.stdout.write(' Downloading... %\r') - finally: - sys.stdout.flush() - print("Downloading task output: {}".format(blob_name)) - storage_client.get_blob_to_path(container, blob_name, output_path, progress_callback=progress) - print(" Output download successful.\n") + storage_client.get_blob_to_path(container, blob_name, output_path)) + print("Output {} download successful".format(blob_name)) def _track_completed_outputs(container, dwnld_dir): job_outputs = storage_client.list_blobs(container) + downloads = [] for output in job_outputs: if output.name.startswith('thumbs/'): continue @@ -63,7 +56,17 @@ def _track_completed_outputs(container, dwnld_dir): if not os.path.isfile(output_file): if not os.path.isdir(os.path.dirname(output_file)): os.makedirs(os.path.dirname(output_file)) - _download_output(container, output.name, output_file, output.properties.content_length) + downloads.append( + threading.Thread( + target=_download_output, + args=(container, output.name, output_file, output.properties.content_length))) + downloads[-1].start() + if len(downloads) >= concurrent_downloads: + for thread in downloads: + thread.join() + downloads = [] + for thread in downloads: + thread.join() def _check_job_stopped(job): From 190f6296f40c5be040d94d639072198e28afc3c1 Mon Sep 17 00:00:00 2001 From: annatisch Date: Fri, 23 Jun 2017 07:18:20 -0700 Subject: [PATCH 21/29] Made task counting asynchronous --- azure_batch_maya/scripts/jobhistory.py | 34 ++++++++++++++++---- azure_batch_maya/scripts/ui/ui_jobhistory.py | 3 +- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/azure_batch_maya/scripts/jobhistory.py b/azure_batch_maya/scripts/jobhistory.py index 848a6d6..ceb2045 100644 --- a/azure_batch_maya/scripts/jobhistory.py +++ b/azure_batch_maya/scripts/jobhistory.py @@ -215,6 +215,32 @@ class AzureBatchJobHistory(object): self.selected_job.set_thumbnail(loading_thumb, 24) maya.refresh() job = self._call(self.batch.job.get, job.id) + self.selected_job.set_status('loading...') + self.selected_job.set_progress('loading...') + self.selected_job.set_tasks('loading...') + self.selected_job.set_submission(job.creation_time.isoformat()) + self.selected_job.set_job(job.id) + self.selected_job.set_pool(job.pool_info.pool_id) + self.selected_job.set_label(job.display_name) + maya.refresh() + self._log.info("Updated {0}".format(job.display_name)) + except Exception as exp: + self._log.warning("Failed to update job details {0}".format(exp)) + self.ui.refresh() + + def load_tasks(self): + """Get a list of tasks associated with the job.""" + try: + job = self.jobs[self.selected_job.index] + except (IndexError, AttributeError): + self._log.warning("Selected job index does not match jobs list.") + if not self.selected_job: + return + self.selected_job.set_status('unknown') + self.selected_job.set_progress('unknown') + self.selected_job.set_tasks('unknown') + return + try: tasks = list(self._call(self.batch.task.list, job.id)) completed_tasks = [t for t in tasks if t.state == batch.models.TaskState.completed] errored_tasks = [t for t in completed_tasks if t.execution_info.exit_code != 0] @@ -225,14 +251,9 @@ class AzureBatchJobHistory(object): else: percentage = (100 * len(completed_tasks)) / (len(tasks)) self.selected_job.set_status(state) - self.selected_job.set_progress(percentage) - self.selected_job.set_submission(job.creation_time.isoformat()) + self.selected_job.set_progress(str(percentage)+'%') self.selected_job.set_tasks(len(tasks)) - self.selected_job.set_job(job.id) - self.selected_job.set_pool(job.pool_info.pool_id) - self.selected_job.set_label(job.display_name) maya.refresh() - self._log.info("Updated {0}".format(job.display_name)) except Exception as exp: self._log.warning("Failed to update job details {0}".format(exp)) self.ui.refresh() @@ -257,7 +278,6 @@ class AzureBatchJobHistory(object): self._log.warning(exp) blobs = [] thumbs = sorted([b.name for b in blobs]) - print(thumbs) self._download_thumbnail(job, thumbs) def cancel_job(self): diff --git a/azure_batch_maya/scripts/ui/ui_jobhistory.py b/azure_batch_maya/scripts/ui/ui_jobhistory.py index ff1ae27..6e242e1 100644 --- a/azure_batch_maya/scripts/ui/ui_jobhistory.py +++ b/azure_batch_maya/scripts/ui/ui_jobhistory.py @@ -285,7 +285,7 @@ class AzureBatchJobInfo(object): """Set the label for progress complete. :param int value: The percent complete. """ - maya.text(self._progress, edit=True, label=" {0}%".format(value)) + maya.text(self._progress, edit=True, label=" {0}".format(value)) def set_submission(self, value): """Set the label for date/time submitted. @@ -389,6 +389,7 @@ class AzureBatchJobInfo(object): (self.cancel_button, 'right', 5, 80), (self.delete_button, 'left', 5, 80)]) self.base.job_selected(self) + maya.execute(self.base.load_tasks) maya.execute(self.base.get_thumbnail) maya.refresh() From fb95c763d590812359098edc01333782adc93d1c Mon Sep 17 00:00:00 2001 From: annatisch Date: Fri, 23 Jun 2017 07:41:40 -0700 Subject: [PATCH 22/29] Bumped version --- CHANGES.txt | 7 +++++++ LICENSE.txt | 2 +- azure_batch_maya/plug-in/AzureBatch.py | 2 +- azure_batch_maya/scripts/tools/job_watcher.py | 2 +- package.py | 2 +- 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index a2435de..193237d 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,11 @@ +2017-06-23 v0.9.3 +----------------- +- Added threading to output downloads +- Added threading to task submission +- Made task counting for job detail display load asynchronously + + 2017-06-12 v0.9.2 ----------------- - Fixed bug in OS detection of Job Watcher diff --git a/LICENSE.txt b/LICENSE.txt index a37a50a..3b6da79 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,5 +1,5 @@ -The Azure Batch Maya Sample ver. 0.9.2 +The Azure Batch Maya Sample ver. 0.9.3 Copyright (c) Microsoft Corporation All rights reserved. diff --git a/azure_batch_maya/plug-in/AzureBatch.py b/azure_batch_maya/plug-in/AzureBatch.py index 6c8c876..cc4db9a 100644 --- a/azure_batch_maya/plug-in/AzureBatch.py +++ b/azure_batch_maya/plug-in/AzureBatch.py @@ -43,7 +43,7 @@ NAMESPACE_PACAKGES = [ "azure-storage==0.32.0", ] -VERSION = "0.9.2" +VERSION = "0.9.3" EULA_PREF = "AzureBatch_EULA" SHELF_FILE = "shelf_AzureBatch.mel" cmd_name = "AzureBatch" diff --git a/azure_batch_maya/scripts/tools/job_watcher.py b/azure_batch_maya/scripts/tools/job_watcher.py index 63dabd8..dcc77eb 100644 --- a/azure_batch_maya/scripts/tools/job_watcher.py +++ b/azure_batch_maya/scripts/tools/job_watcher.py @@ -41,7 +41,7 @@ def _check_valid_dir(directory): def _download_output(container, blob_name, output_path, size): print("Downloading task output: {}".format(blob_name)) - storage_client.get_blob_to_path(container, blob_name, output_path)) + storage_client.get_blob_to_path(container, blob_name, output_path) print("Output {} download successful".format(blob_name)) diff --git a/package.py b/package.py index 41dd003..159e468 100644 --- a/package.py +++ b/package.py @@ -32,7 +32,7 @@ import subprocess import shutil import zipfile -VERSION = "0.9.2" +VERSION = "0.9.3" def main(): """Build Maya Plug-in package""" From 15277ba5fa2c3cc7dfb8ff6933fd926b267e0441 Mon Sep 17 00:00:00 2001 From: annatisch Date: Wed, 28 Jun 2017 07:25:04 -0700 Subject: [PATCH 23/29] Restored auth status label with colour --- azure_batch_maya/scripts/config.py | 2 +- azure_batch_maya/scripts/jobhistory.py | 4 ++-- azure_batch_maya/scripts/ui/ui_config.py | 6 +++++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/azure_batch_maya/scripts/config.py b/azure_batch_maya/scripts/config.py index 6b22a39..a5ff0be 100644 --- a/azure_batch_maya/scripts/config.py +++ b/azure_batch_maya/scripts/config.py @@ -160,7 +160,7 @@ class AzureBatchConfig(object): """ try: filter = batch.models.PoolListOptions(max_results=1, select="id") - self._client.pool.list(filter) + list(self._client.pool.list(filter)) self._storage.list_containers(num_results=1) return True except Exception as exp: diff --git a/azure_batch_maya/scripts/jobhistory.py b/azure_batch_maya/scripts/jobhistory.py index ceb2045..c7de0f4 100644 --- a/azure_batch_maya/scripts/jobhistory.py +++ b/azure_batch_maya/scripts/jobhistory.py @@ -227,7 +227,7 @@ class AzureBatchJobHistory(object): except Exception as exp: self._log.warning("Failed to update job details {0}".format(exp)) self.ui.refresh() - + def load_tasks(self): """Get a list of tasks associated with the job.""" try: @@ -257,7 +257,7 @@ class AzureBatchJobHistory(object): except Exception as exp: self._log.warning("Failed to update job details {0}".format(exp)) self.ui.refresh() - + def get_thumbnail(self): """Check job outputs of the currently selected job to find any available thumbnails. diff --git a/azure_batch_maya/scripts/ui/ui_config.py b/azure_batch_maya/scripts/ui/ui_config.py index 13e5aa1..1bc83bf 100644 --- a/azure_batch_maya/scripts/ui/ui_config.py +++ b/azure_batch_maya/scripts/ui/ui_config.py @@ -141,7 +141,10 @@ class ConfigUI(object): @status.setter def status(self, value): """Plug-in authentication status. Sets contents of label.""" - maya.text(self.auth_status, edit=True, label=value) + if value: + maya.text(self.auth_status, edit=True, label="Authenticated", backgroundColor=[0.23, 0.44, 0.21]) + else: + maya.text(self.auth_status, edit=True, label="Not authenticated", backgroundColor=[0.6, 0.23, 0.23]) @property def logging(self): @@ -181,6 +184,7 @@ class ConfigUI(object): """Set label of authentication button depending on auth status. :param bool auth: Whether plug-in is authenticated. """ + self.status = auth if auth: maya.button( self._authenticate, edit=True, label="Refresh Authentication") From 6282f55be2439e9bdbef5c6562a227a2978b6dbe Mon Sep 17 00:00:00 2001 From: annatisch Date: Tue, 4 Jul 2017 10:56:29 -0700 Subject: [PATCH 24/29] Exposed threading configuration --- azure_batch_maya/scripts/api.py | 8 + azure_batch_maya/scripts/assets.py | 11 +- .../scripts/batch_extensions/_job_utils.py | 8 +- .../operations/job_operations.py | 4 +- azure_batch_maya/scripts/config.py | 49 ++++-- azure_batch_maya/scripts/ui/ui_config.py | 29 +++- azure_batch_maya/scripts/ui/ui_submission.py | 159 +++++++++++++----- 7 files changed, 198 insertions(+), 70 deletions(-) diff --git a/azure_batch_maya/scripts/api.py b/azure_batch_maya/scripts/api.py index 763ba7d..7ea506d 100644 --- a/azure_batch_maya/scripts/api.py +++ b/azure_batch_maya/scripts/api.py @@ -307,6 +307,14 @@ class MayaAPI(object): LOG.debug("MayaAPI exception in 'int_slider': {0}".format(exp).strip()) return None + @staticmethod + def int_field(*args, **kwargs): + try: + return cmds.intField(*args, **kwargs) + except Exception as exp: + LOG.debug("MayaAPI exception in 'int_field': {0}".format(exp).strip()) + return None + @staticmethod def popup_menu(*args, **kwargs): try: diff --git a/azure_batch_maya/scripts/assets.py b/azure_batch_maya/scripts/assets.py index 6f6696b..2262590 100644 --- a/azure_batch_maya/scripts/assets.py +++ b/azure_batch_maya/scripts/assets.py @@ -30,7 +30,6 @@ from default import AzureBatchRenderAssets SYS_SEARCHPATHS = [] USR_SEARCHPATHS = [] BYTES = 1024 -UPLOAD_THREADS = 10 class AzureBatchAssets(object): @@ -49,6 +48,7 @@ class AzureBatchAssets(object): self._session = None self._assets = None self._tab_index = index + self._upload_threads = None self.batch = None self.modules = self._collect_modules() @@ -199,11 +199,13 @@ class AzureBatchAssets(object): return Asset(map_file, [], self.batch, self._log) def _upload_all(self, to_upload, progress, total, project): - """Upload all selected assets in 10 threads.""" + """Upload all selected assets in configured number of threads.""" uploads_running = [] progress_queue = Queue() - for i in range(0, len(to_upload), UPLOAD_THREADS): - for index, asset in enumerate(to_upload[i:i + UPLOAD_THREADS]): + threads = self._upload_threads() + self._log.debug("Uploading assets in {} threads.".format(threads)) + for i in range(0, len(to_upload), threads): + for index, asset in enumerate(to_upload[i:i + threads]): self._log.debug("Starting thread for asset: {}".format(asset.path)) upload = threading.Thread( target=asset.upload, args=(index, progress, progress_queue, project)) @@ -249,6 +251,7 @@ class AzureBatchAssets(object): Called on successful authentication. """ self._session = session + self._upload_threads = session.get_threads self.batch = self._session.batch self._set_searchpaths() self._assets = Assets(self.batch) diff --git a/azure_batch_maya/scripts/batch_extensions/_job_utils.py b/azure_batch_maya/scripts/batch_extensions/_job_utils.py index 2ebe057..3740d4f 100644 --- a/azure_batch_maya/scripts/batch_extensions/_job_utils.py +++ b/azure_batch_maya/scripts/batch_extensions/_job_utils.py @@ -35,10 +35,10 @@ def _bulk_add_tasks(client, job_id, tasks, queue): for task in added_tasks.value: queue.put(task) -def deploy_tasks(client, job_id, tasks): +def deploy_tasks(client, job_id, tasks, threads): MAX_TASKS_COUNT_IN_BATCH = 100 - MAX_SUBMIT_THREADS = 10 - + submit_threads = threads or 10 + def add_task(): start = 0 progress_queue = Queue() @@ -50,7 +50,7 @@ def deploy_tasks(client, job_id, tasks): submit.start() submitting_tasks.append(submit) start = end - if start >= len(tasks) or len(submitting_tasks) >= MAX_SUBMIT_THREADS: + if start >= len(tasks) or len(submitting_tasks) >= submit_threads: while any(s for s in submitting_tasks if s.is_alive()) or not progress_queue.empty(): submitted_tasks.append(progress_queue.get()) progress_queue.task_done() diff --git a/azure_batch_maya/scripts/batch_extensions/operations/job_operations.py b/azure_batch_maya/scripts/batch_extensions/operations/job_operations.py index 246db4d..0abded8 100644 --- a/azure_batch_maya/scripts/batch_extensions/operations/job_operations.py +++ b/azure_batch_maya/scripts/batch_extensions/operations/job_operations.py @@ -89,7 +89,7 @@ class ExtendedJobOperations(JobOperations): except Exception as exp: raise ValueError("Unable to deserialize to ExtendedJobParameter: {}".format(exp)) - def add(self, job, job_add_options=None, custom_headers=None, raw=False, **operation_config): + def add(self, job, job_add_options=None, custom_headers=None, raw=False, threads=None, **operation_config): """Adds a job to the specified account. The Batch service supports two ways to control the work done as part of @@ -176,7 +176,7 @@ class ExtendedJobOperations(JobOperations): # Begin original job add process result = super(ExtendedJobOperations, self).add(job, job_add_options, custom_headers, raw, **operation_config) if task_collection: - job_utils.deploy_tasks(self._parent.task, job.id, task_collection) + job_utils.deploy_tasks(self._parent.task, job.id, task_collection, threads) if auto_complete: # If the option to terminate the job was set, we need to reapply it with a patch # now that the tasks have been added. diff --git a/azure_batch_maya/scripts/config.py b/azure_batch_maya/scripts/config.py index a5ff0be..1a24722 100644 --- a/azure_batch_maya/scripts/config.py +++ b/azure_batch_maya/scripts/config.py @@ -152,6 +152,10 @@ class AzureBatchConfig(object): self.ui.logging = int(self._cfg.get('AzureBatch', 'logging')) except ConfigParser.NoOptionError: self.ui.logging = 10 + try: + self.ui.threads = int(self._cfg.get('AzureBatch', 'threads')) + except ConfigParser.NoOptionError: + self.ui.threads = 20 self.ui.set_authenticate(self._auth) def _auto_authentication(self): @@ -167,16 +171,29 @@ class AzureBatchConfig(object): self._log.info("Failed to authenticate: {0}".format(exp)) return False + def _save_config(self): + """Persist the current plugin configuration to file.""" + config_file = os.path.join(self._data_dir, self._ini_file) + with open(config_file, 'w') as handle: + self._cfg.write(handle) + def set_logging(self, level): """Set the logging level to that specified in the UI. :param str level: The specified logging level. """ - log_level = int(LOG_LEVELS[level]) - self._log.setLevel(log_level) - self._cfg.set('AzureBatch', 'logging', str(level)) + self._log.setLevel(level) + self._cfg.set('AzureBatch', 'logging', level) + self._save_config() + + def set_threads(self, threads): + """Set the number of threads to that specified in the UI. + :param int threads: The specified number of threads. + """ + self._cfg.set('AzureBatch', 'threads', threads) + self._save_config() def save_changes(self): - """Persist configuration changes to file for future sessions.""" + """Persist auth config changes to file for future sessions.""" try: self._cfg.add_section('AzureBatch') except ConfigParser.DuplicateSectionError: @@ -186,10 +203,7 @@ class AzureBatchConfig(object): self._cfg.set('AzureBatch', 'batch_key', self.ui.key) self._cfg.set('AzureBatch', 'storage_account', self.ui.storage) self._cfg.set('AzureBatch', 'storage_key', self.ui.storage_key) - self._cfg.set('AzureBatch', 'logging', self.ui.logging) - config_file = os.path.join(self._data_dir, self._ini_file) - with open(config_file, 'w') as handle: - self._cfg.write(handle) + self._save_config() def authenticate(self): """Begin authentication - initiated by the UI button.""" @@ -202,6 +216,13 @@ class AzureBatchConfig(object): finally: self.ui.set_authenticate(self._auth) self.session() + + def get_threads(self): + """Attempt to retrieve number of threads configured for the plugin.""" + try: + return int(self._cfg.get('AzureBatch', 'threads')) + except ConfigParser.NoOptionError: + return self.ui.threads def get_cached_vm_sku(self): """Attempt to retrieve a selected VM SKU from a previous session.""" @@ -213,9 +234,7 @@ class AzureBatchConfig(object): def store_vm_sku(self, sku): """Cache selected VM SKU for later sessions.""" self._cfg.set('AzureBatch', 'vm_sku', sku) - config_file = os.path.join(self._data_dir, self._ini_file) - with open(config_file, 'w') as handle: - self._cfg.write(handle) + self._save_config() def get_cached_image(self): """Attempt to retrieve a selected image a previous session.""" @@ -227,9 +246,7 @@ class AzureBatchConfig(object): def store_image(self, image): """Cache selected image for later sessions.""" self._cfg.set('AzureBatch', 'image', image) - config_file = os.path.join(self._data_dir, self._ini_file) - with open(config_file, 'w') as handle: - self._cfg.write(handle) + self._save_config() def get_cached_autoscale_formula(self): """Attempt to retrieve an autoscale forumla from a previous session.""" @@ -241,6 +258,4 @@ class AzureBatchConfig(object): def store_autoscale_formula(self, formula): """Cache selected VM SKU for later sessions.""" self._cfg.set('AzureBatch', 'autoscale', formula) - config_file = os.path.join(self._data_dir, self._ini_file) - with open(config_file, 'w') as handle: - self._cfg.write(handle) + self._save_config() diff --git a/azure_batch_maya/scripts/ui/ui_config.py b/azure_batch_maya/scripts/ui/ui_config.py index 1bc83bf..1cd3a50 100644 --- a/azure_batch_maya/scripts/ui/ui_config.py +++ b/azure_batch_maya/scripts/ui/ui_config.py @@ -37,6 +37,17 @@ class ConfigUI(object): maya.text(label="Service: ", align="left") self._endpoint = maya.text_field(height=25, enable=True) + #TODO: Allow set to 0 to disable threads + with utils.Row(2, 2, (70,260), ("left","left")): + maya.text(label="Threads: ", align="left") + self._threads = maya.int_field( + changeCommand=self.set_threads, + height=25, + minValue=1, + maxValue=40, + enable=True, + value=20) + with utils.Row(2, 2, (70,260), ("left","center"), [(1, "bottom", 20),(2,"bottom",15)]): maya.text(label="Logging: ", align="left") @@ -93,6 +104,16 @@ class ConfigUI(object): """AzureBatch Service Endpoint. Sets contents of text field.""" maya.text_field(self._endpoint, edit=True, text=str(value)) + @property + def threads(self): + """Max number of threads used. Retrieves contents of int field.""" + return maya.int_field(self._threads, query=True, value=True) + + @threads.setter + def threads(self, value): + """Max number of threads used. Sets contents of iny field.""" + maya.int_field(self._threads, edit=True, value=int(value)) + @property def account(self): """AzureBatch Unattended Account ID. Retrieves contents of text field.""" @@ -178,7 +199,13 @@ class ConfigUI(object): """Set logging level. Command for logging dropdown selection. :param str level: The selected logging level, e.g. ``debug``. """ - self.base.set_logging(level.lower()) + self.base.set_logging(self.logging) + + def set_threads(self, threads): + """Set number of threads. OnChange command for threads field. + :param int threads: The selected number of threads. + """ + self.base.set_threads(int(threads)) def set_authenticate(self, auth): """Set label of authentication button depending on auth status. diff --git a/azure_batch_maya/scripts/ui/ui_submission.py b/azure_batch_maya/scripts/ui/ui_submission.py index 53b91cb..91034a3 100644 --- a/azure_batch_maya/scripts/ui/ui_submission.py +++ b/azure_batch_maya/scripts/ui/ui_submission.py @@ -29,16 +29,18 @@ class SubmissionUI(object): self.label = "Submit" self.page = maya.form_layout(enableBackground=True) self.select_pool_type = self.AUTO_POOL - self.select_instances = 1 + self.select_dedicated_instances = 1 + self.select_low_pri_instances = 0 with utils.ScrollLayout(height=475, parent=self.page) as scroll: box_label = "Pool Settings" - with utils.FrameLayout(label=box_label, collapsable=True): - self.pool_settings = maya.col_layout( + with utils.FrameLayout(label=box_label, collapsable=True) as pool_settings: + self.pool_settings = pool_settings + maya.col_layout( numberOfColumns=2, columnWidth=((1, 100), (2, 200)), rowSpacing=(1, 10), - rowOffset=((1, "top", 20), (2, "bottom", 20))) + rowOffset=((1, "top", 20),)) maya.text(label="Pools: ", align="right") maya.radio_group( labelArray3=("Auto provision a pool for this job", @@ -50,16 +52,36 @@ class SubmissionUI(object): onCommand1=self.set_pool_auto, onCommand2=self.set_pool_reuse, onCommand3=self.set_pool_new) - self.pool_text = maya.text( - label="Instances: ", align="right") - self.control = maya.int_slider( - field=True, value=self.select_instances, + maya.parent() + self.pool_config = [] + self.pool_config.append(maya.col_layout( + numberOfColumns=4, + columnWidth=((1, 100), (2, 50), (3, 100), (4, 50)), + rowSpacing=(1, 10), + rowOffset=((1, "bottom", 20),), + parent=self.pool_settings)) + self.pool_config.append(maya.text( + label="Dedicated VMs: ", + align="right", + parent=self.pool_config[0])) + self.pool_config.append(maya.int_field( + value=self.select_dedicated_instances, minValue=1, maxValue=self.base.max_pool_size, - fieldMinValue=1, - fieldMaxValue=self.base.max_pool_size, - changeCommand=self.set_pool_instances, - annotation="Number of instances in pool") + changeCommand=self.set_dedicated_instances, + annotation="Number of dedicated VMs in pool", + parent=self.pool_config[0])) + self.pool_config.append(maya.text( + label="Low-pri VMs: ", + align="right", + parent=self.pool_config[0])) + self.pool_config.append(maya.int_field( + value=self.select_low_pri_instances, + minValue=0, + maxValue=self.base.max_pool_size, + changeCommand=self.set_low_pri_instances, + annotation="Number of low-priority VMs in pool", + parent=self.pool_config[0])) maya.parent() box_label = "Render Settings" @@ -190,16 +212,22 @@ class SubmissionUI(object): specification as value. """ if self.select_pool_type == self.EXISTING_POOL: - details = str(maya.menu(self.control, query=True, value=True)) + details = str(maya.menu(self.pool_config[-1], query=True, value=True)) else: - details = self.select_instances + details = (self.select_dedicated_instances, self.select_low_pri_instances) return {self.select_pool_type: details} - def set_pool_instances(self, instances): + def set_dedicated_instances(self, instances): """Update the number of requested instances in a pool based on the instance slider. """ - self.select_instances = instances + self.select_dedicated_instances = instances + + def set_low_pri_instances(self, instances): + """Update the number of requested instances in a pool + based on the instance slider. + """ + self.select_low_pri_instances = instances def set_pool_new(self, *args): """Set selected pool type to be new pool of given size. @@ -207,18 +235,36 @@ class SubmissionUI(object): Command for select_pool_type radio buttons. """ self.select_pool_type = self.NEW_POOL - maya.delete_ui(self.control) - maya.text(self.pool_text, edit=True, label="Instances: ") - self.control = maya.int_slider( - field=True, - value=self.select_instances, + maya.delete_ui(self.pool_config) + self.pool_config = [] + self.pool_config.append(maya.col_layout( + numberOfColumns=4, + columnWidth=((1, 100), (2, 50), (3, 100), (4, 50)), + rowSpacing=(1, 10), + rowOffset=((1, "bottom", 20),), + parent=self.pool_settings)) + self.pool_config.append(maya.text( + label="Dedicated VMs: ", + align="right", + parent=self.pool_config[0])) + self.pool_config.append(maya.int_field( + value=self.select_dedicated_instances, minValue=1, maxValue=self.base.max_pool_size, - fieldMinValue=1, - fieldMaxValue=self.base.max_pool_size, - parent=self.pool_settings, - changeCommand=self.set_pool_instances, - annotation="Number of instances in pool") + changeCommand=self.set_dedicated_instances, + annotation="Number of dedicated VMs in pool", + parent=self.pool_config[0])) + self.pool_config.append(maya.text( + label="Low-pri VMs: ", + align="right", + parent=self.pool_config[0])) + self.pool_config.append(maya.int_field( + value=self.select_low_pri_instances, + minValue=0, + maxValue=self.base.max_pool_size, + changeCommand=self.set_low_pri_instances, + annotation="Number of low-priority VMs in pool", + parent=self.pool_config[0])) def set_pool_auto(self, *args): """Set selected pool type to be new pool of given size. @@ -226,18 +272,36 @@ class SubmissionUI(object): Command for select_pool_type radio buttons. """ self.select_pool_type = self.AUTO_POOL - maya.delete_ui(self.control) - maya.text(self.pool_text, edit=True, label="Instances: ") - self.control = maya.int_slider( - field=True, - value=self.select_instances, + maya.delete_ui(self.pool_config) + self.pool_config = [] + self.pool_config.append(maya.col_layout( + numberOfColumns=4, + columnWidth=((1, 100), (2, 50), (3, 100), (4, 50)), + rowSpacing=(1, 10), + rowOffset=((1, "bottom", 20),), + parent=self.pool_settings)) + self.pool_config.append(maya.text( + label="Dedicated VMs: ", + align="right", + parent=self.pool_config[0])) + self.pool_config.append(maya.int_field( + value=self.select_dedicated_instances, minValue=1, maxValue=self.base.max_pool_size, - fieldMinValue=1, - fieldMaxValue=self.base.max_pool_size, - parent=self.pool_settings, - changeCommand=self.set_pool_instances, - annotation="Number of instances in pool") + changeCommand=self.set_dedicated_instances, + annotation="Number of dedicated VMs in pool", + parent=self.pool_config[0])) + self.pool_config.append(maya.text( + label="Low-pri VMs: ", + align="right", + parent=self.pool_config[0])) + self.pool_config.append(maya.int_field( + value=self.select_low_pri_instances, + minValue=0, + maxValue=self.base.max_pool_size, + changeCommand=self.set_low_pri_instances, + annotation="Number of low-priority VMs in pool", + parent=self.pool_config[0])) def set_pool_reuse(self, *args): """Set selected pool type to be an existing pool with given ID. @@ -246,13 +310,24 @@ class SubmissionUI(object): Command for select_pool_type radio buttons. """ self.select_pool_type = self.EXISTING_POOL - maya.delete_ui(self.control) - maya.text(self.pool_text, edit=True, label="loading...") + maya.delete_ui(self.pool_config) + self.pool_config = [] + self.pool_config.append(maya.col_layout( + numberOfColumns=2, + columnWidth=((1, 100), (2, 200)), + rowSpacing=(1, 10), + rowOffset=((1, "bottom", 20),), + parent=self.pool_settings)) + self.pool_config.append(maya.text( + label="loading...", + align="right", + parent=self.pool_config[0])) maya.refresh() pool_options = self.base.available_pools() - maya.text(self.pool_text, edit=True, label="Pool ID: ") - self.control = maya.menu( - parent=self.pool_settings, - annotation="Use an existing persistent pool ID") + maya.text(self.pool_config[-1], edit=True, label="Pool ID: ") + self.pool_config.append(maya.menu( + annotation="Use an existing persistent pool ID", + parent=self.pool_config[0])) for pool_id in pool_options: maya.menu_option(pool_id) + \ No newline at end of file From 29b65f948f62a42207c94fa3da4920fcef3bcd6c Mon Sep 17 00:00:00 2001 From: annatisch Date: Tue, 4 Jul 2017 10:56:41 -0700 Subject: [PATCH 25/29] Fixed job refresh bug --- azure_batch_maya/scripts/ui/ui_jobhistory.py | 1 + 1 file changed, 1 insertion(+) diff --git a/azure_batch_maya/scripts/ui/ui_jobhistory.py b/azure_batch_maya/scripts/ui/ui_jobhistory.py index 6e242e1..b01ddd8 100644 --- a/azure_batch_maya/scripts/ui/ui_jobhistory.py +++ b/azure_batch_maya/scripts/ui/ui_jobhistory.py @@ -418,6 +418,7 @@ class AzureBatchJobInfo(object): def refresh(self): """Refresh the details of the specified job, and update the UI.""" self.base.update_job(self.index) + maya.execute(self.base.load_tasks) maya.execute(self.base.get_thumbnail) self.selected_dir = utils.get_default_output_path() maya.text_field(self._dir, edit=True, text=self.selected_dir) From 457818e23ac59254444de685dd701646e40bc006 Mon Sep 17 00:00:00 2001 From: annatisch Date: Tue, 4 Jul 2017 10:57:30 -0700 Subject: [PATCH 26/29] Exposed low priority pool VMs --- azure_batch_maya/scripts/pools.py | 24 +++++--- azure_batch_maya/scripts/shared.py | 1 + azure_batch_maya/scripts/submission.py | 10 +++- azure_batch_maya/scripts/tools/job_watcher.py | 9 ++- azure_batch_maya/scripts/ui/ui_pools.py | 60 ++++++++++++++----- 5 files changed, 74 insertions(+), 30 deletions(-) diff --git a/azure_batch_maya/scripts/pools.py b/azure_batch_maya/scripts/pools.py index 1dc24fb..b6c8274 100644 --- a/azure_batch_maya/scripts/pools.py +++ b/azure_batch_maya/scripts/pools.py @@ -100,7 +100,8 @@ class AzureBatchPools(object): nodes = [n for n in _nodes] self.selected_pool.set_id(pool.id) self.selected_pool.set_label(pool.display_name if pool.display_name else pool.id) - self.selected_pool.set_size(pool) + self.selected_pool.set_dedicated_size(pool) + self.selected_pool.set_low_pri_size(pool) self.selected_pool.set_type( "Auto" if pool.id.startswith("Maya_Auto_Pool") else "Provisioned") self.selected_pool.set_state(pool.state.value, nodes) @@ -141,10 +142,10 @@ class AzureBatchPools(object): """Get the target number of VMs in the selected pool.""" try: pool = self.pools[self.selected_pool.index] - return int(pool.target_dedicated_nodes) + return int(pool.target_dedicated_nodes), int(pool.target_low_priority_nodes) except (IndexError, TypeError, AttributeError) as exp: self._log.info("Failed to parse pool target size {0}".format(exp)) - return 0 + return 0, 0 def get_pool_os(self, pool_id): """Get the OS flavor of the specified pool ID.""" @@ -173,7 +174,8 @@ class AzureBatchPools(object): application_licenses=self.environment.get_application_licenses(), vm_size=self.environment.get_vm_sku(), virtual_machine_configuration=pool_config, - target_dedicated_nodes=int(size), + target_dedicated_nodes=int(size[0]), + target_low_priority_nodes=int(size[1]), max_tasks_per_node=1) self._call(self.batch.pool.add, new_pool) self._log.debug("Successfully created pool.") @@ -194,7 +196,8 @@ class AzureBatchPools(object): 'virtualMachineConfiguration': pool_config, 'maxTasksPerNode': 1, 'applicationLicenses': self.environment.get_application_licenses(), - 'targetDedicatedNodes': int(size)} + 'targetDedicatedNodes': int(size[0]), + 'targetLowPriorityNodes': int(size[1])} auto_pool = { 'autoPoolIdPrefix': "Maya_Auto_Pool_", 'poolLifetimeOption': "job", @@ -202,13 +205,16 @@ class AzureBatchPools(object): 'pool': pool_spec} return {'autoPoolSpecification': auto_pool} - def resize_pool(self, new_size): + def resize_pool(self, new_dedicated, new_low_pri): """Resize an existing pool.""" try: pool = self.pools[self.selected_pool.index] - self._log.info("Resizing pool '{}' to {} VMs".format(pool.id, new_size)) - self._call(self.batch.pool.resize, pool.id, {'target_dedicated_nodes':int(new_size), - 'target_low_priority_nodes': 0}) + self._log.info( + "Resizing pool '{}' to {} dedicated VMs" + " and {} low priority VMs".format(pool.id, new_dedicated, new_low_pri)) + self._call(self.batch.pool.resize, pool.id, + {'target_dedicated_nodes':int(new_dedicated), + 'target_low_priority_nodes': int(new_low_pri)}) maya.refresh() except Exception as exp: self._log.info("Failed to resize pool {0}".format(exp)) diff --git a/azure_batch_maya/scripts/shared.py b/azure_batch_maya/scripts/shared.py index c424d70..ee01333 100644 --- a/azure_batch_maya/scripts/shared.py +++ b/azure_batch_maya/scripts/shared.py @@ -61,6 +61,7 @@ class AzureBatchSettings(object): maya.delete_ui("AzureBatch") message = "Batch Plugin Failed to Start: {0}".format(exp) maya.error(message) + raise def start(self): """Start the plugin UI. Depending on whether auto-authentication was diff --git a/azure_batch_maya/scripts/submission.py b/azure_batch_maya/scripts/submission.py index 6b46548..6ab2ed4 100644 --- a/azure_batch_maya/scripts/submission.py +++ b/azure_batch_maya/scripts/submission.py @@ -37,6 +37,7 @@ class AzureBatchSubmission(object): self._log = logging.getLogger('AzureBatchMaya') self._call = call self._tab_index = index + self._submit_threads = None self.max_pool_size = 1000 self.ui = SubmissionUI(self, frame) @@ -174,7 +175,7 @@ class AzureBatchSubmission(object): pool_spec = self.ui.get_pool() if pool_spec.get(1): self._log.info("Using auto-pool.") - return self.pool_manager.create_auto_pool(int(pool_spec[1]), job_name) + return self.pool_manager.create_auto_pool(pool_spec[1], job_name) if pool_spec.get(2): self._log.info("Using existing pool.") pool_id = str(pool_spec[2]) @@ -183,7 +184,7 @@ class AzureBatchSubmission(object): return {'poolId' : pool_id} if pool_spec.get(3): self._log.info("Creating new pool.") - return self.pool_manager.create_pool(int(pool_spec[3]), job_name) + return self.pool_manager.create_pool(pool_spec[3], job_name) def start(self, session, assets, pools, env): """Load submission tab after plug-in has been authenticated. @@ -203,6 +204,7 @@ class AzureBatchSubmission(object): self.pool_manager = pools self.env_manager = env self.data_path = session.path + self._submit_threads = session.get_threads if self.renderer: self.renderer.delete() self._configure_renderer() @@ -285,7 +287,9 @@ class AzureBatchSubmission(object): progress.is_cancelled() self.ui.submit_status("Submitting...") progress.status("Submitting...") - self._call(self.batch.job.add, new_job) + threads = self._submit_threads() + self._log.debug("Submitting using {} threads.".format(threads)) + self._call(self.batch.job.add, new_job, threads=threads) maya.info("Job submitted successfully") if watch_job: diff --git a/azure_batch_maya/scripts/tools/job_watcher.py b/azure_batch_maya/scripts/tools/job_watcher.py index dcc77eb..267818a 100644 --- a/azure_batch_maya/scripts/tools/job_watcher.py +++ b/azure_batch_maya/scripts/tools/job_watcher.py @@ -14,8 +14,8 @@ import threading batch_client = None storage_client = None +concurrent_downloads = None header_line_length = 50 -concurrent_downloads = 20 def header(header): @@ -48,6 +48,7 @@ def _download_output(container, blob_name, output_path, size): def _track_completed_outputs(container, dwnld_dir): job_outputs = storage_client.list_blobs(container) downloads = [] + print("concurrent downloads", concurrent_downloads) for output in job_outputs: if output.name.startswith('thumbs/'): continue @@ -131,7 +132,7 @@ def track_job_progress(id, container, dwnld_dir): def _authenticate(cfg_path): - global batch_client, storage_client + global batch_client, storage_client, concurrent_downloads cfg = ConfigParser.ConfigParser() try: cfg.read(cfg_path) @@ -144,6 +145,10 @@ def _authenticate(cfg_path): cfg.get("AzureBatch", "storage_account"), cfg.get("AzureBatch", "storage_key"), endpoint_suffix="core.windows.net") + try: + concurrent_downloads = cfg.get("AzureBatch", "threads") + except ConfigParser.NoSectionError: + concurrent_downloads = 20 except (EnvironmentError, ConfigParser.NoOptionError, ConfigParser.NoSectionError) as exp: raise ValueError("Failed to authenticate using Maya configuration {0}".format(cfg_path)) diff --git a/azure_batch_maya/scripts/ui/ui_pools.py b/azure_batch_maya/scripts/ui/ui_pools.py index e4f361d..86fe04d 100644 --- a/azure_batch_maya/scripts/ui/ui_pools.py +++ b/azure_batch_maya/scripts/ui/ui_pools.py @@ -147,13 +147,20 @@ class AzureBatchPoolInfo(object): """ maya.text(self._type, edit=True, label=" {0}".format(value)) - def set_size(self, pool): + def set_dedicated_size(self, pool): """Set the number of instances in the pool, both current and target. :param int value: Size of the pool. """ - maya.text(self._size, edit=True, label=" target: {} current: {}".format( + maya.text(self._dedicated_size, edit=True, label=" target: {} current: {}".format( pool.target_dedicated_nodes, pool.current_dedicated_nodes)) + def set_low_pri_size(self, pool): + """Set the number of instances in the pool, both current and target. + :param int value: Size of the pool. + """ + maya.text(self._low_pri_size, edit=True, label=" target: {} current: {}".format( + pool.target_low_priority_nodes, pool.current_low_priority_nodes)) + def set_created(self, value): """Set the date/time the pool was created. :param str value: The datetime string of the pool creation. @@ -214,7 +221,8 @@ class AzureBatchPoolInfo(object): """ self._id = self.display_data("ID: ") self._type = self.display_info("Type: ") - self._size = self.display_info("Size: ") + self._dedicated_size = self.display_info("Dedicated VMs: ") + self._low_pri_size = self.display_info("Low Priority VMs: ") self._created = self.display_info("Created: ") self._state = self.display_info("State: ") self._image = self.display_info("Image: ") @@ -224,21 +232,40 @@ class AzureBatchPoolInfo(object): self.base.pool_selected(self) auto = self.base.is_auto_pool() if not auto: + self.content.append(maya.col_layout( + numberOfColumns=5, + columnWidth=((1, 80), (2, 100), (3, 45), (4, 80), (5, 45)), + rowSpacing=(1, 10), + parent=self.layout)) self.resize_button = utils.ProcButton( - "Resize Pool", "Resizing...", self.resize_pool, - parent=self.listbox, align="center") - self.resize_int = maya.int_slider( - value=self.base.get_pool_size(), + "Resize Pool", + "Resizing...", + self.resize_pool, + parent=self.content[-1], + align="center") + self.dedicated_label = maya.text( + label="Dedicated VMs", + parent=self.content[-1]) + self.resize_dedicated = maya.int_field( + value=self.base.get_pool_size()[0], minValue=0, maxValue=1000, - fieldMinValue=0, - fieldMaxValue=100, - field=True, - width=230, - parent=self.listbox, - annotation="Number of instances to work in pool.") + parent=self.content[-1], + annotation="Number of dedicated VMs in pool.") + self.low_pri_label = maya.text( + label="Low-pri VMs", + parent=self.content[-1]) + self.resize_low_pri = maya.int_field( + value=self.base.get_pool_size()[1], + minValue=0, + maxValue=1000, + parent=self.content[-1], + annotation="Number of Low-priority VMs in pool.") self.content.append(self.resize_button.display) - self.content.append(self.resize_int) + self.content.append(self.dedicated_label) + self.content.append(self.resize_dedicated) + self.content.append(self.low_pri_label) + self.content.append(self.resize_low_pri) self.delete_button = utils.ProcButton("Delete Pool", "Deleting...", self.delete_pool, parent=self.layout, align="center") self.content.append(self.delete_button.display) @@ -295,7 +322,8 @@ class AzureBatchPoolInfo(object): def resize_pool(self, *args): """Resize the specified pool.""" self.resize_button.start() - resize = maya.int_slider(self.resize_int, query=True, value=True) - self.base.resize_pool(resize) + resize_dedicated = maya.int_field(self.resize_dedicated, query=True, value=True) + resize_low_pri = maya.int_field(self.resize_low_pri, query=True, value=True) + self.base.resize_pool(resize_dedicated, resize_low_pri) self.base.update_pool(self.index) self.resize_button.finish() From 222be5fbf00874b4384bb612e953acc0dff36dc1 Mon Sep 17 00:00:00 2001 From: annatisch Date: Wed, 5 Jul 2017 07:59:16 -0700 Subject: [PATCH 27/29] Some fixes --- azure_batch_maya/scripts/config.py | 9 +++------ azure_batch_maya/scripts/jobhistory.py | 2 ++ azure_batch_maya/scripts/tools/job_watcher.py | 1 - 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/azure_batch_maya/scripts/config.py b/azure_batch_maya/scripts/config.py index 1a24722..86abfdb 100644 --- a/azure_batch_maya/scripts/config.py +++ b/azure_batch_maya/scripts/config.py @@ -149,11 +149,11 @@ class AzureBatchConfig(object): except ConfigParser.NoOptionError: self.ui.storage_key = "" try: - self.ui.logging = int(self._cfg.get('AzureBatch', 'logging')) + self.ui.logging = self._cfg.getint('AzureBatch', 'logging') except ConfigParser.NoOptionError: self.ui.logging = 10 try: - self.ui.threads = int(self._cfg.get('AzureBatch', 'threads')) + self.ui.threads = self._cfg.getint('AzureBatch', 'threads') except ConfigParser.NoOptionError: self.ui.threads = 20 self.ui.set_authenticate(self._auth) @@ -219,10 +219,7 @@ class AzureBatchConfig(object): def get_threads(self): """Attempt to retrieve number of threads configured for the plugin.""" - try: - return int(self._cfg.get('AzureBatch', 'threads')) - except ConfigParser.NoOptionError: - return self.ui.threads + return self.ui.threads def get_cached_vm_sku(self): """Attempt to retrieve a selected VM SKU from a previous session.""" diff --git a/azure_batch_maya/scripts/jobhistory.py b/azure_batch_maya/scripts/jobhistory.py index c7de0f4..f39567b 100644 --- a/azure_batch_maya/scripts/jobhistory.py +++ b/azure_batch_maya/scripts/jobhistory.py @@ -286,6 +286,7 @@ class AzureBatchJobHistory(object): job = self.jobs[self.selected_job.index] self._call(self.batch.job.terminate, job.id) self.update_job(self.selected_job.index) + maya.execute(self.base.load_tasks) maya.execute(self.get_thumbnail) maya.refresh() except (IndexError, AttributeError) as exp: @@ -301,6 +302,7 @@ class AzureBatchJobHistory(object): job = self.jobs[self.selected_job.index] self._call(self.batch.job.delete, job.id) self.update_job(self.selected_job.index) + maya.execute(self.base.load_tasks) maya.execute(self.get_thumbnail) maya.refresh() except (IndexError, AttributeError) as exp: diff --git a/azure_batch_maya/scripts/tools/job_watcher.py b/azure_batch_maya/scripts/tools/job_watcher.py index 267818a..ee2c200 100644 --- a/azure_batch_maya/scripts/tools/job_watcher.py +++ b/azure_batch_maya/scripts/tools/job_watcher.py @@ -48,7 +48,6 @@ def _download_output(container, blob_name, output_path, size): def _track_completed_outputs(container, dwnld_dir): job_outputs = storage_client.list_blobs(container) downloads = [] - print("concurrent downloads", concurrent_downloads) for output in job_outputs: if output.name.startswith('thumbs/'): continue From 2682174b257fe5c7c47f0f91de884652afb8bb0b Mon Sep 17 00:00:00 2001 From: annatisch Date: Wed, 5 Jul 2017 08:13:26 -0700 Subject: [PATCH 28/29] Fix job status loading --- azure_batch_maya/scripts/jobhistory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_batch_maya/scripts/jobhistory.py b/azure_batch_maya/scripts/jobhistory.py index f39567b..6f05cf8 100644 --- a/azure_batch_maya/scripts/jobhistory.py +++ b/azure_batch_maya/scripts/jobhistory.py @@ -286,7 +286,7 @@ class AzureBatchJobHistory(object): job = self.jobs[self.selected_job.index] self._call(self.batch.job.terminate, job.id) self.update_job(self.selected_job.index) - maya.execute(self.base.load_tasks) + maya.execute(self.load_tasks) maya.execute(self.get_thumbnail) maya.refresh() except (IndexError, AttributeError) as exp: @@ -302,7 +302,7 @@ class AzureBatchJobHistory(object): job = self.jobs[self.selected_job.index] self._call(self.batch.job.delete, job.id) self.update_job(self.selected_job.index) - maya.execute(self.base.load_tasks) + maya.execute(self.load_tasks) maya.execute(self.get_thumbnail) maya.refresh() except (IndexError, AttributeError) as exp: From e9ccc2eb2782220ba2744adbb8d1fd39d6644aa3 Mon Sep 17 00:00:00 2001 From: annatisch Date: Wed, 5 Jul 2017 08:23:10 -0700 Subject: [PATCH 29/29] Bumped version --- CHANGES.txt | 7 +++++++ LICENSE.txt | 2 +- azure_batch_maya/plug-in/AzureBatch.py | 2 +- package.py | 2 +- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 193237d..917af4b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,10 @@ +2017-07-05 v0.10.0 +------------------ +- Exposed low priority VM allocation for pools +- Exposed thread configuration for uploads/downloads/task submission +- Added coloured label to confirm successful authentication +- Fixed some bugs in job status loading + 2017-06-23 v0.9.3 ----------------- diff --git a/LICENSE.txt b/LICENSE.txt index 3b6da79..71e3894 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,5 +1,5 @@ -The Azure Batch Maya Sample ver. 0.9.3 +The Azure Batch Maya Sample ver. 0.10.0 Copyright (c) Microsoft Corporation All rights reserved. diff --git a/azure_batch_maya/plug-in/AzureBatch.py b/azure_batch_maya/plug-in/AzureBatch.py index cc4db9a..2ad7f45 100644 --- a/azure_batch_maya/plug-in/AzureBatch.py +++ b/azure_batch_maya/plug-in/AzureBatch.py @@ -43,7 +43,7 @@ NAMESPACE_PACAKGES = [ "azure-storage==0.32.0", ] -VERSION = "0.9.3" +VERSION = "0.10.0" EULA_PREF = "AzureBatch_EULA" SHELF_FILE = "shelf_AzureBatch.mel" cmd_name = "AzureBatch" diff --git a/package.py b/package.py index 159e468..85fec48 100644 --- a/package.py +++ b/package.py @@ -32,7 +32,7 @@ import subprocess import shutil import zipfile -VERSION = "0.9.3" +VERSION = "0.10.0" def main(): """Build Maya Plug-in package"""