diff --git a/AzureBatchMaya.pyproj b/AzureBatchMaya.pyproj index dfab7f3..2ec5b3e 100644 --- a/AzureBatchMaya.pyproj +++ b/AzureBatchMaya.pyproj @@ -10,10 +10,8 @@ . Maya.Client Maya.Client - - - - + {a76b797f-9426-4cd7-a078-9af9c9e4b437} + 2.7 azure_batch_maya\scripts\;azure_batch_maya\scripts\ui\;tests\data\;tests\data\modules\ @@ -89,7 +87,7 @@ Code - + Code @@ -115,7 +113,7 @@ Code - + Code @@ -149,9 +147,6 @@ Code - - Code - @@ -182,7 +177,7 @@ - + Code @@ -194,6 +189,8 @@ + + Code @@ -206,6 +203,19 @@ Code + + + {a76b797f-9426-4cd7-a078-9af9c9e4b437} + {9a7a9026-48c1-4688-9d5d-e5699d47d074} + 2.7 + env (Python 64-bit 2.7) + Scripts\python.exe + Scripts\pythonw.exe + Lib\ + PYTHONPATH + Amd64 + + 10.0 $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\Python Tools\Microsoft.PythonTools.targets diff --git a/CHANGES.txt b/CHANGES.txt index 346acc1..917af4b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1 +1,30 @@ -v0.9.0 - Core re-written to use latest Azure Batch Python SDK and Batch Extensions template files. \ No newline at end of file +2017-07-05 v0.10.0 +------------------ +- Exposed low priority VM allocation for pools +- Exposed thread configuration for uploads/downloads/task submission +- Added coloured label to confirm successful authentication +- Fixed some bugs in job status loading + + +2017-06-23 v0.9.3 +----------------- +- Added threading to output downloads +- Added threading to task submission +- Made task counting for job detail display load asynchronously + + +2017-06-12 v0.9.2 +----------------- +- Fixed bug in OS detection of Job Watcher +- Updated VM Image references + + +2017-05-24 v0.9.1 +----------------- +- Expanded Pool display UI to show pools created via the Azure portal +- Fixed bug where Pool couldn't be displayed if created with a non-current image + + +2017-05-23 v0.9.0 +----------------- +- Core re-written to use latest Azure Batch Python SDK and Batch Extensions template files. \ No newline at end of file diff --git a/LICENSE.txt b/LICENSE.txt index ff0015b..71e3894 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,5 +1,5 @@ -The Azure Batch Maya Sample ver. 0.9.0 +The Azure Batch Maya Sample ver. 0.10.0 Copyright (c) Microsoft Corporation All rights reserved. diff --git a/README.md b/README.md index 4b90d56..4073e0a 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ ensure the updated dependencies are loaded correctly. Before using the plug-in, it will need to be authenticated using your Azure Batch and Azure Storage account keys. In order to retrieve this information: -1. Open the Azure management portal (ms.portal.azure.com). +1. Open the Azure management portal (portal.azure.com). 2. Select Azure Batch Accounts in the left-hand menu. This can be found under `More Services` in the `Compute` category. 3. Select your account in the list. Copy and paste the account URL into `Service` field of the plug-in UI. Paste the account name into the `Batch Account` field. 4. In the portal, select `Keys` on the left-hand menu. Copy and paste one of the access keys into the `Batch Key` field in the plug-in. diff --git a/azure_batch_maya/mel/create_shelf.mel b/azure_batch_maya/mel/create_shelf.mel index 854b330..a5eee01 100644 --- a/azure_batch_maya/mel/create_shelf.mel +++ b/azure_batch_maya/mel/create_shelf.mel @@ -9,7 +9,7 @@ global proc run_guiStarter() global proc openMissionControl() { - python("import webbrowser\nwebbrowser.open(\"https://ms.portal.azure.com\", 2, True)"); + python("import webbrowser\nwebbrowser.open(\"https://portal.azure.com\", 2, True)"); } diff --git a/azure_batch_maya/modules/arnold_renderer.py b/azure_batch_maya/modules/arnold_renderer.py index 7ad4f8d..36acb35 100644 --- a/azure_batch_maya/modules/arnold_renderer.py +++ b/azure_batch_maya/modules/arnold_renderer.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os import sys @@ -41,26 +18,39 @@ from default import AzureBatchRenderJob, AzureBatchRenderAssets class ArnoldRenderJob(AzureBatchRenderJob): + render_engine = 'arnold' + def __init__(self): - self._renderer = "arnold" - self.label = "Arnold" + self._renderer = 'arnold' + self.label = 'Arnold' + self.log_levels = [ + "0 - Errors", + "1 - Warnings + Info", + "2 - Debug" + ] def settings(self): - if self.scene_name == "": + if self.scene_name == '': job_name = "Untitled" else: job_name = str(os.path.splitext(os.path.basename(self.scene_name))[0]) - file_prefix = mel.eval("getAttr defaultRenderGlobals.imageFilePrefix") + file_prefix = cmds.getAttr("defaultRenderGlobals.imageFilePrefix") if file_prefix: file_prefix = os.path.split(file_prefix)[1] else: file_prefix = "" - self.job_name = self.display_string("Job Name: ", job_name) - self.output_name = self.display_string("Output Prefix: ", file_prefix) + self.job_name = self.display_string("Job name: ", job_name) + self.output_name = self.display_string("Output prefix: ", file_prefix) self.start = self.display_int("Start frame: ", self.start_frame, edit=True) self.end = self.display_int("End frame: ", self.end_frame, edit=True) self.step = self.display_int("Frame step: ", self.frame_step, edit=True) + try: + log_level = cmds.getAttr("defaultArnoldRenderOptions.log_verbosity") + except ValueError: + log_level = 1 + self.logging = self.display_menu("Logging: ", self.log_levels, log_level+1) + def get_title(self): return str(cmds.textField(self.job_name, query=True, text=True)) @@ -68,45 +58,48 @@ class ArnoldRenderJob(AzureBatchRenderJob): return True def get_jobdata(self): - if self.scene_name == "": + if self.scene_name == '': raise ValueError("Current Maya scene has not been saved to disk.") pending_changes = cmds.file(query=True, modified=True) if not pending_changes: return self.scene_name, [self.scene_name] - options = ["Save and Continue", - "Don't Save and Continue", - "Cancel"] - answer = cmds.confirmDialog(title='Unsaved Changes', - message='There are unsaved changes. Proceed?', - button=options, - defaultButton=options[0], - cancelButton=options[2], - dismissString=options[2]) - if answer == options[2]: - raise Exception("Submission Aborted") - if answer == options[0]: + options = { + 'save': "Save and continue", + 'nosave': "Continue without saving", + 'cancel': "Cancel" + } + answer = cmds.confirmDialog(title="Unsaved Changes", + message="There are unsaved changes. Continue?", + button=options.values(), + defaultButton=options['save'], + cancelButton=options['cancel'], + dismissString=options['cancel']) + if answer == options['cancel']: + raise Exception("Submission cancelled") + if answer == options['save']: cmds.SaveScene() return self.scene_name, [self.scene_name] def get_params(self): params = {} - params["frameStart"] = cmds.intField(self.start, query=True, value=True) - params["frameEnd"] = cmds.intField(self.end, query=True, value=True) - params["frameStep"] = cmds.intField(self.step, query=True, value=True) - params["renderer"] = "arnold" + params['frameStart'] = cmds.intField(self.start, query=True, value=True) + params['frameEnd'] = cmds.intField(self.end, query=True, value=True) + params['frameStep'] = cmds.intField(self.step, query=True, value=True) + params['renderer'] = self._renderer + params['logLevel'] = int(cmds.optionMenu(self.logging, query=True, select=True)) - 1 return params class ArnoldRenderAssets(AzureBatchRenderAssets): assets = [] - render_engine = "arnold" + render_engine = 'arnold' file_nodes = { - "aiStandIn": ["dso"], - "aiPhotometricLight": ["aiFilename"], - "aiVolume": ["dso", "filename"], - "aiImage": ["filename"] + 'aiStandIn': ['dso'], + 'aiPhotometricLight': ['aiFilename'], + 'aiVolume': ['dso', 'filename'], + 'aiImage': ['filename'] } def check_path(self, path): @@ -126,7 +119,7 @@ class ArnoldRenderAssets(AzureBatchRenderAssets): nodes = cmds.ls(type=node_type) for node in nodes: for attr in attributes: - collected.append(cmds.getAttr(node + "." + attr)) + collected.append(cmds.getAttr(node + '.' + attr)) for path in collected: self.assets.append(self.check_path(path)) return self.assets diff --git a/azure_batch_maya/modules/default.py b/azure_batch_maya/modules/default.py index d7a1128..9eed7fb 100644 --- a/azure_batch_maya/modules/default.py +++ b/azure_batch_maya/modules/default.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os @@ -94,12 +71,13 @@ class AzureBatchRenderJob(object): else: return cmds.text(label=value, align='left') - def display_menu(self, label, options): + def display_menu(self, label, options, selected): cmds.text(label=label, align='right') menu = cmds.optionMenu() for opt in options: cmds.menuItem(label=opt) cmds.setParent('..') + cmds.optionMenu(menu, edit=True, select=selected) return menu def display_button(self, label, cmd): @@ -153,7 +131,6 @@ class AzureBatchRenderJob(object): horizontalScrollBarThickness=0, verticalScrollBarThickness=3, parent=layout, - #width=360, height=260) self.subLayout = cmds.rowColumnLayout( diff --git a/azure_batch_maya/modules/maya_software.py b/azure_batch_maya/modules/maya_software.py index e234697..8a32da1 100644 --- a/azure_batch_maya/modules/maya_software.py +++ b/azure_batch_maya/modules/maya_software.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os import sys @@ -38,8 +15,10 @@ from default import AzureBatchRenderJob, AzureBatchRenderAssets class AzureBatchMayaJob(AzureBatchRenderJob): + render_engine = "mayaSoftware" + def __init__(self): - self._renderer = "mayaSoftware" + self._renderer = "sw" self.label = "Maya Software" def settings(self): @@ -66,36 +45,35 @@ class AzureBatchMayaJob(AzureBatchRenderJob): return True def get_jobdata(self): - if self.scene_name == "": + if self.scene_name == '': raise ValueError("Current Maya scene has not been saved to disk.") + pending_changes = cmds.file(query=True, modified=True) if not pending_changes: - return [self.scene_name] - options = ["Save and Continue", - "Don't Save and Continue", - "Cancel"] - answer = cmds.confirmDialog(title='Unsaved Changes', - message='There are unsaved changes. Proceed?', - button=options, - defaultButton=options[0], - cancelButton=options[2], - dismissString=options[2]) - if answer == options[2]: - raise Exception("Submission Aborted") - if answer == options[0]: + return self.scene_name, [self.scene_name] + options = { + 'save': "Save and continue", + 'nosave': "Continue without saving", + 'cancel': "Cancel" + } + answer = cmds.confirmDialog(title="Unsaved Changes", + message="There are unsaved changes. Continue?", + button=options.values(), + defaultButton=options['save'], + cancelButton=options['cancel'], + dismissString=options['cancel']) + if answer == options['cancel']: + raise Exception("Submission cancelled") + if answer == options['save']: cmds.SaveScene() - return [self.scene_name] + return self.scene_name, [self.scene_name] def get_params(self): params = {} - params["StartFrame"] = cmds.intField(self.start, query=True, value=True) - params["EndFrame"] = cmds.intField(self.end, query=True, value=True) - params["Renderer"] = "sw" - params["JobFile"] = os.path.basename(self.scene_name) - filename = str(cmds.textField(self.output_name, query=True, text=True)) - if '/' in filename or '\\' in filename: - raise ValueError("Subfolders not supported in output filename.") - params["OutputName"] = filename + params["frameStart"] = cmds.intField(self.start, query=True, value=True) + params["frameEnd"] = cmds.intField(self.end, query=True, value=True) + params["frameStep"] = cmds.intField(self.step, query=True, value=True) + params["renderer"] = self._renderer return params diff --git a/azure_batch_maya/plug-in/AzureBatch.py b/azure_batch_maya/plug-in/AzureBatch.py index 5889d04..2ad7f45 100644 --- a/azure_batch_maya/plug-in/AzureBatch.py +++ b/azure_batch_maya/plug-in/AzureBatch.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import urllib import os @@ -53,44 +30,43 @@ warnings.simplefilter('ignore') INSTALL_DIR = os.path.normpath( os.path.join(cmds.internalVar(userScriptDir=True), 'azure-batch-libs')) sys.path.append(INSTALL_DIR) - + REQUIREMENTS = [ "pathlib==1.0.1", ] NAMESPACE_PACAKGES = [ - #"azure-mgmt-nspkg==2.0.0", "azure-mgmt-batch==4.0.0", - "azure-mgmt-storage==1.0.0rc1", + "azure-mgmt-storage==1.0.0", "azure-common==1.1.5", "azure-batch==3.0.0", "azure-storage==0.32.0", ] -VERSION = "0.9.0" -SLA_PREF = "AzureBatch_SLA" +VERSION = "0.10.0" +EULA_PREF = "AzureBatch_EULA" SHELF_FILE = "shelf_AzureBatch.mel" cmd_name = "AzureBatch" fMayaExitingCB = None os.environ["AZUREBATCH_VERSION"] = VERSION -def sla_prompt(): - """Open prompt for T's & C's agreement.""" +def eula_prompt(): + """Open prompt for terms and conditions.""" current_file = inspect.getfile(inspect.currentframe()) current_dir = os.path.dirname(os.path.abspath(current_file)) - SLA = os.path.join(current_dir, "SLA.html") + eula = os.path.join(current_dir, "EULA.html") form = cmds.setParent(q=True) cmds.formLayout(form, e=True, width=500) heading = cmds.text( - l='Maya Cloud Rendering Service Level Agreement', font="boldLabelFont") + l='Maya Cloud Rendering License Agreement', font="boldLabelFont") text = cmds.text(l="By loading this plug-in you are agreeing to " "the following terms and conditions.") - if not os.path.exists(SLA): - raise RuntimeError("SLA notice not found at {0}".format(SLA)) + if not os.path.exists(eula): + raise RuntimeError("EULA notice not found at {0}".format(eula)) - with open(SLA, "rb") as sla_text: - html = sla_text.read() + with open(eula, "rb") as eula_text: + html = eula_text.read() unicode = html.decode("windows-1252") encoded_str = unicode.encode("ascii", "xmlcharrefreplace") read = cmds.scrollField(editable=False, wordWrap=True, height=300, @@ -388,7 +364,7 @@ def install_pkg(package): def install_namespace_pkg(package, namespace): - """Azure packages have issues installing one by one as the don't + """Azure packages have issues installing one by one as they don't unpackage correctly into the namespace directory. So we have to install to a temp directory and move it to the right place. @@ -418,19 +394,19 @@ def install_namespace_pkg(package, namespace): def initializePlugin(obj): """Initialize Plug-in""" print("Initializing Azure Batch plug-in") - existing = cmds.optionVar(exists=SLA_PREF) + existing = cmds.optionVar(exists=EULA_PREF) if not existing: - agree = cmds.layoutDialog(ui=sla_prompt, title="Azure Batch Maya Client") + agree = cmds.layoutDialog(ui=eula_prompt, title="Azure Batch Maya Client") if str(agree) != 'Agree': raise RuntimeError("Plugin initialization aborted.") - cmds.optionVar(stringValue=(SLA_PREF, VERSION)) + cmds.optionVar(stringValue=(EULA_PREF, VERSION)) else: - agreed = cmds.optionVar(query=SLA_PREF) + agreed = cmds.optionVar(query=EULA_PREF) if StrictVersion(agreed) < VERSION: - agree = cmds.layoutDialog(ui=sla_prompt, title="AzureBatch Maya Client") + agree = cmds.layoutDialog(ui=eula_prompt, title="Azure Batch Maya Client") if str(agree) != 'Agree': raise RuntimeError("Plugin initialization aborted.") - cmds.optionVar(stringValue=(SLA_PREF, VERSION)) + cmds.optionVar(stringValue=(EULA_PREF, VERSION)) print("Checking for dependencies...") missing_libs = [] @@ -483,7 +459,7 @@ def initializePlugin(obj): raise ImportError("Please restart Maya. Azure Batch installed " "Python dependencies.") - print("Dependency check complete!") + print("Dependency check complete") plugin = OpenMayaMPx.MFnPlugin( obj, "Microsoft Corporation", VERSION, "Any") plugin.registerCommand(cmd_name, cmd_creator) @@ -507,7 +483,7 @@ def initializePlugin(obj): def uninitializePlugin(obj): """Remove and uninstall plugin.""" - print("Removing AzureBatch plug-in") + print("Removing Azure Batch plug-in") plugin = MFnPlugin(obj) plugin.deregisterCommand(cmd_name) try: @@ -520,7 +496,7 @@ def uninitializePlugin(obj): if cmds.window("AzureBatch", exists=1): cmds.deleteUI("AzureBatch") AzureBatchSetup.remove_environment() - print("Finished clearing up all AzureBatch components") + print("Finished clearing up all Azure Batch components") """Check for environment and set up if not found.""" @@ -528,5 +504,5 @@ try: sys.path.extend(os.environ["AZUREBATCH_SCRIPTS"].split(os.pathsep)) sys.path.append(os.environ['AZUREBATCH_MODULES']) except KeyError as e: - print("Couldn't find AzureBatch environment, setting up now...") + print("Couldn't find Azure Batch environment, setting up now...") setup_module() diff --git a/azure_batch_maya/plug-in/SLA.html b/azure_batch_maya/plug-in/EULA.html similarity index 100% rename from azure_batch_maya/plug-in/SLA.html rename to azure_batch_maya/plug-in/EULA.html diff --git a/azure_batch_maya/scripts/api.py b/azure_batch_maya/scripts/api.py index 3505ecf..7ea506d 100644 --- a/azure_batch_maya/scripts/api.py +++ b/azure_batch_maya/scripts/api.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- try: from maya import cmds, mel, utils @@ -32,7 +9,6 @@ try: import maya.OpenMayaMPx as omp except ImportError: print("No maya module found.") - import os import logging LOG = logging.getLogger('AzureBatchMaya') @@ -331,6 +307,14 @@ class MayaAPI(object): LOG.debug("MayaAPI exception in 'int_slider': {0}".format(exp).strip()) return None + @staticmethod + def int_field(*args, **kwargs): + try: + return cmds.intField(*args, **kwargs) + except Exception as exp: + LOG.debug("MayaAPI exception in 'int_field': {0}".format(exp).strip()) + return None + @staticmethod def popup_menu(*args, **kwargs): try: diff --git a/azure_batch_maya/scripts/assets.py b/azure_batch_maya/scripts/assets.py index 55934f6..2262590 100644 --- a/azure_batch_maya/scripts/assets.py +++ b/azure_batch_maya/scripts/assets.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import logging from datetime import datetime @@ -52,15 +29,16 @@ from default import AzureBatchRenderAssets SYS_SEARCHPATHS = [] USR_SEARCHPATHS = [] -UPLOAD_THREADS = 10 +BYTES = 1024 class AzureBatchAssets(object): """Handler for asset file functionality.""" - def __init__(self, frame, call): + def __init__(self, index, frame, call): """Create new Asset Handler. + :param index: The UI tab index. :param frame: The shared plug-in UI frame. :type frame: :class:`.AzureBatchUI` :param func call: The shared REST API call wrapper. @@ -69,6 +47,8 @@ class AzureBatchAssets(object): self._call = call self._session = None self._assets = None + self._tab_index = index + self._upload_threads = None self.batch = None self.modules = self._collect_modules() @@ -154,6 +134,12 @@ class AzureBatchAssets(object): self.renderer = AzureBatchRenderAssets() self._log.debug("Configured renderer to {0}".format(self.renderer.render_engine)) + def _switch_tab(self): + """Make this tab the currently displayed tab. If this tab is already + open, this will do nothing. + """ + self.frame.select_tab(self._tab_index) + def _collect_assets(self): """Called on upload. If the asset tab has not yet been loaded before job submission is attempted, then the asset references have not yet @@ -174,7 +160,7 @@ class AzureBatchAssets(object): for rule in maya.workspace(fileRuleList=True): project_dir = maya.workspace(fileRuleEntry=rule) remote_path = utils.get_remote_directory(maya.workspace(en=project_dir), os_flavor) - if os_flavor == 'Windows': + if os_flavor == utils.OperatingSystem.windows: full_remote_path = "X:\\\\" + remote_path else: full_remote_path = "/X/" + remote_path @@ -203,7 +189,7 @@ class AzureBatchAssets(object): handle.write("loadPlugin \"{}\";\n".format(plugin)) handle.write("dirmap -en true;\n") for local, remote in pathmap.items(): - if os_flavor == 'Windows': + if os_flavor == utils.OperatingSystem.windows: full_remote_path = "X:\\\\" + remote(os_flavor) else: full_remote_path = "/X/" + remote(os_flavor) @@ -213,18 +199,19 @@ class AzureBatchAssets(object): return Asset(map_file, [], self.batch, self._log) def _upload_all(self, to_upload, progress, total, project): - """Upload all selected assets in 10 threads.""" + """Upload all selected assets in configured number of threads.""" uploads_running = [] progress_queue = Queue() - for i in range(0, len(to_upload), UPLOAD_THREADS): - for index, asset in enumerate(to_upload[i:i + UPLOAD_THREADS]): + threads = self._upload_threads() + self._log.debug("Uploading assets in {} threads.".format(threads)) + for i in range(0, len(to_upload), threads): + for index, asset in enumerate(to_upload[i:i + threads]): self._log.debug("Starting thread for asset: {}".format(asset.path)) upload = threading.Thread( target=asset.upload, args=(index, progress, progress_queue, project)) upload.start() uploads_running.append(upload) self._log.debug("Batch of asset uploads pending: {}".format(threading.active_count())) - while any(t for t in uploads_running if t.is_alive()) or not progress_queue.empty(): uploaded = progress_queue.get() if isinstance(uploaded, Exception): @@ -232,18 +219,23 @@ class AzureBatchAssets(object): elif callable(uploaded): uploaded() else: - total = total - (uploaded/1024/1024) + total = total - (uploaded/BYTES/BYTES) self.ui.upload_status("Uploading {0}...".format(self._format_size(total))) progress_queue.task_done() - def _format_size(self, data): + def _format_size(self, nbytes): """Format the data size in bytes to nicely display for upload progress. """ - if data > 1024: - return "{:0.2f}GB".format(data/1014) - else: - return "{:0.2f}MB".format(data) + suffixes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] + if nbytes == 0: + return '0 B' + i = 0 + while nbytes >= BYTES and i < len(suffixes)-1: + nbytes /= BYTES + i += 1 + f = ('%.2f' % nbytes).rstrip('0').rstrip('.') + return '%s %s' % (f, suffixes[i]) def _total_data(self, files): """Format the combined size of the files to display @@ -252,13 +244,14 @@ class AzureBatchAssets(object): data = float(0) for asset in files: data += asset.size - return data/1024/1024 + return data/BYTES/BYTES def configure(self, session): """Populate the Batch client for the current sessions of the asset tab. Called on successful authentication. """ self._session = session + self._upload_threads = session.get_threads self.batch = self._session.batch self._set_searchpaths() self._assets = Assets(self.batch) @@ -288,14 +281,14 @@ class AzureBatchAssets(object): def add_files(self, files, column_layout, scroll_layout): """Function called by the 'Add File(s)' button in the UI for adding arbitrary - file references to the collection that included with the next job subbmission. + file references to the collection to be included with the next job subbmission. """ for f in files: self._assets.add_asset(f, self.ui, column_layout, scroll_layout) def add_dir(self, dirs, column_layout, scroll_layout): """Function called by the 'Add Directory' button in the UI for adding arbitrary - file references to the collection that included with the next job subbmission. + file references to the collection to be included with the next job subbmission. """ for folder in dirs: for root, _, files in os.walk(folder): @@ -310,8 +303,8 @@ class AzureBatchAssets(object): if the upload process is part of job submission. :param progress_bar: The progress of the current process. This is only populated if the upload process is part of job submission. - :param job_id: The ID of the job being submitted. This is only populated is the - upload process if path of job submission. + :param job_id: The ID of the job being submitted. This is only populated if the + upload process is part of job submission. :param load_plugins: A list of plugins to be added to the pre-render script for loading on the server. Only populated if part of a job submission. :param os_flavor: The OS flavor of the rendering pool. Only set as part of the job @@ -339,7 +332,7 @@ class AzureBatchAssets(object): progress_bar.is_cancelled() progress_bar.status('Uploading files...') progress_bar.max(len(asset_refs)) - self.frame.select_tab(3) + self._switch_tab() self.ui.disable(False) self.ui.upload_button.start() payload = self._total_data(asset_refs) @@ -364,7 +357,7 @@ class AzureBatchAssets(object): else: maya.error(str(exp)) finally: - # If part of job submission errors and progress bar + # If part of job submission, errors and progress bar # will be handled back in submission.py if not job_set: progress_bar.end() @@ -548,7 +541,7 @@ class Assets(object): class Asset(object): - """Representation of a single asset, managing it's file reference, + """Representation of a single asset, managing its file reference, display listing and upload of the file. """ @@ -652,7 +645,7 @@ class Asset(object): def make_visible(self, index): """Attempt to auto-scroll the asset display list so that the progress of currently uploading assets remains in view. - TODO: Thie needs some work.... + TODO: This needs some work.... """ if index == 0: while maya.scroll_layout(self.scroll_layout, query=True, scrollAreaValue=True)[0] > 0: diff --git a/azure_batch_maya/scripts/batch_extensions/_file_utils.py b/azure_batch_maya/scripts/batch_extensions/_file_utils.py index 4edf860..5ee2201 100644 --- a/azure_batch_maya/scripts/batch_extensions/_file_utils.py +++ b/azure_batch_maya/scripts/batch_extensions/_file_utils.py @@ -226,6 +226,7 @@ class FileUtils(object): def __init__(self, get_storage_client): self.resource_file_cache = {} + self.container_sas_cache = {} self.resolve_storage_account = get_storage_client def filter_resource_cache(self, container, prefix): @@ -264,7 +265,11 @@ class FileUtils(object): def get_container_sas(self, file_group_name): storage_client = self.resolve_storage_account() container = _get_container_name(file_group_name) - return _generate_container_sas_token(container, storage_client) + try: + return self.container_sas_cache[container] + except KeyError: + self.container_sas_cache[container] = _generate_container_sas_token(container, storage_client) + return self.container_sas_cache[container] def get_container_list(self, source): """List blob references in container.""" diff --git a/azure_batch_maya/scripts/batch_extensions/_job_utils.py b/azure_batch_maya/scripts/batch_extensions/_job_utils.py index 94bd3b7..3740d4f 100644 --- a/azure_batch_maya/scripts/batch_extensions/_job_utils.py +++ b/azure_batch_maya/scripts/batch_extensions/_job_utils.py @@ -5,7 +5,12 @@ from msrest.exceptions import ValidationError, ClientRequestError from azure.batch.models import BatchErrorException - +import threading +try: + from queue import Queue +except ImportError: + from Queue import Queue + # pylint: disable=too-few-public-methods @@ -25,16 +30,34 @@ def _handle_batch_exception(action): raise Exception(ex) -def deploy_tasks(client, job_id, tasks): +def _bulk_add_tasks(client, job_id, tasks, queue): + added_tasks = client.add_collection(job_id, tasks) + for task in added_tasks.value: + queue.put(task) + +def deploy_tasks(client, job_id, tasks, threads): MAX_TASKS_COUNT_IN_BATCH = 100 - + submit_threads = threads or 10 + def add_task(): start = 0 - while start < len(tasks): + progress_queue = Queue() + submitting_tasks = [] + submitted_tasks = [] + while True: end = min(start + MAX_TASKS_COUNT_IN_BATCH, len(tasks)) - client.add_collection(job_id, tasks[start:end]) + submit = threading.Thread(target=_bulk_add_tasks, args=(client, job_id, tasks[start:end], progress_queue)) + submit.start() + submitting_tasks.append(submit) start = end - + if start >= len(tasks) or len(submitting_tasks) >= submit_threads: + while any(s for s in submitting_tasks if s.is_alive()) or not progress_queue.empty(): + submitted_tasks.append(progress_queue.get()) + progress_queue.task_done() + submitting_tasks = [] + if start >= len(tasks): + break + return submitted_tasks _handle_batch_exception(add_task) diff --git a/azure_batch_maya/scripts/batch_extensions/_template_utils.py b/azure_batch_maya/scripts/batch_extensions/_template_utils.py index b23e286..8b432b0 100644 --- a/azure_batch_maya/scripts/batch_extensions/_template_utils.py +++ b/azure_batch_maya/scripts/batch_extensions/_template_utils.py @@ -436,7 +436,7 @@ def _parse_arm_parameter(name, template_obj, parameters): user_value = user_value['value'] except TypeError: pass - if not user_value: + if user_value is None: raise ValueError("No value supplied for parameter '{}' and no default value".format(name)) if isinstance(user_value, dict): # If substitute value is a complex object - it may require diff --git a/azure_batch_maya/scripts/batch_extensions/batch_extensions_client.py b/azure_batch_maya/scripts/batch_extensions/batch_extensions_client.py index 7f73314..6da3ffc 100644 --- a/azure_batch_maya/scripts/batch_extensions/batch_extensions_client.py +++ b/azure_batch_maya/scripts/batch_extensions/batch_extensions_client.py @@ -106,11 +106,11 @@ class BatchExtensionsClient(BatchServiceClient): # the Batch account in the subscription # Example URL: https://batchaccount.westus.batch.azure.com region = urlsplit(self.config.base_url).netloc.split('.', 2)[1] - accounts = [x for x in client.batch_account.list() - if x.name == self._account and x.location == region] + accounts = (x for x in client.batch_account.list() + if x.name == self._account and x.location == region) try: - account = accounts[0] - except IndexError: + account = next(accounts) + except StopIteration: raise ValueError('Couldn\'t find the account named {} in subscription {} ' 'in region {}'.format( self._account, self._subscription, region)) @@ -125,7 +125,7 @@ class BatchExtensionsClient(BatchServiceClient): keys = storage_client.storage_accounts.list_keys(storage_resource_group, storage_account) storage_key = keys.keys[0].value # pylint: disable=no-member - self.resolved_storage_client = CloudStorageAccount(storage_account, storage_key)\ + self._resolved_storage_client = CloudStorageAccount(storage_account, storage_key)\ .create_block_blob_service() - return self.resolved_storage_client + return self._resolved_storage_client diff --git a/azure_batch_maya/scripts/batch_extensions/operations/job_operations.py b/azure_batch_maya/scripts/batch_extensions/operations/job_operations.py index 246db4d..0abded8 100644 --- a/azure_batch_maya/scripts/batch_extensions/operations/job_operations.py +++ b/azure_batch_maya/scripts/batch_extensions/operations/job_operations.py @@ -89,7 +89,7 @@ class ExtendedJobOperations(JobOperations): except Exception as exp: raise ValueError("Unable to deserialize to ExtendedJobParameter: {}".format(exp)) - def add(self, job, job_add_options=None, custom_headers=None, raw=False, **operation_config): + def add(self, job, job_add_options=None, custom_headers=None, raw=False, threads=None, **operation_config): """Adds a job to the specified account. The Batch service supports two ways to control the work done as part of @@ -176,7 +176,7 @@ class ExtendedJobOperations(JobOperations): # Begin original job add process result = super(ExtendedJobOperations, self).add(job, job_add_options, custom_headers, raw, **operation_config) if task_collection: - job_utils.deploy_tasks(self._parent.task, job.id, task_collection) + job_utils.deploy_tasks(self._parent.task, job.id, task_collection, threads) if auto_complete: # If the option to terminate the job was set, we need to reapply it with a patch # now that the tasks have been added. diff --git a/azure_batch_maya/scripts/config.py b/azure_batch_maya/scripts/config.py index 08d3ea9..86abfdb 100644 --- a/azure_batch_maya/scripts/config.py +++ b/azure_batch_maya/scripts/config.py @@ -1,35 +1,13 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import ConfigParser import os import logging import sys +import traceback from ui_config import ConfigUI from api import MayaAPI as maya @@ -49,16 +27,20 @@ LOG_LEVELS = { class AzureBatchConfig(object): """Handler for authentication and configuration of the SDK clients.""" - def __init__(self, frame, start): + def __init__(self, index, frame, start): """Create new configuration Handler. + :param index: The UI tab index. :param frame: The shared plug-in UI frame. :type frame: :class:`.AzureBatchUI` :param func call: The shared REST API call wrapper. """ + self.ui = None self.session = start + self._tab_index = index self._data_dir = os.path.join(maya.prefs_dir(), 'AzureBatchData') self._ini_file = "azure_batch.ini" + self._user_agent = "batchmaya/{}".format(os.environ.get('AZUREBATCH_VERSION')) self._cfg = ConfigParser.ConfigParser() self._client = None self._log = None @@ -92,25 +74,31 @@ class AzureBatchConfig(object): os.makedirs(self._data_dir) config_file = os.path.join(self._data_dir, self._ini_file) if not os.path.exists(config_file): - self._log = self._configure_logging(10) + self._log = self._configure_logging(LOG_LEVELS['debug']) return try: self._cfg.read(config_file) self._storage = storage.BlockBlobService( - self._cfg.get("AzureBatch", "storage_account"), - self._cfg.get("AzureBatch", "storage_key"), - endpoint_suffix="core.windows.net") + self._cfg.get('AzureBatch', 'storage_account'), + self._cfg.get('AzureBatch', 'storage_key')) self._storage.MAX_SINGLE_PUT_SIZE = 2 * 1024 * 1024 credentials = SharedKeyCredentials( - self._cfg.get("AzureBatch", "batch_account"), - self._cfg.get("AzureBatch", "batch_key")) + self._cfg.get('AzureBatch', 'batch_account'), + self._cfg.get('AzureBatch', 'batch_key')) self._client = batch.BatchExtensionsClient( - credentials, base_url=self._cfg.get("AzureBatch", "batch_url"), + credentials, base_url=self._cfg.get('AzureBatch', 'batch_url'), storage_client=self._storage) + self._client.config.add_user_agent(self._user_agent) self._log = self._configure_logging( - self._cfg.get("AzureBatch", "logging")) - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError) as exp: - print(exp) #TODO: Better error handling + self._cfg.get('AzureBatch', 'logging')) + except Exception as exp: + # We should only worry about this if it happens when authenticating + # using the UI, otherwise it's expected. + if self.ui: + raise ValueError("Invalid Configuration: {}".format(exp)) + else: + # We'll need a place holder logger + self._log = self._configure_logging(LOG_LEVELS['debug']) def _configure_logging(self, log_level): """Configure the logger. Setup the file output and format @@ -137,33 +125,37 @@ class AzureBatchConfig(object): configuration file. """ try: - self._cfg.add_section("AzureBatch") + self._cfg.add_section('AzureBatch') except ConfigParser.DuplicateSectionError: pass try: - self.ui.endpoint = self._cfg.get("AzureBatch", "batch_url") + self.ui.endpoint = self._cfg.get('AzureBatch', 'batch_url') except ConfigParser.NoOptionError: self.ui.endpoint = "" try: - self.ui.account = self._cfg.get("AzureBatch", "batch_account") + self.ui.account = self._cfg.get('AzureBatch', 'batch_account') except ConfigParser.NoOptionError: self.ui.account = "" try: - self.ui.key = self._cfg.get("AzureBatch", "batch_key") + self.ui.key = self._cfg.get('AzureBatch', 'batch_key') except ConfigParser.NoOptionError: self.ui.key = "" try: - self.ui.storage = self._cfg.get("AzureBatch", "storage_account") + self.ui.storage = self._cfg.get('AzureBatch', 'storage_account') except ConfigParser.NoOptionError: self.ui.storage = "" try: - self.ui.storage_key = self._cfg.get("AzureBatch", "storage_key") + self.ui.storage_key = self._cfg.get('AzureBatch', 'storage_key') except ConfigParser.NoOptionError: self.ui.storage_key = "" try: - self.ui.logging = int(self._cfg.get("AzureBatch", "logging")) + self.ui.logging = self._cfg.getint('AzureBatch', 'logging') except ConfigParser.NoOptionError: self.ui.logging = 10 + try: + self.ui.threads = self._cfg.getint('AzureBatch', 'threads') + except ConfigParser.NoOptionError: + self.ui.threads = 20 self.ui.set_authenticate(self._auth) def _auto_authentication(self): @@ -172,82 +164,95 @@ class AzureBatchConfig(object): """ try: filter = batch.models.PoolListOptions(max_results=1, select="id") - self._client.pool.list(filter) - self._storage.create_container("batch-maya-assets", fail_on_exist=False) + list(self._client.pool.list(filter)) + self._storage.list_containers(num_results=1) return True except Exception as exp: - self._log.info("Could not get authenticate session: {0}".format(exp)) + self._log.info("Failed to authenticate: {0}".format(exp)) return False + def _save_config(self): + """Persist the current plugin configuration to file.""" + config_file = os.path.join(self._data_dir, self._ini_file) + with open(config_file, 'w') as handle: + self._cfg.write(handle) + def set_logging(self, level): """Set the logging level to that specified in the UI. :param str level: The specified logging level. """ - log_level = int(LOG_LEVELS[level]) - self._log.setLevel(log_level) - self._cfg.set("AzureBatch", "logging", str(level)) + self._log.setLevel(level) + self._cfg.set('AzureBatch', 'logging', level) + self._save_config() + + def set_threads(self, threads): + """Set the number of threads to that specified in the UI. + :param int threads: The specified number of threads. + """ + self._cfg.set('AzureBatch', 'threads', threads) + self._save_config() def save_changes(self): - """Persist configuration changes to file for future sessions.""" + """Persist auth config changes to file for future sessions.""" try: - self._cfg.add_section("AzureBatch") + self._cfg.add_section('AzureBatch') except ConfigParser.DuplicateSectionError: pass - self._cfg.set("AzureBatch", "batch_url", self.ui.endpoint) - self._cfg.set("AzureBatch", "batch_account", self.ui.account) - self._cfg.set("AzureBatch", "batch_key", self.ui.key) - self._cfg.set("AzureBatch", "storage_account", self.ui.storage) - self._cfg.set("AzureBatch", "storage_key", self.ui.storage_key) - self._cfg.set("AzureBatch", "logging", self.ui.logging) - config_file = os.path.join(self._data_dir, self._ini_file) - with open(config_file, 'w') as handle: - self._cfg.write(handle) + self._cfg.set('AzureBatch', 'batch_url', self.ui.endpoint) + self._cfg.set('AzureBatch', 'batch_account', self.ui.account) + self._cfg.set('AzureBatch', 'batch_key', self.ui.key) + self._cfg.set('AzureBatch', 'storage_account', self.ui.storage) + self._cfg.set('AzureBatch', 'storage_key', self.ui.storage_key) + self._save_config() def authenticate(self): """Begin authentication - initiated by the UI button.""" - self._configure_plugin() - self._auth = self._auto_authentication() - self.ui.set_authenticate(self._auth) - self.session() + try: + self._configure_plugin() + self._auth = self._auto_authentication() + except ValueError as exp: + maya.error(str(exp)) + self._auth = False + finally: + self.ui.set_authenticate(self._auth) + self.session() + + def get_threads(self): + """Attempt to retrieve number of threads configured for the plugin.""" + return self.ui.threads def get_cached_vm_sku(self): """Attempt to retrieve a selected VM SKU from a previous session.""" try: - return self._cfg.get("AzureBatch", "vm_sku") + return self._cfg.get('AzureBatch', 'vm_sku') except ConfigParser.NoOptionError: return None def store_vm_sku(self, sku): """Cache selected VM SKU for later sessions.""" - self._cfg.set("AzureBatch", "vm_sku", sku) - config_file = os.path.join(self._data_dir, self._ini_file) - with open(config_file, 'w') as handle: - self._cfg.write(handle) + self._cfg.set('AzureBatch', 'vm_sku', sku) + self._save_config() def get_cached_image(self): """Attempt to retrieve a selected image a previous session.""" try: - return self._cfg.get("AzureBatch", "image") + return self._cfg.get('AzureBatch', 'image') except ConfigParser.NoOptionError: return None def store_image(self, image): """Cache selected image for later sessions.""" - self._cfg.set("AzureBatch", "image", image) - config_file = os.path.join(self._data_dir, self._ini_file) - with open(config_file, 'w') as handle: - self._cfg.write(handle) + self._cfg.set('AzureBatch', 'image', image) + self._save_config() def get_cached_autoscale_formula(self): """Attempt to retrieve an autoscale forumla from a previous session.""" try: - return self._cfg.get("AzureBatch", "autoscale") + return self._cfg.get('AzureBatch', 'autoscale') except ConfigParser.NoOptionError: return None def store_autoscale_formula(self, formula): """Cache selected VM SKU for later sessions.""" - self._cfg.set("AzureBatch", "autoscale", formula) - config_file = os.path.join(self._data_dir, self._ini_file) - with open(config_file, 'w') as handle: - self._cfg.write(handle) + self._cfg.set('AzureBatch', 'autoscale', formula) + self._save_config() diff --git a/azure_batch_maya/scripts/environment.py b/azure_batch_maya/scripts/environment.py index a6fac2d..456bf8e 100644 --- a/azure_batch_maya/scripts/environment.py +++ b/azure_batch_maya/scripts/environment.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os import logging @@ -32,7 +9,7 @@ import json from api import MayaAPI as maya from api import MayaCallbacks as callback - +import utils from ui_environment import EnvironmentUI @@ -41,16 +18,16 @@ MAYA_IMAGES = { { 'node_sku_id': 'batch.node.windows amd64', 'publisher': 'batch', - 'offer': 'autodesk-maya-arnold-win2016-preview', - 'sku': 'maya2017', + 'offer': 'rendering-windows2016', + 'sku': 'rendering', 'version': 'latest' }, 'Batch CentOS Preview': { 'node_sku_id': 'batch.node.centos 7', 'publisher': 'batch', - 'offer': 'autodesk-maya-arnold-centos73-preview', - 'sku': 'maya2017', + 'offer': 'autodesk-maya-arnold-centos73', + 'sku': 'maya-arnold-2017', 'version': 'latest' }, } @@ -63,9 +40,10 @@ LICENSES = [ class AzureBatchEnvironment(object): """Handler for rendering environment configuration functionality.""" - def __init__(self, frame, call): + def __init__(self, index, frame, call): """Create new Environment Handler. + :param index: The UI tab index. :param frame: The shared plug-in UI frame. :type frame: :class:`.AzureBatchUI` :param func call: The shared REST API call wrapper. @@ -73,6 +51,7 @@ class AzureBatchEnvironment(object): self._log = logging.getLogger('AzureBatchMaya') self._call = call self._session = None + self._tab_index = index self.licenses = {} self._get_plugin_licenses() @@ -141,8 +120,8 @@ class AzureBatchEnvironment(object): if pool_image: return pool_image[0] else: - self._log.debug("Pool using unknown image reference: {}".format(image_ref['offer'])) - return "" + self._log.debug("Pool using unknown image reference: {}".format(image_ref.offer)) + return image_ref.offer def get_vm_sku(self): return self.ui.get_sku() @@ -152,16 +131,16 @@ class AzureBatchEnvironment(object): windows_offers = [value['offer'] for value in MAYA_IMAGES.values() if 'windows' in value['node_sku_id']] linux_offers = [value['offer'] for value in MAYA_IMAGES.values() if value['offer'] not in windows_offers] if pool_image.offer in windows_offers: - return 'Windows' + return utils.OperatingSystem.windows elif pool_image.offer in linux_offers: - return 'Linux' + return utils.OperatingSystem.linux else: raise ValueError('Selected pool is not using a valid Maya image.') - if 'Windows' in self.ui.get_image(): - return 'Windows' + if utils.OperatingSystem.windows.value in self.ui.get_image(): + return utils.OperatingSystem.windows else: - return 'Linux' + return utils.OperatingSystem.linux def get_environment_settings(self): env_vars = self.ui.get_env_vars() diff --git a/azure_batch_maya/scripts/exception.py b/azure_batch_maya/scripts/exception.py index bf68332..93559ae 100644 --- a/azure_batch_maya/scripts/exception.py +++ b/azure_batch_maya/scripts/exception.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import traceback import logging diff --git a/azure_batch_maya/scripts/history.py b/azure_batch_maya/scripts/jobhistory.py similarity index 71% rename from azure_batch_maya/scripts/history.py rename to azure_batch_maya/scripts/jobhistory.py index c2f483f..6f05cf8 100644 --- a/azure_batch_maya/scripts/history.py +++ b/azure_batch_maya/scripts/jobhistory.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os import logging @@ -37,17 +14,18 @@ import shutil import re from api import MayaAPI as maya -from ui_history import HistoryUI +from ui_jobhistory import JobHistoryUI import azure.batch as batch -class AzureBatchHistory(object): +class AzureBatchJobHistory(object): """Handler for job display functionality.""" - def __init__(self, frame, call): + def __init__(self, index, frame, call): """Create new job history Handler. + :param index: The UI tab index. :param frame: The shared plug-in UI frame. :type frame: :class:`.AzureBatchUI` :param func call: The shared REST API call wrapper. @@ -55,13 +33,14 @@ class AzureBatchHistory(object): self._log = logging.getLogger('AzureBatchMaya') self._call = call self._session = None + self._tab_index = index self.batch = None self.index = 0 - self.per_call = 5 + self.jobs_per_page = 5 self.count = 0 self.min = True self.max = False - self.ui = HistoryUI(self, frame) + self.ui = JobHistoryUI(self, frame) self.all_jobs = [] self.jobs = [] self.selected_job = None @@ -70,17 +49,28 @@ class AzureBatchHistory(object): """Get the pixel height of the job thumbnail to display. Note: This function only works under Python 2.7 """ + # The first 8 bytes of data are the PNG signature, the next 4 + # are the type field of the first chunk (which we don't need). + png_signature_bytes = 12 + # The IHDR header is always the first chunk in a valid PNG image + png_header = 'IHDR' + # X and Y image dimensions are the first 8 bytes of the IDHR chunk + dimensions_bytes = 8 + # 120 pixels is the default height of a thumbnail image y = 120 + # Byte order = network (!), format = unsigned long (L) + excepted_byte_format = '!LL' with open(image, 'rb') as im: - im.read(12) - if im.read(4) == 'IHDR': - x, y = struct.unpack("!LL", im.read(8)) + png_signature = im.read(png_signature_bytes) + valid_png = im.read(len(png_header)) == png_header + if valid_png: + x, y = struct.unpack(excepted_byte_format, im.read(dimensions_bytes)) return y def _download_thumbnail(self, job, thumbs): """Download a preview thumbnail for the selected job. - Only certain output formats are supported. If not thumbnail exists - then we display the default 'no preview' image. + Only certain output formats are supported. If the thumbnail doesn't + exist then we display the default 'no preview' image. TODO: Remove direct storage reference to use batch.download. :param job: The selected job object. @@ -105,7 +95,7 @@ class AzureBatchHistory(object): self._log.debug("Thumbnail path: {}".format(thumb_path)) try: if not os.path.isfile(thumb_path): - self._log.info("Downloading task thumb: {}".format(thumbs[-1])) + self._log.info("Downloading task thumbnail: {}".format(thumbs[-1])) self.storage.get_blob_to_path('fgrp-' + job.id, thumbs[-1], thumb_path) self._log.info(" thumbnail download successful.\n") except Exception as exp: @@ -120,25 +110,25 @@ class AzureBatchHistory(object): """Calculate the paging progress label, including which page is currently displayed out of how many. """ - if (self.index + self.per_call) > self.count: - extra = (self.index + self.per_call) - self.count - new_range = ((self.index + self.per_call) - extra) + if (self.index + self.jobs_per_page) > self.count: + extra = (self.index + self.jobs_per_page) - self.count + new_range = ((self.index + self.jobs_per_page) - extra) self.ui.num_jobs = "{0} - {1} of {2}".format( min((self.index + 1), new_range), new_range, self.count) else: self.ui.num_jobs = "{0} - {1} of {2}".format( - min((self.index + 1), self.count), (self.index + self.per_call), self.count) + min((self.index + 1), self.count), (self.index + self.jobs_per_page), self.count) def _set_min_max(self): """Determine whether we are currently displaying the first or - last page, for that the forward and back buttons can be disabled + last page, so that the forward and back buttons can be disabled accordingly. """ self.min = True if self.index < 1 else False - if (self.count % self.per_call) == 0: - self.max = (self.index >= (self.count - self.per_call)) or (self.per_call > self.count) + if (self.count % self.jobs_per_page) == 0: + self.max = (self.index >= (self.count - self.jobs_per_page)) or (self.jobs_per_page > self.count) else: - self.max = self.index >= (self.count - self.per_call + (self.per_call - (self.count % self.per_call))) + self.max = self.index >= (self.count - self.jobs_per_page + (self.jobs_per_page - (self.count % self.jobs_per_page))) self.ui.last_page = not self.max self.ui.first_page = not self.min @@ -172,7 +162,7 @@ class AzureBatchHistory(object): def show_jobs(self): """Display the current page of jobs.""" - self.jobs = self.all_jobs[self.index:self.index + self.per_call] + self.jobs = self.all_jobs[self.index:self.index + self.jobs_per_page] self._set_num_jobs() self._set_min_max() display_jobs = [] @@ -182,11 +172,11 @@ class AzureBatchHistory(object): def show_next_jobs(self): """Show the next page of jobs.""" - self.index = min(self.index + self.per_call, self.count) + self.index = min(self.index + self.jobs_per_page, self.count) def show_prev_jobs(self): """Show the previous page of jobs.""" - self.index = max(self.index - self.per_call, 0) + self.index = max(self.index - self.jobs_per_page, 0) def show_first_jobs(self): """Return to the first page of jobs (most recently submitted).""" @@ -194,14 +184,14 @@ class AzureBatchHistory(object): def show_last_jobs(self): """Skip to the last page of jobs (first ones submitted).""" - if (self.count % self.per_call) == 0: - self.index = self.count - self.per_call + if (self.count % self.jobs_per_page) == 0: + self.index = self.count - self.jobs_per_page else: - self.index = self.count - self.per_call + \ - (self.per_call - (self.count % self.per_call)) + self.index = self.count - self.jobs_per_page + \ + (self.jobs_per_page - (self.count % self.jobs_per_page)) def job_selected(self, job_ui): - """A job has been selected, so it's details need to be retrieved + """A job has been selected, so its details need to be retrieved and displayed. This is also called when a job entry has been refreshed. """ @@ -225,6 +215,32 @@ class AzureBatchHistory(object): self.selected_job.set_thumbnail(loading_thumb, 24) maya.refresh() job = self._call(self.batch.job.get, job.id) + self.selected_job.set_status('loading...') + self.selected_job.set_progress('loading...') + self.selected_job.set_tasks('loading...') + self.selected_job.set_submission(job.creation_time.isoformat()) + self.selected_job.set_job(job.id) + self.selected_job.set_pool(job.pool_info.pool_id) + self.selected_job.set_label(job.display_name) + maya.refresh() + self._log.info("Updated {0}".format(job.display_name)) + except Exception as exp: + self._log.warning("Failed to update job details {0}".format(exp)) + self.ui.refresh() + + def load_tasks(self): + """Get a list of tasks associated with the job.""" + try: + job = self.jobs[self.selected_job.index] + except (IndexError, AttributeError): + self._log.warning("Selected job index does not match jobs list.") + if not self.selected_job: + return + self.selected_job.set_status('unknown') + self.selected_job.set_progress('unknown') + self.selected_job.set_tasks('unknown') + return + try: tasks = list(self._call(self.batch.task.list, job.id)) completed_tasks = [t for t in tasks if t.state == batch.models.TaskState.completed] errored_tasks = [t for t in completed_tasks if t.execution_info.exit_code != 0] @@ -235,18 +251,13 @@ class AzureBatchHistory(object): else: percentage = (100 * len(completed_tasks)) / (len(tasks)) self.selected_job.set_status(state) - self.selected_job.set_progress(percentage) - self.selected_job.set_submission(job.creation_time.isoformat()) + self.selected_job.set_progress(str(percentage)+'%') self.selected_job.set_tasks(len(tasks)) - self.selected_job.set_job(job.id) - self.selected_job.set_pool(job.pool_info.pool_id) - self.selected_job.set_label(job.display_name) maya.refresh() - self._log.info("Updated {0}".format(job.display_name)) except Exception as exp: self._log.warning("Failed to update job details {0}".format(exp)) self.ui.refresh() - + def get_thumbnail(self): """Check job outputs of the currently selected job to find any available thumbnails. @@ -267,7 +278,6 @@ class AzureBatchHistory(object): self._log.warning(exp) blobs = [] thumbs = sorted([b.name for b in blobs]) - print(thumbs) self._download_thumbnail(job, thumbs) def cancel_job(self): @@ -276,6 +286,7 @@ class AzureBatchHistory(object): job = self.jobs[self.selected_job.index] self._call(self.batch.job.terminate, job.id) self.update_job(self.selected_job.index) + maya.execute(self.load_tasks) maya.execute(self.get_thumbnail) maya.refresh() except (IndexError, AttributeError) as exp: @@ -291,6 +302,7 @@ class AzureBatchHistory(object): job = self.jobs[self.selected_job.index] self._call(self.batch.job.delete, job.id) self.update_job(self.selected_job.index) + maya.execute(self.load_tasks) maya.execute(self.get_thumbnail) maya.refresh() except (IndexError, AttributeError) as exp: diff --git a/azure_batch_maya/scripts/pools.py b/azure_batch_maya/scripts/pools.py index 7c8f7de..b6c8274 100644 --- a/azure_batch_maya/scripts/pools.py +++ b/azure_batch_maya/scripts/pools.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os import logging @@ -42,9 +19,10 @@ from ui_pools import PoolsUI class AzureBatchPools(object): """Handler for pool functionality.""" - def __init__(self, frame, call): + def __init__(self, index, frame, call): """Create new Pool Handler. + :param index: The UI tab index. :param frame: The shared plug-in UI frame. :type frame: :class:`.AzureBatchUI` :param func call: The shared REST API call wrapper. @@ -52,6 +30,7 @@ class AzureBatchPools(object): self._log = logging.getLogger('AzureBatchMaya') self._call = call self._session = None + self._tab_index = index self.batch = None self.ui = PoolsUI(self, frame) @@ -77,10 +56,15 @@ class AzureBatchPools(object): """ #if lazy and self.pools: # return [pool.id for pool in self.pools if not pool.auto] - self.pools = [p for p in self._call(self.batch.pool.list) if p.id.startswith('Maya_')] + all_pools = self._call(self.batch.pool.list) + self.pools = [] + for pool in all_pools: + if pool.virtual_machine_configuration and \ + pool.virtual_machine_configuration.image_reference.publisher == 'batch': + self.pools.append(pool) self.pools.sort(key=lambda x: x.creation_time, reverse=True) self.count = len(self.pools) - return [pool.id for pool in self.pools if pool.id.startswith("Maya_Pool")] + return [pool.id for pool in self.pools if not pool.id.startswith("Maya_Auto_Pool")] def get_pools(self): """Retrieves the currently running pools and populates the UI @@ -114,13 +98,13 @@ class AzureBatchPools(object): pool = self._call(self.batch.pool.get, pool.id) _nodes = self._call(self.batch.compute_node.list, pool.id) nodes = [n for n in _nodes] + self.selected_pool.set_id(pool.id) self.selected_pool.set_label(pool.display_name if pool.display_name else pool.id) - self.selected_pool.set_size(pool.current_dedicated_nodes) - self.selected_pool.set_target(pool.target_dedicated_nodes) + self.selected_pool.set_dedicated_size(pool) + self.selected_pool.set_low_pri_size(pool) self.selected_pool.set_type( "Auto" if pool.id.startswith("Maya_Auto_Pool") else "Provisioned") self.selected_pool.set_state(pool.state.value, nodes) - self.selected_pool.set_tasks(pool.max_tasks_per_node) self.selected_pool.set_allocation(pool.allocation_state.value) self.selected_pool.set_created(pool.creation_time) self.selected_pool.set_licenses(pool.application_licenses) @@ -158,10 +142,10 @@ class AzureBatchPools(object): """Get the target number of VMs in the selected pool.""" try: pool = self.pools[self.selected_pool.index] - return int(pool.target_dedicated_nodes) + return int(pool.target_dedicated_nodes), int(pool.target_low_priority_nodes) except (IndexError, TypeError, AttributeError) as exp: self._log.info("Failed to parse pool target size {0}".format(exp)) - return 0 + return 0, 0 def get_pool_os(self, pool_id): """Get the OS flavor of the specified pool ID.""" @@ -190,7 +174,8 @@ class AzureBatchPools(object): application_licenses=self.environment.get_application_licenses(), vm_size=self.environment.get_vm_sku(), virtual_machine_configuration=pool_config, - target_dedicated_nodes=int(size), + target_dedicated_nodes=int(size[0]), + target_low_priority_nodes=int(size[1]), max_tasks_per_node=1) self._call(self.batch.pool.add, new_pool) self._log.debug("Successfully created pool.") @@ -211,7 +196,8 @@ class AzureBatchPools(object): 'virtualMachineConfiguration': pool_config, 'maxTasksPerNode': 1, 'applicationLicenses': self.environment.get_application_licenses(), - 'targetDedicatedNodes': int(size)} + 'targetDedicatedNodes': int(size[0]), + 'targetLowPriorityNodes': int(size[1])} auto_pool = { 'autoPoolIdPrefix': "Maya_Auto_Pool_", 'poolLifetimeOption': "job", @@ -219,13 +205,16 @@ class AzureBatchPools(object): 'pool': pool_spec} return {'autoPoolSpecification': auto_pool} - def resize_pool(self, new_size): + def resize_pool(self, new_dedicated, new_low_pri): """Resize an existing pool.""" try: pool = self.pools[self.selected_pool.index] - self._log.info("Resizing pool '{}' to {} VMs".format(pool.id, new_size)) - self._call(self.batch.pool.resize, pool.id, {'target_dedicated_nodes':int(new_size), - 'target_low_priority_nodes': 0}) + self._log.info( + "Resizing pool '{}' to {} dedicated VMs" + " and {} low priority VMs".format(pool.id, new_dedicated, new_low_pri)) + self._call(self.batch.pool.resize, pool.id, + {'target_dedicated_nodes':int(new_dedicated), + 'target_low_priority_nodes': int(new_low_pri)}) maya.refresh() except Exception as exp: self._log.info("Failed to resize pool {0}".format(exp)) diff --git a/azure_batch_maya/scripts/shared.py b/azure_batch_maya/scripts/shared.py index dc0ac29..ee01333 100644 --- a/azure_batch_maya/scripts/shared.py +++ b/azure_batch_maya/scripts/shared.py @@ -1,40 +1,17 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import logging -import webbrowser import os -import threading +import sys +import traceback from ui_shared import AzureBatchUI from config import AzureBatchConfig from submission import AzureBatchSubmission -from history import AzureBatchHistory +from jobhistory import AzureBatchJobHistory from assets import AzureBatchAssets from pools import AzureBatchPools from environment import AzureBatchEnvironment @@ -51,6 +28,15 @@ ACCEPTED_ERRORS = [ class AzureBatchSettings(object): + tab_index = { + 'AUTH': 1, + 'SUBMIT': 2, + 'ASSETS': 3, + 'POOLS': 4, + 'JOBHISTORY': 5, + 'ENV': 6 + } + @staticmethod def starter(): """Called by the mel script when the shelf button is clicked.""" @@ -63,30 +49,31 @@ class AzureBatchSettings(object): self._log = logging.getLogger('AzureBatchMaya') try: self.frame = AzureBatchUI(self) - self.config = AzureBatchConfig(self.frame, self.start) - self.submission = AzureBatchSubmission(self.frame, self.call) - self.assets = AzureBatchAssets(self.frame, self.call) - self.pools = AzureBatchPools(self.frame, self.call) - self.history = AzureBatchHistory(self.frame, self.call) - self.env = AzureBatchEnvironment(self.frame, self.call) + self.config = AzureBatchConfig(self.tab_index['AUTH'], self.frame, self.start) + self.submission = AzureBatchSubmission(self.tab_index['SUBMIT'], self.frame, self.call) + self.assets = AzureBatchAssets(self.tab_index['ASSETS'], self.frame, self.call) + self.pools = AzureBatchPools(self.tab_index['POOLS'], self.frame, self.call) + self.jobhistory = AzureBatchJobHistory(self.tab_index['JOBHISTORY'], self.frame, self.call) + self.env = AzureBatchEnvironment(self.tab_index['ENV'], self.frame, self.call) self.start() except Exception as exp: if (maya.window("AzureBatch", q=1, exists=1)): maya.delete_ui("AzureBatch") message = "Batch Plugin Failed to Start: {0}".format(exp) maya.error(message) + raise def start(self): """Start the plugin UI. Depending on whether auto-authentication was successful, the plugin will start by displaying the submission tab. - Otherwise the UI will be disables, and the log in tab will be displayed. + Otherwise the UI will be disabled, and the login tab will be displayed. """ try: self._log.debug("Starting AzureBatchShared...") if self.config.auth: self.frame.is_logged_in() self.env.configure(self.config) - self.history.configure(self.config) + self.jobhistory.configure(self.config) self.assets.configure(self.config) self.pools.configure(self.config, self.env) self.submission.start(self.config, self.assets, self.pools, self.env) @@ -119,4 +106,6 @@ class AzureBatchSettings(object): except Exception as exp: if (maya.window("AzureBatch", q=1, exists=1)): maya.delete_ui("AzureBatch") + exc_type, exc_value, exc_traceback = sys.exc_info() + self._log.error("".join(traceback.format_exception(exc_type, exc_value, exc_traceback))) raise ValueError("Error: {0}".format(exp)) diff --git a/azure_batch_maya/scripts/submission.py b/azure_batch_maya/scripts/submission.py index aa3fce3..6ab2ed4 100644 --- a/azure_batch_maya/scripts/submission.py +++ b/azure_batch_maya/scripts/submission.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os import sys @@ -49,16 +26,20 @@ from default import AzureBatchRenderJob class AzureBatchSubmission(object): """Handler for job submission functionality.""" - def __init__(self, frame, call): + def __init__(self, index, frame, call): """Create new Submission Handler. + :param index: The UI tab index. :param frame: The shared plug-in UI frame. :type frame: :class:`.AzureBatchUI` :param func call: The shared REST API call wrapper. """ self._log = logging.getLogger('AzureBatchMaya') self._call = call + self._tab_index = index + self._submit_threads = None + self.max_pool_size = 1000 self.ui = SubmissionUI(self, frame) self.modules = self._collect_modules() self.renderer = None @@ -111,6 +92,12 @@ class AzureBatchSubmission(object): self.renderer = AzureBatchRenderJob() self._log.debug("Configured renderer to {0}".format(self.renderer.render_engine)) + def _switch_tab(self): + """Make this tab the currently displayed tab. If this tab is already + open, this will do nothing. + """ + self.frame.select_tab(self._tab_index) + def _check_outputs(self): """Check whether at least one of the scene cameras is marked as renderable and that at least one layer is a render layer. If not, there will be no @@ -137,10 +124,10 @@ class AzureBatchSubmission(object): culprits to the ignore list if necessary. """ try: - with open(os.path.join(os.environ["AZUREBATCH_TOOLS"], + with open(os.path.join(os.environ['AZUREBATCH_TOOLS'], "supported_plugins.json"), 'r') as plugins: supported_plugins = json.load(plugins) - with open(os.path.join(os.environ["AZUREBATCH_TOOLS"], + with open(os.path.join(os.environ['AZUREBATCH_TOOLS'], "ignored_plugins.json"), 'r') as plugins: ignored_plugins = json.load(plugins) except EnvironmentError: @@ -154,7 +141,7 @@ class AzureBatchSubmission(object): "yet supported.\nRendering may be affected.\n") for plugin in unsupported_plugins: warning += plugin + "\n" - options = ["Continue", "Cancel"] + options = ['Continue', 'Cancel'] answer = maya.confirm(warning, options) if answer == options[-1]: raise CancellationException("Submission Aborted") @@ -188,16 +175,16 @@ class AzureBatchSubmission(object): pool_spec = self.ui.get_pool() if pool_spec.get(1): self._log.info("Using auto-pool.") - return self.pool_manager.create_auto_pool(int(pool_spec[1]), job_name) + return self.pool_manager.create_auto_pool(pool_spec[1], job_name) if pool_spec.get(2): self._log.info("Using existing pool.") pool_id = str(pool_spec[2]) if pool_id == "None": raise PoolException("No pool selected.") - return {"poolId" : pool_id} + return {'poolId' : pool_id} if pool_spec.get(3): self._log.info("Creating new pool.") - return self.pool_manager.create_pool(int(pool_spec[3]), job_name) + return self.pool_manager.create_pool(pool_spec[3], job_name) def start(self, session, assets, pools, env): """Load submission tab after plug-in has been authenticated. @@ -217,6 +204,7 @@ class AzureBatchSubmission(object): self.pool_manager = pools self.env_manager = env self.data_path = session.path + self._submit_threads = session.get_threads if self.renderer: self.renderer.delete() self._configure_renderer() @@ -262,7 +250,8 @@ class AzureBatchSubmission(object): batch_parameters['displayName'] = self.renderer.get_title() batch_parameters['metadata'] = [{"name": "JobType", "value": "Maya"}] template_file = os.path.join( - os.environ['AZUREBATCH_TEMPLATES'], 'arnold-basic-{}.json'.format(pool_os.lower())) + os.environ['AZUREBATCH_TEMPLATES'], + "{}-basic-{}.json".format(self.renderer.render_engine, pool_os.value.lower())) batch_parameters['applicationTemplateInfo'] = {'filePath': template_file} application_params = {} batch_parameters['applicationTemplateInfo']['parameters'] = application_params @@ -280,7 +269,7 @@ class AzureBatchSubmission(object): application_params['assetScript'] = map_url application_params['thumbScript'] = thumb_url application_params['workspace'] = workspace_url - self.frame.select_tab(2) + self._switch_tab() self.ui.submit_status("Configuring job...") progress.status("Configuring job...") @@ -298,7 +287,9 @@ class AzureBatchSubmission(object): progress.is_cancelled() self.ui.submit_status("Submitting...") progress.status("Submitting...") - self._call(self.batch.job.add, new_job) + threads = self._submit_threads() + self._log.debug("Submitting using {} threads.".format(threads)) + self._call(self.batch.job.add, new_job, threads=threads) maya.info("Job submitted successfully") if watch_job: @@ -308,10 +299,10 @@ class AzureBatchSubmission(object): except Exception as exp: self._log.error(str(exp)) exc_type, exc_value, exc_traceback = sys.exc_info() - self._log.debug("".join(traceback.format_exception(exc_type, exc_value, exc_traceback))) + self._log.debug(''.join(traceback.format_exception(exc_type, exc_value, exc_traceback))) maya.error(str(exp)) finally: if progress: progress.end() - self.frame.select_tab(2) + self._switch_tab() self.renderer.disable(True) diff --git a/azure_batch_maya/scripts/tools/install_pip.py b/azure_batch_maya/scripts/tools/install_pip.py index 848dacc..ddbab6a 100644 --- a/azure_batch_maya/scripts/tools/install_pip.py +++ b/azure_batch_maya/scripts/tools/install_pip.py @@ -1,9 +1,8 @@ # coding=utf-8 -# -------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os import tempfile diff --git a/azure_batch_maya/scripts/tools/job_watcher.py b/azure_batch_maya/scripts/tools/job_watcher.py index d197cf4..ee2c200 100644 --- a/azure_batch_maya/scripts/tools/job_watcher.py +++ b/azure_batch_maya/scripts/tools/job_watcher.py @@ -1,9 +1,8 @@ # coding=utf-8 -# -------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import webbrowser import ConfigParser @@ -11,16 +10,17 @@ import time import sys import os import re - +import threading batch_client = None storage_client = None +concurrent_downloads = None +header_line_length = 50 def header(header): header_chars = len(header) - total_len = 50 - dashes = total_len - header_chars + dashes = header_line_length - header_chars mult = int(dashes/2) padded = "\n\n" + mult*"-" + header + mult*"-" if dashes % 2 > 0: @@ -40,22 +40,14 @@ def _check_valid_dir(directory): def _download_output(container, blob_name, output_path, size): - def progress(data, total): - try: - percent = float(data)*100/float(size) - sys.stdout.write(' Downloading... {0}%\r'.format(int(percent))) - except: - sys.stdout.write(' Downloading... %\r') - finally: - sys.stdout.flush() - print("Downloading task output: {}".format(blob_name)) - storage_client.get_blob_to_path(container, blob_name, output_path, progress_callback=progress) - print(" Output download successful.\n") + storage_client.get_blob_to_path(container, blob_name, output_path) + print("Output {} download successful".format(blob_name)) def _track_completed_outputs(container, dwnld_dir): job_outputs = storage_client.list_blobs(container) + downloads = [] for output in job_outputs: if output.name.startswith('thumbs/'): continue @@ -64,7 +56,17 @@ def _track_completed_outputs(container, dwnld_dir): if not os.path.isfile(output_file): if not os.path.isdir(os.path.dirname(output_file)): os.makedirs(os.path.dirname(output_file)) - _download_output(container, output.name, output_file, output.properties.content_length) + downloads.append( + threading.Thread( + target=_download_output, + args=(container, output.name, output_file, output.properties.content_length))) + downloads[-1].start() + if len(downloads) >= concurrent_downloads: + for thread in downloads: + thread.join() + downloads = [] + for thread in downloads: + thread.join() def _check_job_stopped(job): @@ -88,7 +90,7 @@ def _check_job_stopped(job): if job.state in stopped_status: print(header("Job has stopped")) print("Job status: {0}".format(job.state)) - raise RuntimeError("Job is no longer running. Status: {0}".format(job.state)) + raise RuntimeError("Job is no longer active. State: {0}".format(job.state)) elif job.state == JobState.completed: print(header("Job has completed")) return True @@ -115,7 +117,7 @@ def track_job_progress(id, container, dwnld_dir): percentage = (100 * len(completed_tasks)) / len(tasks) print("Running - {}%".format(percentage)) if errored_tasks: - print(" - Warning: some tasks have completed with a non-zero exit code.") + print(" - Warning: some tasks have failed.") _track_completed_outputs(container, dwnld_dir) if _check_job_stopped(job): @@ -129,7 +131,7 @@ def track_job_progress(id, container, dwnld_dir): def _authenticate(cfg_path): - global batch_client, storage_client + global batch_client, storage_client, concurrent_downloads cfg = ConfigParser.ConfigParser() try: cfg.read(cfg_path) @@ -142,6 +144,10 @@ def _authenticate(cfg_path): cfg.get("AzureBatch", "storage_account"), cfg.get("AzureBatch", "storage_key"), endpoint_suffix="core.windows.net") + try: + concurrent_downloads = cfg.get("AzureBatch", "threads") + except ConfigParser.NoSectionError: + concurrent_downloads = 20 except (EnvironmentError, ConfigParser.NoOptionError, ConfigParser.NoSectionError) as exp: raise ValueError("Failed to authenticate using Maya configuration {0}".format(cfg_path)) diff --git a/azure_batch_maya/scripts/ui/ui_assets.py b/azure_batch_maya/scripts/ui/ui_assets.py index 7df7008..d4ed9b3 100644 --- a/azure_batch_maya/scripts/ui/ui_assets.py +++ b/azure_batch_maya/scripts/ui/ui_assets.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os import utils diff --git a/azure_batch_maya/scripts/ui/ui_config.py b/azure_batch_maya/scripts/ui/ui_config.py index fc22123..1cd3a50 100644 --- a/azure_batch_maya/scripts/ui/ui_config.py +++ b/azure_batch_maya/scripts/ui/ui_config.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import utils @@ -60,6 +37,17 @@ class ConfigUI(object): maya.text(label="Service: ", align="left") self._endpoint = maya.text_field(height=25, enable=True) + #TODO: Allow set to 0 to disable threads + with utils.Row(2, 2, (70,260), ("left","left")): + maya.text(label="Threads: ", align="left") + self._threads = maya.int_field( + changeCommand=self.set_threads, + height=25, + minValue=1, + maxValue=40, + enable=True, + value=20) + with utils.Row(2, 2, (70,260), ("left","center"), [(1, "bottom", 20),(2,"bottom",15)]): maya.text(label="Logging: ", align="left") @@ -116,6 +104,16 @@ class ConfigUI(object): """AzureBatch Service Endpoint. Sets contents of text field.""" maya.text_field(self._endpoint, edit=True, text=str(value)) + @property + def threads(self): + """Max number of threads used. Retrieves contents of int field.""" + return maya.int_field(self._threads, query=True, value=True) + + @threads.setter + def threads(self, value): + """Max number of threads used. Sets contents of iny field.""" + maya.int_field(self._threads, edit=True, value=int(value)) + @property def account(self): """AzureBatch Unattended Account ID. Retrieves contents of text field.""" @@ -164,7 +162,10 @@ class ConfigUI(object): @status.setter def status(self, value): """Plug-in authentication status. Sets contents of label.""" - maya.text(self.auth_status, edit=True, label=value) + if value: + maya.text(self.auth_status, edit=True, label="Authenticated", backgroundColor=[0.23, 0.44, 0.21]) + else: + maya.text(self.auth_status, edit=True, label="Not authenticated", backgroundColor=[0.6, 0.23, 0.23]) @property def logging(self): @@ -198,12 +199,19 @@ class ConfigUI(object): """Set logging level. Command for logging dropdown selection. :param str level: The selected logging level, e.g. ``debug``. """ - self.base.set_logging(level.lower()) + self.base.set_logging(self.logging) + + def set_threads(self, threads): + """Set number of threads. OnChange command for threads field. + :param int threads: The selected number of threads. + """ + self.base.set_threads(int(threads)) def set_authenticate(self, auth): """Set label of authentication button depending on auth status. :param bool auth: Whether plug-in is authenticated. """ + self.status = auth if auth: maya.button( self._authenticate, edit=True, label="Refresh Authentication") diff --git a/azure_batch_maya/scripts/ui/ui_environment.py b/azure_batch_maya/scripts/ui/ui_environment.py index 519a7ff..dbd6b90 100644 --- a/azure_batch_maya/scripts/ui/ui_environment.py +++ b/azure_batch_maya/scripts/ui/ui_environment.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os import utils diff --git a/azure_batch_maya/scripts/ui/ui_history.py b/azure_batch_maya/scripts/ui/ui_jobhistory.py similarity index 93% rename from azure_batch_maya/scripts/ui/ui_history.py rename to azure_batch_maya/scripts/ui/ui_jobhistory.py index aad6a41..b01ddd8 100644 --- a/azure_batch_maya/scripts/ui/ui_history.py +++ b/azure_batch_maya/scripts/ui/ui_jobhistory.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os import webbrowser @@ -33,14 +10,14 @@ from api import MayaAPI as maya import utils -class HistoryUI(object): +class JobHistoryUI(object): """Class to create the 'Jobs' tab in the plug-in UI""" def __init__(self, base, frame): """Create 'Jobs' tab and add to UI frame. :param base: The base class for handling jobs monitoring functionality. - :type base: :class:`.AzureBatchHistory` + :type base: :class:`.AzureBatchJobHistory` :param frame: The shared plug-in UI frame. :type frame: :class:`.AzureBatchUI` """ @@ -260,7 +237,7 @@ class AzureBatchJobInfo(object): """Create a new job reference. :param base: The base class for handling jobs monitoring functionality. - :type base: :class:`.AzureBatchHistory` + :type base: :class:`.AzureBatchJobHistory` :param int index: The index of where this reference is displayed on the current page. :param layout: The layout on which the job details will be displayed. @@ -308,7 +285,7 @@ class AzureBatchJobInfo(object): """Set the label for progress complete. :param int value: The percent complete. """ - maya.text(self._progress, edit=True, label=" {0}%".format(value)) + maya.text(self._progress, edit=True, label=" {0}".format(value)) def set_submission(self, value): """Set the label for date/time submitted. @@ -328,10 +305,11 @@ class AzureBatchJobInfo(object): def set_job(self, value): """Set the label for the job ID, and format the portal URL reference for the job with this ID. + Except that with the current portal we have no way to directly link to a job. :param str value: The job ID. """ maya.text_field(self._job, edit=True, text=value) - self.url = "https://ms.portal.azure.com" + self.url = "https://portal.azure.com" def set_pool(self, value): """Set the label for the pool ID that this job ran on. @@ -411,6 +389,7 @@ class AzureBatchJobInfo(object): (self.cancel_button, 'right', 5, 80), (self.delete_button, 'left', 5, 80)]) self.base.job_selected(self) + maya.execute(self.base.load_tasks) maya.execute(self.base.get_thumbnail) maya.refresh() @@ -439,6 +418,7 @@ class AzureBatchJobInfo(object): def refresh(self): """Refresh the details of the specified job, and update the UI.""" self.base.update_job(self.index) + maya.execute(self.base.load_tasks) maya.execute(self.base.get_thumbnail) self.selected_dir = utils.get_default_output_path() maya.text_field(self._dir, edit=True, text=self.selected_dir) diff --git a/azure_batch_maya/scripts/ui/ui_pools.py b/azure_batch_maya/scripts/ui/ui_pools.py index 35bb155..86fe04d 100644 --- a/azure_batch_maya/scripts/ui/ui_pools.py +++ b/azure_batch_maya/scripts/ui/ui_pools.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os @@ -170,17 +147,19 @@ class AzureBatchPoolInfo(object): """ maya.text(self._type, edit=True, label=" {0}".format(value)) - def set_size(self, value): - """Set the number of instances in the pool. + def set_dedicated_size(self, pool): + """Set the number of instances in the pool, both current and target. :param int value: Size of the pool. """ - maya.text(self._size, edit=True, label=" {0}".format(value)) + maya.text(self._dedicated_size, edit=True, label=" target: {} current: {}".format( + pool.target_dedicated_nodes, pool.current_dedicated_nodes)) - def set_target(self, value): - """Set the target number of instances in the pool. - :param int value: The target size of the pool. + def set_low_pri_size(self, pool): + """Set the number of instances in the pool, both current and target. + :param int value: Size of the pool. """ - maya.text(self._target, edit=True, label=" {0}".format(value)) + maya.text(self._low_pri_size, edit=True, label=" target: {} current: {}".format( + pool.target_low_priority_nodes, pool.current_low_priority_nodes)) def set_created(self, value): """Set the date/time the pool was created. @@ -205,11 +184,11 @@ class AzureBatchPoolInfo(object): value += "{} nodes {} ".format(node_states[state], state.value) maya.text(self._state, edit=True, label=" {0}".format(value)) - def set_tasks(self, value): - """Set the tasks per TVM allowed in the pool. - :param int value: Tasks per TVM. + def set_id(self, value): + """Set the pool ID field. + :param str value: Pool ID. """ - maya.text(self._tasks, edit=True, label=" {0}".format(value)) + maya.text_field(self._id, edit=True, text=value) def set_allocation(self, value): """Set the allocation state of the pool. @@ -240,12 +219,12 @@ class AzureBatchPoolInfo(object): """Command for the expanding of the pool reference frame layout. Loads latest details for the specified pool and populates UI. """ + self._id = self.display_data("ID: ") self._type = self.display_info("Type: ") - self._size = self.display_info("Current Size: ") - self._target = self.display_info("Target Size: ") + self._dedicated_size = self.display_info("Dedicated VMs: ") + self._low_pri_size = self.display_info("Low Priority VMs: ") self._created = self.display_info("Created: ") self._state = self.display_info("State: ") - self._tasks = self.display_info("Tasks per VM: ") self._image = self.display_info("Image: ") self._allocation = self.display_info("Allocation State: ") self._licenses = self.display_info("Licenses: ") @@ -253,21 +232,40 @@ class AzureBatchPoolInfo(object): self.base.pool_selected(self) auto = self.base.is_auto_pool() if not auto: + self.content.append(maya.col_layout( + numberOfColumns=5, + columnWidth=((1, 80), (2, 100), (3, 45), (4, 80), (5, 45)), + rowSpacing=(1, 10), + parent=self.layout)) self.resize_button = utils.ProcButton( - "Resize Pool", "Resizing...", self.resize_pool, - parent=self.listbox, align="center") - self.resize_int = maya.int_slider( - value=self.base.get_pool_size(), + "Resize Pool", + "Resizing...", + self.resize_pool, + parent=self.content[-1], + align="center") + self.dedicated_label = maya.text( + label="Dedicated VMs", + parent=self.content[-1]) + self.resize_dedicated = maya.int_field( + value=self.base.get_pool_size()[0], minValue=0, maxValue=1000, - fieldMinValue=0, - fieldMaxValue=100, - field=True, - width=230, - parent=self.listbox, - annotation="Number of instances to work in pool.") + parent=self.content[-1], + annotation="Number of dedicated VMs in pool.") + self.low_pri_label = maya.text( + label="Low-pri VMs", + parent=self.content[-1]) + self.resize_low_pri = maya.int_field( + value=self.base.get_pool_size()[1], + minValue=0, + maxValue=1000, + parent=self.content[-1], + annotation="Number of Low-priority VMs in pool.") self.content.append(self.resize_button.display) - self.content.append(self.resize_int) + self.content.append(self.dedicated_label) + self.content.append(self.resize_dedicated) + self.content.append(self.low_pri_label) + self.content.append(self.resize_low_pri) self.delete_button = utils.ProcButton("Delete Pool", "Deleting...", self.delete_pool, parent=self.layout, align="center") self.content.append(self.delete_button.display) @@ -305,6 +303,16 @@ class AzureBatchPoolInfo(object): self.content.append(input) return input + def display_data(self, label): + """Display text data as a non-editable text field with heading. + :param str label: The text for the data heading. + """ + self.content.append( + maya.text(label=label, parent=self.listbox, align="right")) + input = maya.text_field(text="", parent=self.listbox, editable=False) + self.content.append(input) + return input + def delete_pool(self, *args): """Delete the specified pool.""" self.delete_button.start() @@ -314,7 +322,8 @@ class AzureBatchPoolInfo(object): def resize_pool(self, *args): """Resize the specified pool.""" self.resize_button.start() - resize = maya.int_slider(self.resize_int, query=True, value=True) - self.base.resize_pool(resize) + resize_dedicated = maya.int_field(self.resize_dedicated, query=True, value=True) + resize_low_pri = maya.int_field(self.resize_low_pri, query=True, value=True) + self.base.resize_pool(resize_dedicated, resize_low_pri) self.base.update_pool(self.index) self.resize_button.finish() diff --git a/azure_batch_maya/scripts/ui/ui_shared.py b/azure_batch_maya/scripts/ui/ui_shared.py index f6c2af7..9e3edcd 100644 --- a/azure_batch_maya/scripts/ui/ui_shared.py +++ b/azure_batch_maya/scripts/ui/ui_shared.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os diff --git a/azure_batch_maya/scripts/ui/ui_submission.py b/azure_batch_maya/scripts/ui/ui_submission.py index 2e47836..91034a3 100644 --- a/azure_batch_maya/scripts/ui/ui_submission.py +++ b/azure_batch_maya/scripts/ui/ui_submission.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import os import utils @@ -35,6 +12,11 @@ from api import MayaAPI as maya class SubmissionUI(object): """Class to create the 'Submit' tab in the plug-in UI""" + AUTO_POOL = 1 + EXISTING_POOL = 2 + NEW_POOL = 3 + + def __init__(self, base, frame): """Create 'Submit' tab and add to UI frame. @@ -46,38 +28,60 @@ class SubmissionUI(object): self.base = base self.label = "Submit" self.page = maya.form_layout(enableBackground=True) - self.select_pool_type = 1 - self.select_instances = 1 + self.select_pool_type = self.AUTO_POOL + self.select_dedicated_instances = 1 + self.select_low_pri_instances = 0 with utils.ScrollLayout(height=475, parent=self.page) as scroll: box_label = "Pool Settings" - with utils.FrameLayout(label=box_label, collapsable=True): - self.pool_settings = maya.col_layout( + with utils.FrameLayout(label=box_label, collapsable=True) as pool_settings: + self.pool_settings = pool_settings + maya.col_layout( numberOfColumns=2, columnWidth=((1, 100), (2, 200)), rowSpacing=(1, 10), - rowOffset=((1, "top", 20), (2, "bottom", 20))) + rowOffset=((1, "top", 20),)) maya.text(label="Pools: ", align="right") maya.radio_group( labelArray3=("Auto provision a pool for this job", - "Reuse an existing persistent pool", - "Create a new persistent pool"), + "Reuse an existing persistent pool", + "Create a new persistent pool"), numberOfRadioButtons=3, select=self.select_pool_type, vertical=True, onCommand1=self.set_pool_auto, onCommand2=self.set_pool_reuse, onCommand3=self.set_pool_new) - self.pool_text = maya.text( - label="Instances: ", align="right") - self.control = maya.int_slider( - field=True, value=self.select_instances, + maya.parent() + self.pool_config = [] + self.pool_config.append(maya.col_layout( + numberOfColumns=4, + columnWidth=((1, 100), (2, 50), (3, 100), (4, 50)), + rowSpacing=(1, 10), + rowOffset=((1, "bottom", 20),), + parent=self.pool_settings)) + self.pool_config.append(maya.text( + label="Dedicated VMs: ", + align="right", + parent=self.pool_config[0])) + self.pool_config.append(maya.int_field( + value=self.select_dedicated_instances, minValue=1, - maxValue=1000, - fieldMinValue=1, - fieldMaxValue=1000, - changeCommand=self.set_pool_instances, - annotation="Number of instances in pool") + maxValue=self.base.max_pool_size, + changeCommand=self.set_dedicated_instances, + annotation="Number of dedicated VMs in pool", + parent=self.pool_config[0])) + self.pool_config.append(maya.text( + label="Low-pri VMs: ", + align="right", + parent=self.pool_config[0])) + self.pool_config.append(maya.int_field( + value=self.select_low_pri_instances, + minValue=0, + maxValue=self.base.max_pool_size, + changeCommand=self.set_low_pri_instances, + annotation="Number of low-priority VMs in pool", + parent=self.pool_config[0])) maya.parent() box_label = "Render Settings" @@ -207,55 +211,97 @@ class SubmissionUI(object): :returns: A dictionary with selected pool type as key and pool specification as value. """ - if self.select_pool_type == 2: - details = str(maya.menu(self.control, query=True, value=True)) + if self.select_pool_type == self.EXISTING_POOL: + details = str(maya.menu(self.pool_config[-1], query=True, value=True)) else: - details = self.select_instances + details = (self.select_dedicated_instances, self.select_low_pri_instances) return {self.select_pool_type: details} - def set_pool_instances(self, instances): + def set_dedicated_instances(self, instances): """Update the number of requested instances in a pool based on the instance slider. """ - self.select_instances = instances + self.select_dedicated_instances = instances + + def set_low_pri_instances(self, instances): + """Update the number of requested instances in a pool + based on the instance slider. + """ + self.select_low_pri_instances = instances def set_pool_new(self, *args): """Set selected pool type to be new pool of given size. Displays the pool size UI control. Command for select_pool_type radio buttons. """ - self.select_pool_type = 3 - maya.delete_ui(self.control) - maya.text(self.pool_text, edit=True, label="Instances: ") - self.control = maya.int_slider( - field=True, - value=self.select_instances, + self.select_pool_type = self.NEW_POOL + maya.delete_ui(self.pool_config) + self.pool_config = [] + self.pool_config.append(maya.col_layout( + numberOfColumns=4, + columnWidth=((1, 100), (2, 50), (3, 100), (4, 50)), + rowSpacing=(1, 10), + rowOffset=((1, "bottom", 20),), + parent=self.pool_settings)) + self.pool_config.append(maya.text( + label="Dedicated VMs: ", + align="right", + parent=self.pool_config[0])) + self.pool_config.append(maya.int_field( + value=self.select_dedicated_instances, minValue=1, - maxValue=1000, - fieldMinValue=1, - fieldMaxValue=1000, - parent=self.pool_settings, - changeCommand=self.set_pool_instances, - annotation="Number of instances in pool") + maxValue=self.base.max_pool_size, + changeCommand=self.set_dedicated_instances, + annotation="Number of dedicated VMs in pool", + parent=self.pool_config[0])) + self.pool_config.append(maya.text( + label="Low-pri VMs: ", + align="right", + parent=self.pool_config[0])) + self.pool_config.append(maya.int_field( + value=self.select_low_pri_instances, + minValue=0, + maxValue=self.base.max_pool_size, + changeCommand=self.set_low_pri_instances, + annotation="Number of low-priority VMs in pool", + parent=self.pool_config[0])) def set_pool_auto(self, *args): """Set selected pool type to be new pool of given size. Displays the pool size UI control. Command for select_pool_type radio buttons. """ - self.select_pool_type = 1 - maya.delete_ui(self.control) - maya.text(self.pool_text, edit=True, label="Instances: ") - self.control = maya.int_slider( - field=True, - value=self.select_instances, + self.select_pool_type = self.AUTO_POOL + maya.delete_ui(self.pool_config) + self.pool_config = [] + self.pool_config.append(maya.col_layout( + numberOfColumns=4, + columnWidth=((1, 100), (2, 50), (3, 100), (4, 50)), + rowSpacing=(1, 10), + rowOffset=((1, "bottom", 20),), + parent=self.pool_settings)) + self.pool_config.append(maya.text( + label="Dedicated VMs: ", + align="right", + parent=self.pool_config[0])) + self.pool_config.append(maya.int_field( + value=self.select_dedicated_instances, minValue=1, - maxValue=1000, - fieldMinValue=1, - fieldMaxValue=1000, - parent=self.pool_settings, - changeCommand=self.set_pool_instances, - annotation="Number of instances in pool") + maxValue=self.base.max_pool_size, + changeCommand=self.set_dedicated_instances, + annotation="Number of dedicated VMs in pool", + parent=self.pool_config[0])) + self.pool_config.append(maya.text( + label="Low-pri VMs: ", + align="right", + parent=self.pool_config[0])) + self.pool_config.append(maya.int_field( + value=self.select_low_pri_instances, + minValue=0, + maxValue=self.base.max_pool_size, + changeCommand=self.set_low_pri_instances, + annotation="Number of low-priority VMs in pool", + parent=self.pool_config[0])) def set_pool_reuse(self, *args): """Set selected pool type to be an existing pool with given ID. @@ -263,19 +309,25 @@ class SubmissionUI(object): in a dropdown menu. Command for select_pool_type radio buttons. """ - self.select_pool_type = 2 - maya.delete_ui(self.control) - maya.text(self.pool_text, edit=True, label="loading...") + self.select_pool_type = self.EXISTING_POOL + maya.delete_ui(self.pool_config) + self.pool_config = [] + self.pool_config.append(maya.col_layout( + numberOfColumns=2, + columnWidth=((1, 100), (2, 200)), + rowSpacing=(1, 10), + rowOffset=((1, "bottom", 20),), + parent=self.pool_settings)) + self.pool_config.append(maya.text( + label="loading...", + align="right", + parent=self.pool_config[0])) maya.refresh() pool_options = self.base.available_pools() - maya.text(self.pool_text, edit=True, label="Pool ID: ") - self.control = maya.menu( - parent=self.pool_settings, - annotation="Use an existing persistent pool ID") + maya.text(self.pool_config[-1], edit=True, label="Pool ID: ") + self.pool_config.append(maya.menu( + annotation="Use an existing persistent pool ID", + parent=self.pool_config[0])) for pool_id in pool_options: maya.menu_option(pool_id) - #with utils.Dropdown(None, parent=self.pool_settings, annotation="Use an existing persistent pool ID") as pools: - # self.control = pools - # for pool_id in pool_options: - # self.control.add_item(pool_id) - + \ No newline at end of file diff --git a/azure_batch_maya/scripts/utils.py b/azure_batch_maya/scripts/utils.py index 4eae703..e99c759 100644 --- a/azure_batch_maya/scripts/utils.py +++ b/azure_batch_maya/scripts/utils.py @@ -1,37 +1,16 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- -from api import MayaAPI as maya +from enum import Enum import os import logging import platform import pathlib +from api import MayaAPI as maya + from batch_extensions import _file_utils as file_utils from exception import CancellationException, FileUploadException @@ -67,7 +46,7 @@ def get_remote_file_path(assetpath): """ def generate_path(os_flavor, fullpath=assetpath): local_sep = os.sep - remote_sep = '\\' if os_flavor == 'Windows' else '/' + remote_sep = '\\' if os_flavor == OperatingSystem.windows else '/' path = shorten_path(*os.path.split(fullpath)) if ':' in path: drive_letter, path = path.split(':', 1) @@ -82,7 +61,7 @@ def get_remote_directory(dir_path, os_flavor): path according to the remote OS. """ local_sep = os.sep - remote_sep = '\\' if os_flavor == 'Windows' else '/' + remote_sep = '\\' if os_flavor == OperatingSystem.windows else '/' if ':' in dir_path: drive_letter, dir_path = dir_path.split(':', 1) dir_path = drive_letter + local_sep + dir_path[1:] @@ -95,7 +74,7 @@ def format_scene_path(scene_file, os_flavor): be on the render node. """ scene_path = get_remote_file_path(scene_file)(os_flavor) - if os_flavor == 'Windows': + if os_flavor == OperatingSystem.windows: return "X:\\\\" + scene_path + '\\\\' + os.path.basename(scene_file) else: return "/X/" + scene_path + '/' + os.path.basename(scene_file) @@ -119,6 +98,12 @@ def get_os(): return platform.system() +class OperatingSystem(Enum): + windows = 'Windows' + linux = 'Linux' + darwin = 'Darwin' + + class Row(object): """UI row class.""" @@ -177,7 +162,6 @@ class Layout(object): return self.layout def __exit__(self, type, value, traceback): - #TODO: Exception handling should go in here maya.parent() @@ -447,12 +431,12 @@ class JobWatcher(object): self.job_watcher = os.path.join( os.path.dirname(__file__), "tools", "job_watcher.py") platform = get_os() - if platform == "Windows": + if platform == OperatingSystem.windows.value: self.proc_cmd = 'system("WMIC PROCESS where (Name=\'mayapy.exe\') get Commandline")' self.start_cmd = 'system("start mayapy {0}")' self.quotes = '\\"' self.splitter = 'mayapy' - elif platform == "Darwin": + elif platform == OperatingSystem.darwin.value: self.proc_cmd = 'system("ps -ef")' self.start_cmd = 'system("osascript -e \'tell application \\"Terminal\\" to do script \\"python {0}\\"\'")' self.quotes = '\\\\\\"' diff --git a/azure_batch_maya/templates/arnold-basic-linux.json b/azure_batch_maya/templates/arnold-basic-linux.json index 7df5e19..f2525ff 100644 --- a/azure_batch_maya/templates/arnold-basic-linux.json +++ b/azure_batch_maya/templates/arnold-basic-linux.json @@ -1,6 +1,6 @@ { "templateMetadata": { - "description": "Sample application template for working with Blender." + "description": "Application template for working with Maya and Arnold on CentOS." }, "parameters": { "sceneFile": { @@ -11,17 +11,12 @@ }, "renderer": { "type": "string", - "defaultValue": "file", + "defaultValue": "arnold", "metadata": { "description": "The Maya renderer to be used for the render" }, "allowedValues": [ - "arnold", - "default", - "sw", - "turtlebake", - "turtle", - "vr" + "arnold" ] }, "projectData": { @@ -71,6 +66,13 @@ "metadata": { "description": "The file group where outputs will be stored" } + }, + "logLevel": { + "type": "int", + "defaultValue": 1, + "metadata": { + "description": "Arnold logging verbosity" + } } }, "jobPreparationTask": { @@ -113,7 +115,7 @@ "elevationLevel": "admin" } }, - "commandLine": "sudo mkdir -m a=rwx -p \"/X\";sudo mount --rbind $AZ_BATCH_JOB_PREP_WORKING_DIR/assets /X;Render -renderer [parameters('renderer')] -proj \"$AZ_BATCH_JOB_PREP_WORKING_DIR\" -verb -preRender renderPrep -rd \"$AZ_BATCH_TASK_WORKING_DIR/images\" -s {0} -e {0} \"[parameters('sceneFile')]\";err=$?;python /mnt/resource/batch/tasks/workitems/[parameters('outputs')]/job-1/jobpreparation/wd/thumbnail.py $err;sudo umount \"/X\";exit $err", + "commandLine": "sudo mkdir -m a=rwx -p \"/X\";sudo mount --rbind $AZ_BATCH_JOB_PREP_WORKING_DIR/assets /X;Render -renderer [parameters('renderer')] -proj \"$AZ_BATCH_JOB_PREP_WORKING_DIR\" -ai:ltc 1 -ai:lve [parameters('logLevel')] -verb -preRender renderPrep -rd \"$AZ_BATCH_TASK_WORKING_DIR/images\" -s {0} -e {0} \"[parameters('sceneFile')]\";err=$?;python /mnt/resource/batch/tasks/workitems/[parameters('outputs')]/job-1/jobpreparation/wd/thumbnail.py $err;sudo umount \"/X\";exit $err", "environmentSettings": [ { "name": "MAYA_SCRIPT_PATH", @@ -122,10 +124,6 @@ { "name": "FLEXLM_TIMEOUT", "value": "5000000" - }, - { - "name": "MAYA_RENDER_DESC_PATH", - "value": "/opt/solidangle/mtoa/2017/" } ], "outputFiles": [ diff --git a/azure_batch_maya/templates/arnold-basic-windows.json b/azure_batch_maya/templates/arnold-basic-windows.json index ed983e3..9539097 100644 --- a/azure_batch_maya/templates/arnold-basic-windows.json +++ b/azure_batch_maya/templates/arnold-basic-windows.json @@ -1,6 +1,6 @@ { "templateMetadata": { - "description": "Sample application template for working with Blender." + "description": "Application template for working with Maya and Arnold on Windows." }, "parameters": { "sceneFile": { @@ -11,17 +11,12 @@ }, "renderer": { "type": "string", - "defaultValue": "file", + "defaultValue": "arnold", "metadata": { "description": "The Maya renderer to be used for the render" }, "allowedValues": [ - "arnold", - "default", - "sw", - "turtlebake", - "turtle", - "vr" + "arnold" ] }, "projectData": { @@ -71,6 +66,13 @@ "metadata": { "description": "The file group where outputs will be stored" } + }, + "logLevel": { + "type": "int", + "defaultValue": 1, + "metadata": { + "description": "Arnold logging verbosity" + } } }, "jobPreparationTask": { @@ -107,7 +109,7 @@ ], "repeatTask": { "displayName": "Frame {0}", - "commandLine": "subst X: %AZ_BATCH_JOB_PREP_WORKING_DIR%\\assets & render -renderer [parameters('renderer')] -proj \"%AZ_BATCH_JOB_PREP_WORKING_DIR%\" -verb -preRender renderPrep -rd \"%AZ_BATCH_TASK_WORKING_DIR%\\images\" -s {0} -e {0} \"[parameters('sceneFile')]\" & call mayapy %AZ_BATCH_JOB_PREP_WORKING_DIR%\\thumbnail.py %^errorlevel%", + "commandLine": "subst X: %AZ_BATCH_JOB_PREP_WORKING_DIR%\\assets & render -renderer [parameters('renderer')] -proj \"%AZ_BATCH_JOB_PREP_WORKING_DIR%\" -ai:ltc 1 -ai:lve [parameters('logLevel')] -verb -preRender renderPrep -rd \"%AZ_BATCH_TASK_WORKING_DIR%\\images\" -s {0} -e {0} \"[parameters('sceneFile')]\" & call mayapy %AZ_BATCH_JOB_PREP_WORKING_DIR%\\thumbnail.py %^errorlevel%", "environmentSettings": [ { "name": "MAYA_SCRIPT_PATH", diff --git a/azure_batch_maya/templates/mayaSoftware-basic-linux.json b/azure_batch_maya/templates/mayaSoftware-basic-linux.json new file mode 100644 index 0000000..f6b8787 --- /dev/null +++ b/azure_batch_maya/templates/mayaSoftware-basic-linux.json @@ -0,0 +1,174 @@ +{ + "templateMetadata": { + "description": "Application template for working with Maya on CentOS." + }, + "parameters": { + "sceneFile": { + "type": "string", + "metadata": { + "description": "The Maya scene file to be rendered" + } + }, + "renderer": { + "type": "string", + "defaultValue": "sw", + "metadata": { + "description": "The Maya renderer to be used for the render" + }, + "allowedValues": [ + "sw" + ] + }, + "projectData": { + "type": "string", + "metadata": { + "description": "The file group where the input data is stored" + } + }, + "assetScript": { + "type": "string", + "metadata": { + "description": "The SAS URL to a pre-render asset path redirection script" + } + }, + "thumbScript": { + "type": "string", + "metadata": { + "description": "The SAS URL to the thumbnail generation script" + } + }, + "frameStart": { + "type": "int", + "metadata": { + "description": "Index of the first frame to render" + } + }, + "workspace": { + "type": "string", + "metadata": { + "description": "The SAS URL to the project workspace" + } + }, + "frameStep": { + "type": "int", + "metadata": { + "description": "Incremental step in frame sequeunce" + } + }, + "frameEnd": { + "type": "int", + "metadata": { + "description": "Index of the last frame to render" + } + }, + "outputs": { + "type": "string", + "metadata": { + "description": "The file group where outputs will be stored" + } + } + }, + "jobPreparationTask": { + "resourceFiles": [ + { + "source": { + "fileGroup": "[parameters('projectData')]" + }, + "filePath": "assets/" + }, + { + "blobSource": "[parameters('assetScript')]", + "filePath": "scripts/renderPrep.mel" + }, + { + "blobSource": "[parameters('thumbScript')]", + "filePath": "thumbnail.py" + }, + { + "blobSource": "[parameters('workspace')]", + "filePath": "workspace.mel" + } + ], + "commandLine": "dir" + }, + "taskFactory": { + "type": "parametricSweep", + "parameterSets": [ + { + "start": "[parameters('frameStart')]", + "end": "[parameters('frameEnd')]", + "step": "[parameters('frameStep')]" + } + ], + "repeatTask": { + "displayName": "Frame {0}", + "userIdentity": { + "autoUser": { + "scope": "task", + "elevationLevel": "admin" + } + }, + "commandLine": "sudo mkdir -m a=rwx -p \"/X\";sudo mount --rbind $AZ_BATCH_JOB_PREP_WORKING_DIR/assets /X;Render -renderer [parameters('renderer')] -proj \"$AZ_BATCH_JOB_PREP_WORKING_DIR\" -verb -preRender renderPrep -rd \"$AZ_BATCH_TASK_WORKING_DIR/images\" -s {0} -e {0} \"[parameters('sceneFile')]\";err=$?;python /mnt/resource/batch/tasks/workitems/[parameters('outputs')]/job-1/jobpreparation/wd/thumbnail.py $err;sudo umount \"/X\";exit $err", + "environmentSettings": [ + { + "name": "MAYA_SCRIPT_PATH", + "value": "/mnt/resource/batch/tasks/workitems/[parameters('outputs')]/job-1/jobpreparation/wd/scripts" + }, + { + "name": "FLEXLM_TIMEOUT", + "value": "5000000" + } + ], + "outputFiles": [ + { + "filePattern": "images/**/*", + "destination": { + "autoStorage": { + "fileGroup": "[parameters('outputs')]" + } + }, + "uploadOptions": { + "uploadCondition": "taskSuccess" + } + }, + { + "filePattern": "thumbs/*.png", + "destination": { + "autoStorage": { + "fileGroup": "[parameters('outputs')]", + "path": "thumbs" + } + }, + "uploadOptions": { + "uploadCondition": "taskSuccess" + } + }, + { + "filePattern": "../stdout.txt", + "destination": { + "autoStorage": { + "fileGroup": "[parameters('outputs')]", + "path": "logs/frame_{0}.log" + } + }, + "uploadOptions": { + "uploadCondition": "taskCompletion" + } + }, + { + "filePattern": "../stderr.txt", + "destination": { + "autoStorage": { + "fileGroup": "[parameters('outputs')]", + "path": "logs/frame_{0}_error.log" + } + }, + "uploadOptions": { + "uploadCondition": "taskCompletion" + } + } + ] + } + }, + "onAllTasksComplete": "terminateJob" +} diff --git a/azure_batch_maya/templates/mayaSoftware-basic-windows.json b/azure_batch_maya/templates/mayaSoftware-basic-windows.json new file mode 100644 index 0000000..6f2be91 --- /dev/null +++ b/azure_batch_maya/templates/mayaSoftware-basic-windows.json @@ -0,0 +1,168 @@ +{ + "templateMetadata": { + "description": "Application template for working with Maya on Windows." + }, + "parameters": { + "sceneFile": { + "type": "string", + "metadata": { + "description": "The Maya scene file to be rendered" + } + }, + "renderer": { + "type": "string", + "defaultValue": "sw", + "metadata": { + "description": "The Maya renderer to be used for the render" + }, + "allowedValues": [ + "sw" + ] + }, + "projectData": { + "type": "string", + "metadata": { + "description": "The file group where the input data is stored" + } + }, + "assetScript": { + "type": "string", + "metadata": { + "description": "The SAS URL to a pre-render asset path redirection script" + } + }, + "thumbScript": { + "type": "string", + "metadata": { + "description": "The SAS URL to the thumbnail generation script" + } + }, + "workspace": { + "type": "string", + "metadata": { + "description": "The SAS URL to the project workspace" + } + }, + "frameStart": { + "type": "int", + "metadata": { + "description": "Index of the first frame to render" + } + }, + "frameStep": { + "type": "int", + "metadata": { + "description": "Incremental step in frame sequeunce" + } + }, + "frameEnd": { + "type": "int", + "metadata": { + "description": "Index of the last frame to render" + } + }, + "outputs": { + "type": "string", + "metadata": { + "description": "The file group where outputs will be stored" + } + } + }, + "jobPreparationTask": { + "resourceFiles": [ + { + "source": { + "fileGroup": "[parameters('projectData')]" + }, + "filePath": "assets\\" + }, + { + "blobSource": "[parameters('assetScript')]", + "filePath": "scripts\\renderPrep.mel" + }, + { + "blobSource": "[parameters('thumbScript')]", + "filePath": "thumbnail.py" + }, + { + "blobSource": "[parameters('workspace')]", + "filePath": "workspace.mel" + } + ], + "commandLine": "dir" + }, + "taskFactory": { + "type": "parametricSweep", + "parameterSets": [ + { + "start": "[parameters('frameStart')]", + "end": "[parameters('frameEnd')]", + "step": "[parameters('frameStep')]" + } + ], + "repeatTask": { + "displayName": "Frame {0}", + "commandLine": "subst X: %AZ_BATCH_JOB_PREP_WORKING_DIR%\\assets & render -renderer [parameters('renderer')] -proj \"%AZ_BATCH_JOB_PREP_WORKING_DIR%\" -verb -preRender renderPrep -rd \"%AZ_BATCH_TASK_WORKING_DIR%\\images\" -s {0} -e {0} \"[parameters('sceneFile')]\" & call mayapy %AZ_BATCH_JOB_PREP_WORKING_DIR%\\thumbnail.py %^errorlevel%", + "environmentSettings": [ + { + "name": "MAYA_SCRIPT_PATH", + "value": "%AZ_BATCH_JOB_PREP_WORKING_DIR%\\scripts" + }, + { + "name": "FLEXLM_TIMEOUT", + "value": "5000000" + } + ], + "outputFiles": [ + { + "filePattern": "images/**/*", + "destination": { + "autoStorage": { + "fileGroup": "[parameters('outputs')]" + } + }, + "uploadOptions": { + "uploadCondition": "taskSuccess" + } + }, + { + "filePattern": "thumbs/*.png", + "destination": { + "autoStorage": { + "fileGroup": "[parameters('outputs')]", + "path": "thumbs" + } + }, + "uploadOptions": { + "uploadCondition": "taskSuccess" + } + }, + { + "filePattern": "../stdout.txt", + "destination": { + "autoStorage": { + "fileGroup": "[parameters('outputs')]", + "path": "logs/frame_{0}.log" + } + }, + "uploadOptions": { + "uploadCondition": "taskCompletion" + } + }, + { + "filePattern": "../stderr.txt", + "destination": { + "autoStorage": { + "fileGroup": "[parameters('outputs')]", + "path": "logs/frame_{0}_error.log" + } + }, + "uploadOptions": { + "uploadCondition": "taskCompletion" + } + } + ] + } + }, + "onAllTasksComplete": "terminateJob" +} diff --git a/docs/submitting_jobs.md b/docs/submitting_jobs.md index 7cbae19..8855f51 100644 --- a/docs/submitting_jobs.md +++ b/docs/submitting_jobs.md @@ -76,7 +76,7 @@ You can also delete the pool and resize the number of VMs in the pool. A pool ca ## Monitoring jobs -Once you have submitted a job, you can monitor it both via the [Azure Management Portal](http://ms.portal.azure.com/) and the `Jobs` tab of the plug-in. +Once you have submitted a job, you can monitor it both via the [Azure Management Portal](http://portal.azure.com/) and the `Jobs` tab of the plug-in. Selecting a listed job will display the current state of the job. You can also use this tab to cancel and delete jobs, as well as download the outputs and rendering logs. To download outputs, use the `Outputs` field to set the desired destination directory, and click the center button (with the gear icon) to start a background process that will watch the job and download outputs as it progresses. You can close Maya without disrupting the download. diff --git a/package.py b/package.py index 0918f81..85fec48 100644 --- a/package.py +++ b/package.py @@ -32,7 +32,7 @@ import subprocess import shutil import zipfile -VERSION = "0.9.0" +VERSION = "0.10.0" def main(): """Build Maya Plug-in package""" diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e7b8e1e..0000000 --- a/tests/__init__.py +++ /dev/null @@ -1,79 +0,0 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- - - -import sys -import os - - -if sys.version_info[:2] < (2, 7, ): - try: - import unittest2 - from unittest2 import TestLoader, TextTestRunner - - except ImportError: - print("The Batch Maya Plugin test suite requires " - "the unittest2 package to run on Python 2.6 and " - "below.\nPlease install this package to continue.") - sys.exit() -else: - import unittest - from unittest import TestLoader, TextTestRunner - -if sys.version_info[:2] >= (3, 3, ): - from unittest import mock -else: - try: - import mock - except ImportError: - print("The Batch Maya Plugin test suite requires " - "the mock package to run on Python 3.2 and below.\n" - "Please install this package to continue.") - raise - - -if __name__ == '__main__': - - runner = TextTestRunner(verbosity=2) - - test_dir = os.path.dirname(__file__) - top_dir = os.path.dirname(test_dir) - src_dir = os.path.join(top_dir, 'azure_batch_maya', 'scripts') - mod_dir = os.path.join(test_dir, 'data', 'modules') - ui_dir = os.path.join(src_dir, 'ui') - tools_dir = os.path.join(src_dir, 'tools') - os.environ["AZUREBATCH_ICONS"] = os.path.join(top_dir, 'azure_batch_maya', 'icons') - os.environ["AZUREBATCH_MODULES"] = mod_dir - os.environ["AZUREBATCH_SCRIPTS"] = "{0};{1};{2}".format(src_dir, ui_dir, tools_dir) - sys.path.extend([src_dir, ui_dir, tools_dir, mod_dir]) - - test_loader = TestLoader() - suite = test_loader.discover(test_dir, - pattern="test_*.py", - top_level_dir=top_dir) - runner.run(suite) diff --git a/tests/test_assets.py b/tests/test_assets.py index 24113d1..c257900 100644 --- a/tests/test_assets.py +++ b/tests/test_assets.py @@ -1,31 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- - +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import sys import os @@ -49,6 +25,7 @@ from assets import Asset, Assets, AzureBatchAssets from exception import FileUploadException from utils import ProgressBar, ProcButton + class TestAsset(unittest.TestCase): def setUp(self): @@ -438,6 +415,7 @@ class TestAzureBatchAssets(unittest.TestCase): os.environ["AZUREBATCH_TEMPLATES"] = os.path.join(top_dir, 'azure_batch_maya', 'templates') os.environ["AZUREBATCH_MODULES"] = mod_dir os.environ["AZUREBATCH_SCRIPTS"] = "{0};{1};{2}".format(src_dir, ui_dir, tools_dir) + os.environ["AZUREBATCH_VERSION"] = "0.1" return super(TestAzureBatchAssets, self).setUp() @@ -445,7 +423,7 @@ class TestAzureBatchAssets(unittest.TestCase): @mock.patch("assets.callback") @mock.patch("assets.AssetsUI") def test_batchassets_create(self, mock_ui, mock_call, mock_collect): - assets = AzureBatchAssets("frame", "call") + assets = AzureBatchAssets(3, "frame", "call") mock_ui.assert_called_with(assets, "frame") mock_collect.assert_called_with() #mock_call.after_new.assert_called_with(assets.callback_refresh) diff --git a/tests/test_extensions.py b/tests/test_extensions.py index d6de9ce..ea61c88 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -20,7 +20,6 @@ from batch_extensions import _pool_utils as pool_utils from batch_extensions import _file_utils as file_utils - class TestBatchExtensions(unittest.TestCase): # pylint: disable=attribute-defined-outside-init,no-member,too-many-public-methods diff --git a/tests/test_jobwatcher.py b/tests/test_jobwatcher.py index 81961e3..44d6dd2 100644 --- a/tests/test_jobwatcher.py +++ b/tests/test_jobwatcher.py @@ -1,3 +1,8 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + try: import unittest2 as unittest except ImportError: diff --git a/tests/test_pools.py b/tests/test_pools.py index 0112655..8d78449 100644 --- a/tests/test_pools.py +++ b/tests/test_pools.py @@ -1,31 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- - +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import sys import os @@ -61,7 +37,7 @@ class AzureTestBatchPools(unittest.TestCase): @mock.patch("pools.PoolsUI") def test_pools_initialize(self, mock_ui): - pools = AzureBatchPools("frame", "call") + pools = AzureBatchPools(4, "frame", "call") mock_ui.assert_called_with(pools, "frame") def test_pools_configure(self): @@ -74,9 +50,15 @@ class AzureTestBatchPools(unittest.TestCase): self.mock_self.batch.pool = mock.create_autospec(batch.operations.ExtendedPoolOperations) pool1 = mock.create_autospec(models.CloudPool) pool1.id = "12345" + pool1.virtual_machine_configuration = mock.create_autospec(models.VirtualMachineConfiguration) + pool1.virtual_machine_configuration.image_reference = mock.create_autospec(models.ImageReference) + pool1.virtual_machine_configuration.image_reference.publisher = "MicrosoftWindows" pool1.creation_time = datetime.datetime.now() pool2 = mock.create_autospec(models.CloudPool) pool2.id = "67890" + pool2.virtual_machine_configuration = mock.create_autospec(models.VirtualMachineConfiguration) + pool2.virtual_machine_configuration.image_reference = mock.create_autospec(models.ImageReference) + pool2.virtual_machine_configuration.image_reference.publisher = "LinuxUbuntu" pool2.creation_time = datetime.datetime.now() self.mock_self._call = lambda x: [pool1, pool2] @@ -85,7 +67,9 @@ class AzureTestBatchPools(unittest.TestCase): self.assertEqual(len(self.mock_self.pools), 0) pool1.id = "Maya_Pool_A" + pool1.virtual_machine_configuration.image_reference.publisher = "batch" pool2.id = "Maya_Auto_Pool_B" + pool2.virtual_machine_configuration.image_reference.publisher = "batch" ids = AzureBatchPools.list_pools(self.mock_self) self.assertEqual(ids, ["Maya_Pool_A"]) self.assertEqual(len(self.mock_self.pools), 2) diff --git a/tests/test_submission.py b/tests/test_submission.py index e49537c..2ce9d10 100644 --- a/tests/test_submission.py +++ b/tests/test_submission.py @@ -1,30 +1,7 @@ -#------------------------------------------------------------------------- -# -# Azure Batch Maya Plugin -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- import sys import os @@ -48,6 +25,7 @@ from pools import AzureBatchPools from environment import AzureBatchEnvironment from shared import AzureBatchSettings from exception import CancellationException +from utils import OperatingSystem from batch_extensions import BatchExtensionsClient from batch_extensions.batch_auth import SharedKeyCredentials @@ -56,11 +34,11 @@ from batch_extensions import models from azure.storage.blob import BlockBlobService -LIVE = True def print_status(status): print(status) + class TestBatchSubmission(unittest.TestCase): def setUp(self): @@ -74,6 +52,7 @@ class TestBatchSubmission(unittest.TestCase): os.environ["AZUREBATCH_TEMPLATES"] = os.path.join(top_dir, 'azure_batch_maya', 'templates') os.environ["AZUREBATCH_MODULES"] = mod_dir os.environ["AZUREBATCH_SCRIPTS"] = "{0};{1};{2}".format(src_dir, ui_dir, tools_dir) + os.environ["AZUREBATCH_VERSION"] = "0.1" self.mock_self = mock.create_autospec(AzureBatchSubmission) self.mock_self.batch = mock.create_autospec(BatchExtensionsClient) self.mock_self.batch.job = mock.create_autospec(operations.ExtendedJobOperations) @@ -93,7 +72,7 @@ class TestBatchSubmission(unittest.TestCase): @mock.patch("submission.callback") @mock.patch("submission.SubmissionUI") def test_submission_create(self, mock_ui, mock_call, mock_mods): - submission = AzureBatchSubmission("frame", "call") + submission = AzureBatchSubmission(2, "frame", "call") mock_mods.assert_called_with() mock_ui.assert_called_with(submission, "frame") #mock_call.after_new.assert_called_with(mock.ANY) @@ -161,11 +140,11 @@ class TestBatchSubmission(unittest.TestCase): mock_utils.format_scene_path.return_value = "test_file_path" self.mock_self._configure_pool = lambda t: AzureBatchSubmission._configure_pool(self.mock_self, t) self.mock_self._check_plugins.return_value = [] - self.mock_self._get_os_flavor.return_value = 'Windows' + self.mock_self._get_os_flavor.return_value = OperatingSystem.windows self.mock_self.pool_manager.create_auto_pool.return_value = {'autoPool': 'auto-pool'} self.mock_self.pool_manager.create_pool.return_value = {'poolId': 'new-pool'} self.mock_self.env_manager.get_environment_settings.return_value = [{'name':'foo', 'value':'bar'}] - self.mock_self.renderer = mock.Mock() + self.mock_self.renderer = mock.Mock(render_engine='arnold') self.mock_self.renderer.get_jobdata.return_value = ("a", "b") self.mock_self.renderer.get_params.return_value = {"foo": "bar"} self.mock_self.renderer.get_title.return_value = "job name"