diff --git a/azure_batch_maya/scripts/environment.py b/azure_batch_maya/scripts/environment.py index e29913b..1e03586 100644 --- a/azure_batch_maya/scripts/environment.py +++ b/azure_batch_maya/scripts/environment.py @@ -139,6 +139,7 @@ class AzureBatchEnvironment(object): return image_reference if self.get_image_type().value == PoolImageMode.CONTAINER_IMAGE.value: image = self.ui.get_container_image_reference() + image.pop('node_sku_id') image_reference = models.ImageReference(**image) return image_reference diff --git a/azure_batch_maya/scripts/poolImageFilter.py b/azure_batch_maya/scripts/poolImageFilter.py index 7537f69..8b18ff8 100644 --- a/azure_batch_maya/scripts/poolImageFilter.py +++ b/azure_batch_maya/scripts/poolImageFilter.py @@ -12,6 +12,7 @@ import logging import sys import traceback import adal +import copy class PoolImageFilter(object): @@ -24,7 +25,7 @@ class PoolImageFilter(object): self.containerImages = poolImageProvider.getContainerImages() def getSelectedImage(self, selectedOS, selectedMaya, selectedVRay = None, selectedArnold = None): - results = self.containerImages + results = copy.deepcopy(self.containerImages) results = filterImagesByOS(results, selectedOS) diff --git a/azure_batch_maya/scripts/poolImageProvider.py b/azure_batch_maya/scripts/poolImageProvider.py index 0d3fee5..48c3d8b 100644 --- a/azure_batch_maya/scripts/poolImageProvider.py +++ b/azure_batch_maya/scripts/poolImageProvider.py @@ -28,276 +28,185 @@ MARKETPLACE_IMAGES = { }, } +CONTAINER_BASE_IMAGE_REFERENCES = { + 'centos-75-container' : + { + 'node_sku_id': 'batch.node.centos 7', + 'publisher' : 'microsoft-azure-batch', + 'offer' : 'centos-container', + 'sku' : '7-5', + 'version' : 'latest' + }, + 'ubuntu-1604lts-container' : + { + 'node_sku_id': 'batch.node.ubuntu 16.04', + 'publisher' : 'microsoft-azure-batch', + 'offer' : 'ubuntu-server-container', + 'sku' : '16-04-lts', + 'version' : 'latest' + }, + 'windowsserver-2016-container' : + { + 'node_sku_id': 'batch.node.windows amd64', + 'publisher' : 'MicrosoftWindowsServer', + 'offer' : 'WindowsServer', + 'sku' : '2016-DataCenter-With-Containers', + 'version' : 'latest' + }, +} + CONTAINER_IMAGES = { 'azurebatchrendering/centos_maya2017update5:latest': { 'OS': 'CentOS 75', 'Maya': '2017-Update5', - 'node_sku_id': 'batch.node.centos 7', - 'ImageReference' : - { - 'publisher' : 'microsoft-azure-batch', - 'offer' : 'centos-container', - 'sku' : '7-5', - 'version' : 'latest' - }, + 'ImageReference' : 'centos-75-container' }, 'azurebatchrendering/centos_maya2017update5_arnold2011:latest': { 'OS': 'CentOS 75', 'Maya': '2017-Update5', 'Arnold': '2.0.1.1', - 'node_sku_id': 'batch.node.centos 7', - 'ImageReference' : - { - 'publisher' : 'microsoft-azure-batch', - 'offer' : 'centos-container', - 'sku' : '7-5', - 'version' : 'latest' - }, + 'ImageReference' : 'centos-75-container' }, 'azurebatchrendering/centos_maya2017update5_arnold3101:latest': { 'OS': 'CentOS 75', 'Maya': '2017-Update5', 'Arnold': '3.1.0.1', - 'node_sku_id': 'batch.node.centos 7', - 'ImageReference' : - { - 'publisher' : 'microsoft-azure-batch', - 'offer' : 'centos-container', - 'sku' : '7-5', - 'version' : 'latest' - }, + 'ImageReference' : 'centos-75-container' }, 'azurebatchrendering/centos_maya2017update5_vray36004:latest': { 'OS': 'CentOS 75', 'Maya': '2017-Update5', 'VRay': '3.60.04', - 'node_sku_id': 'batch.node.centos 7', - 'ImageReference' : - { - 'publisher' : 'microsoft-azure-batch', - 'offer' : 'centos-container', - 'sku' : '7-5', - 'version' : 'latest' - }, + 'ImageReference' : 'centos-75-container' }, 'azurebatchrendering/centos_maya2018update2:latest': { 'OS': 'CentOS 75', 'Maya': '2018-Update2', - 'node_sku_id': 'batch.node.centos 7', - 'ImageReference' : - { - 'publisher' : 'microsoft-azure-batch', - 'offer' : 'centos-container', - 'sku' : '7-5', - 'version' : 'latest' - }, + 'ImageReference' : 'centos-75-container' }, 'azurebatchrendering/centos_maya2018update2_arnold2023:latest': { 'OS': 'CentOS 75', 'Maya': '2018-Update2', 'Arnold': '2.0.2.3', - 'node_sku_id': 'batch.node.centos 7', - 'ImageReference' : - { - 'publisher' : 'microsoft-azure-batch', - 'offer' : 'centos-container', - 'sku' : '7-5', - 'version' : 'latest' - }, + 'ImageReference' : 'centos-75-container' }, 'azurebatchrendering/centos_maya2018update2_arnold2103:latest': { 'OS': 'CentOS 75', 'Maya': '2018-Update2', 'Arnold': '2.1.0.3', - 'node_sku_id': 'batch.node.centos 7', - 'ImageReference' : - { - 'publisher' : 'microsoft-azure-batch', - 'offer' : 'centos-container', - 'sku' : '7-5', - 'version' : 'latest' - }, + 'ImageReference' : 'centos-75-container' }, 'azurebatchrendering/centos_maya2018update2_arnold3101:latest': { 'OS': 'CentOS 75', 'Maya': '2018-Update2', 'Arnold': '3.1.0.1', - 'node_sku_id': 'batch.node.centos 7', - 'ImageReference' : - { - 'publisher' : 'microsoft-azure-batch', - 'offer' : 'centos-container', - 'sku' : '7-5', - 'version' : 'latest' - }, + 'ImageReference' : 'centos-75-container' }, 'azurebatchrendering/centos_maya2018update2_vray36004:latest': { 'OS': 'CentOS 75', 'Maya': '2018-Update2', 'VRay': '3.60.04', - 'node_sku_id': 'batch.node.centos 7', - 'ImageReference' : - { - 'publisher' : 'microsoft-azure-batch', - 'offer' : 'centos-container', - 'sku' : '7-5', - 'version' : 'latest' - }, + 'ImageReference' : 'centos-75-container' }, 'azurebatchrendering/centos_maya2018update3:latest': { 'OS': 'CentOS 75', 'Maya': '2018-Update3', - 'node_sku_id': 'batch.node.centos 7', - 'ImageReference' : - { - 'publisher' : 'microsoft-azure-batch', - 'offer' : 'centos-container', - 'sku' : '7-5', - 'version' : 'latest' - }, + 'ImageReference' : 'centos-75-container' }, 'azurebatchrendering/centos_maya2018update3_arnold2023:latest': { 'OS': 'CentOS 75', 'Maya': '2018-Update3', 'Arnold': '2.0.2.3', - 'node_sku_id': 'batch.node.centos 7', - 'ImageReference' : - { - 'publisher' : 'microsoft-azure-batch', - 'offer' : 'centos-container', - 'sku' : '7-5', - 'version' : 'latest' - }, + 'ImageReference' : 'centos-75-container' }, 'azurebatchrendering/centos_maya2018update3_arnold2103:latest': { 'OS': 'CentOS 75', 'Maya': '2018-Update3', 'Arnold': '2.1.0.3', - 'node_sku_id': 'batch.node.centos 7', - 'ImageReference' : - { - 'publisher' : 'microsoft-azure-batch', - 'offer' : 'centos-container', - 'sku' : '7-5', - 'version' : 'latest' - }, + 'ImageReference' : 'centos-75-container' }, 'azurebatchrendering/centos_maya2018update3_arnold3101:latest': { 'OS': 'CentOS 75', 'Maya': '2018-Update3', 'Arnold': '3.1.0.1', - 'node_sku_id': 'batch.node.centos 7', - 'ImageReference' : - { - 'publisher' : 'microsoft-azure-batch', - 'offer' : 'centos-container', - 'sku' : '7-5', - 'version' : 'latest' - }, + 'ImageReference' : 'centos-75-container' }, 'azurebatchrendering/centos_maya2018update3_vray36004:latest': { 'OS': 'CentOS 75', 'Maya': '2018-Update3', 'VRay': '3.60.04', - 'node_sku_id': 'batch.node.centos 7', - 'ImageReference' : - { - 'publisher' : 'microsoft-azure-batch', - 'offer' : 'centos-container', - 'sku' : '7-5', - 'version' : 'latest' - }, + 'ImageReference' : 'centos-75-container' }, 'azurebatchrendering/centos_maya2018update4:latest': { 'OS': 'CentOS 75', 'Maya': '2018-Update4', - 'node_sku_id': 'batch.node.centos 7', - 'ImageReference' : - { - 'publisher' : 'microsoft-azure-batch', - 'offer' : 'centos-container', - 'sku' : '7-5', - 'version' : 'latest' - }, + 'ImageReference' : 'centos-75-container' }, 'azurebatchrendering/centos_maya2018update4_arnold2023:latest': { 'OS': 'CentOS 75', 'Maya': '2018-Update4', 'Arnold': '2.0.2.3', - 'node_sku_id': 'batch.node.centos 7', - 'ImageReference' : - { - 'publisher' : 'microsoft-azure-batch', - 'offer' : 'centos-container', - 'sku' : '7-5', - 'version' : 'latest' - }, + 'ImageReference' : 'centos-75-container' }, 'azurebatchrendering/centos_maya2018update4_arnold2103:latest': { 'OS': 'CentOS 75', 'Maya': '2018-Update4', 'Arnold': '2.1.0.3', - 'node_sku_id': 'batch.node.centos 7', - 'ImageReference' : - { - 'publisher' : 'microsoft-azure-batch', - 'offer' : 'centos-container', - 'sku' : '7-5', - 'version' : 'latest' - }, + 'ImageReference' : 'centos-75-container' }, 'azurebatchrendering/centos_maya2018update4_arnold3101:latest': { 'OS': 'CentOS 75', 'Maya': '2018-Update4', 'Arnold': '3.1.0.1', - 'node_sku_id': 'batch.node.centos 7', - 'ImageReference' : - { - 'publisher' : 'microsoft-azure-batch', - 'offer' : 'centos-container', - 'sku' : '7-5', - 'version' : 'latest' - }, + 'ImageReference' : 'centos-75-container' }, 'azurebatchrendering/centos_maya2018update4_vray36004:latest': { 'OS': 'CentOS 75', 'Maya': '2018-Update4', 'VRay': '3.60.04', - 'node_sku_id': 'batch.node.centos 7', - 'ImageReference' : - { - 'publisher' : 'microsoft-azure-batch', - 'offer' : 'centos-container', - 'sku' : '7-5', - 'version' : 'latest' - }, + 'ImageReference' : 'centos-75-container' }, } class PoolImageProvider(object): #TODO provide read only storage client and load entries from table storage - def __init__(self, container_images = CONTAINER_IMAGES): + def __init__(self, base_images = CONTAINER_BASE_IMAGE_REFERENCES, container_images = CONTAINER_IMAGES): + self.base_images = base_images self.container_images = container_images def getContainerImages(self): - return OrderedDict(sorted(self.container_images.items(), key=lambda t: t[0])) + merged_images = {} + + for (image_id, image_properties) in self.container_images.items(): + merged_image_properties = {} + + for (property, value) in image_properties.items(): + if property == 'ImageReference': + value = self.base_images[value] + merged_image_properties[property] = value + merged_images[image_id] = merged_image_properties + + return OrderedDict(sorted(merged_images.items(), key=lambda t: t[0])) + + def getContainerBaseImageReferences(self): + return OrderedDict(sorted(self.base_images.items(), key=lambda t: t[0])) diff --git a/azure_batch_maya/scripts/ui/ui_containerImage.py b/azure_batch_maya/scripts/ui/ui_containerImage.py index eeee6a5..77579d0 100644 --- a/azure_batch_maya/scripts/ui/ui_containerImage.py +++ b/azure_batch_maya/scripts/ui/ui_containerImage.py @@ -128,7 +128,7 @@ class ContainerImageUI(object): def selected_image_node_sku_id(self): selectedImageId, selectedImage = self.fetch_selected_image() - return selectedImage["node_sku_id"] + return selectedImage["ImageReference"]["node_sku_id"] def selected_image_container_images(self): selectedImageId, selectedImage = self.fetch_selected_image() diff --git a/tests/test_poolImages.py b/tests/test_poolImages.py index 2b4d6f4..badcfd1 100644 --- a/tests/test_poolImages.py +++ b/tests/test_poolImages.py @@ -14,33 +14,60 @@ import sys from poolImageFilter import PoolImageFilter from poolImageProvider import PoolImageProvider +CONTAINER_BASE_IMAGE_REFERENCES = { + 'centos-75-container' : + { + 'node_sku_id': 'batch.node.centos 7', + 'publisher' : 'microsoft-azure-batch', + 'offer' : 'centos-container', + 'sku' : '7-5', + 'version' : 'latest' + }, + 'ubuntu-1604lts-container' : + { + 'node_sku_id': 'batch.node.ubuntu 16.04', + 'publisher' : 'microsoft-azure-batch', + 'offer' : 'ubuntu-server-container', + 'sku' : '16-04-lts', + 'version' : 'latest' + }, + 'windowsserver-2016-container' : + { + 'node_sku_id': 'batch.node.windows amd64', + 'publisher' : 'MicrosoftWindowsServer', + 'offer' : 'WindowsServer', + 'sku' : '2016-DataCenter-With-Containers', + 'version' : 'latest' + }, +} + CONTAINER_IMAGES = { 'batchrendering/linux/maya2017:update5': { 'OS': 'CentOS 73', 'Maya': '2017-Update5', - 'ImageReference' : 'CentOS73WithContainers' + 'ImageReference' : 'centos-75-container' }, 'batchrendering/linux/mtoa-maya2017:2.0.1.1': { 'OS': 'CentOS 73', 'Maya': '2017-Update5', 'Arnold': '2.0.1.1', - 'ImageReference' : 'CentOS73WithContainers' + 'ImageReference' : 'centos-75-container' }, 'batchrendering/linux/vrayformaya-maya2017:35203': { 'OS': 'CentOS 73', 'Maya': '2017-Update5', 'VRay': '3.52.03', - 'ImageReference' : 'CentOS73WithContainers' + 'ImageReference' : 'centos-75-container' }, 'batchrendering/linux/vrayformaya-maya2017:35203': { 'OS': 'CentOS 73', 'Maya': '2017-Update5', 'VRay': '3.52.03', - 'ImageReference' : 'CentOS73WithContainers' + 'ImageReference' : 'centos-75-container' }, 'dummyCombinedVrayArnold': { @@ -48,35 +75,35 @@ CONTAINER_IMAGES = { 'Maya': '2017-Update5', 'VRay': '3.52.03', 'Arnold': '2.0.1.1', - 'ImageReference' : 'CentOS73WithContainers' + 'ImageReference' : 'centos-75-container' }, 'dummy2018-1': { 'OS': 'CentOS 73', 'Maya': '2018-Update1', - 'ImageReference' : 'CentOS73WithContainers' + 'ImageReference' : 'centos-75-container' }, 'dummy2018-2': { 'OS': 'CentOS 73', 'Maya': '2018-Update1', 'Arnold': '2.0.1.1', - 'ImageReference' : 'CentOS73WithContainers' + 'ImageReference' : 'centos-75-container' }, 'dummy2018-3': { 'OS': 'CentOS 73', 'Maya': '2018-Update1', 'VRay': '3.52.03', - 'ImageReference' : 'CentOS73WithContainers' + 'ImageReference' : 'centos-75-container' }, 'dummy2018-4': { 'OS': 'CentOS 73', 'Maya': '2018-Update1', 'VRay': '3.52.03', - 'ImageReference' : 'CentOS73WithContainers' + 'ImageReference' : 'centos-75-container' }, 'dummy2018-5': { @@ -84,7 +111,7 @@ CONTAINER_IMAGES = { 'Maya': '2018-Update1', 'VRay': '3.52.03', 'Arnold': '2.0.1.1', - 'ImageReference' : 'CentOS73WithContainers' + 'ImageReference' : 'centos-75-container' }, 'dummyExtraVersions 1': @@ -93,7 +120,7 @@ CONTAINER_IMAGES = { 'Maya': '2017-Update5', 'VRay': '1.0.0.1', 'Arnold': '1.0.0.1', - 'ImageReference' : 'CentOS73WithContainers' + 'ImageReference' : 'centos-75-container' }, 'dummyExtraVersions 2': { @@ -101,7 +128,7 @@ CONTAINER_IMAGES = { 'Maya': '2017-Update5', 'VRay': '1.0.0.1', 'Arnold': '1.0.0.2', - 'ImageReference' : 'CentOS73WithContainers' + 'ImageReference' : 'centos-75-container' }, 'dummyExtraVersions 3': { @@ -109,7 +136,7 @@ CONTAINER_IMAGES = { 'Maya': '2017-Update5', 'VRay': '1.0.0.2', 'Arnold': '1.0.0.1', - 'ImageReference' : 'CentOS73WithContainers' + 'ImageReference' : 'centos-75-container' }, 'dummyExtraVersions 4': { @@ -117,7 +144,7 @@ CONTAINER_IMAGES = { 'Maya': '2017-Update5', 'VRay': '1.0.0.4', 'Arnold': '1.0.0.1', - 'ImageReference' : 'CentOS73WithContainers' + 'ImageReference' : 'centos-75-container' }, 'dummyExtraVersions 5': { @@ -125,15 +152,23 @@ CONTAINER_IMAGES = { 'Maya': '2017-Update5', 'VRay': '1.0.0.1', 'Arnold': '1.0.0.3', - 'ImageReference' : 'CentOS73WithContainers' + 'ImageReference' : 'centos-75-container' }, - 'dummyWindows': + 'dummyWindows 2017': { 'OS': 'WindowsServer2016', 'Maya': '2017-Update5', 'VRay': '1.0.0.1', 'Arnold': '1.0.0.3', - 'ImageReference' : 'Windows2016WithContainers' + 'ImageReference' : 'windowsserver-2016-container' + }, + 'dummyWindows 2018': + { + 'OS': 'WindowsServer2016', + 'Maya': '2018-Update4', + 'VRay': '1.0.0.1', + 'Arnold': '3.0.0.1', + 'ImageReference' : 'windowsserver-2016-container' }, } @@ -142,7 +177,7 @@ CONTAINER_IMAGES = { class TestPoolImages(unittest.TestCase): def setUp(self): - self.poolImages = PoolImageFilter(PoolImageProvider(CONTAINER_IMAGES)) + self.poolImages = PoolImageFilter(PoolImageProvider(base_images = CONTAINER_BASE_IMAGE_REFERENCES, container_images=CONTAINER_IMAGES)) return super(TestPoolImages, self).setUp() #getOSDisplayList @@ -150,71 +185,71 @@ class TestPoolImages(unittest.TestCase): osDisplayList = self.poolImages.getOSDisplayList() self.assertEqual(len(osDisplayList), 2) - self.assertSetEqual(osDisplayList, set(['WindowsServer2016', 'CentOS 73'])) + self.assertListEqual(osDisplayList, ['CentOS 73', 'WindowsServer2016']) #getMayaDisplayList def testGetMayaDisplayList_Unfiltered_ReturnsCompleteSet(self): mayaDisplayList = self.poolImages.getMayaDisplayList() - self.assertEqual(len(mayaDisplayList), 2) - self.assertSetEqual(mayaDisplayList, set(['2018-Update1', '2017-Update5'])) + self.assertEqual(len(mayaDisplayList), 3) + self.assertListEqual(mayaDisplayList, ['2017-Update5', '2018-Update1', '2018-Update4']) def testGetMayaDisplayList_FilteredByOS_ReturnsFilteredSet(self): mayaDisplayList = self.poolImages.getMayaDisplayList(selectedOS = 'WindowsServer2016') - self.assertEqual(len(mayaDisplayList), 1) - self.assertSetEqual(mayaDisplayList, set(['2017-Update5'])) + self.assertEqual(len(mayaDisplayList), 2) + self.assertListEqual(mayaDisplayList, ['2017-Update5', '2018-Update4']) #getVRayDisplayList def testGetVRayDisplayList_Unfiltered_ReturnsCompleteSet(self): vrayDisplayList = self.poolImages.getVrayDisplayList() self.assertEqual(len(vrayDisplayList), 4) - self.assertSetEqual(vrayDisplayList, set(['3.52.03', '1.0.0.1', '1.0.0.2', '1.0.0.4'])) + self.assertListEqual(vrayDisplayList, ['1.0.0.1', '1.0.0.2', '1.0.0.4', '3.52.03']) def testGetVrayDisplayList_FilteredByOS_ReturnsFilteredSet(self): vrayDisplayList = self.poolImages.getVrayDisplayList(selectedOS = 'WindowsServer2016') self.assertEqual(len(vrayDisplayList), 1) - self.assertSetEqual(vrayDisplayList, set(['1.0.0.1'])) + self.assertListEqual(vrayDisplayList, ['1.0.0.1']) def testGetVrayDisplayList_FilteredByOS_AndMaya_ReturnsFilteredSet(self): vrayDisplayList = self.poolImages.getVrayDisplayList(selectedOS = 'CentOS 73', selectedMaya ='2018-Update1') self.assertEqual(len(vrayDisplayList), 1) - self.assertSetEqual(vrayDisplayList, set(['3.52.03'])) + self.assertListEqual(vrayDisplayList, ['3.52.03']) def testGetVrayDisplayList_FilteredByOS_AndMaya_AndArnold_ReturnsFilteredSet(self): vrayDisplayList = self.poolImages.getVrayDisplayList(selectedOS = 'CentOS 73', selectedMaya ='2017-Update5', selectedArnold = '1.0.0.1') self.assertEqual(len(vrayDisplayList), 3) - self.assertSetEqual(vrayDisplayList, set(['1.0.0.1', '1.0.0.2', '1.0.0.4'])) + self.assertListEqual(vrayDisplayList, ['1.0.0.1', '1.0.0.2', '1.0.0.4']) #getArnoldDisplayList def testGetArnoldDisplayList_Unfiltered_ReturnsCompleteSet(self): arnoldDisplayList = self.poolImages.getArnoldDisplayList() - self.assertEqual(len(arnoldDisplayList), 4) - self.assertSetEqual(arnoldDisplayList, set(['2.0.1.1', '1.0.0.1', '1.0.0.2', '1.0.0.3'])) + self.assertEqual(len(arnoldDisplayList), 5) + self.assertListEqual(arnoldDisplayList, ['1.0.0.1', '1.0.0.2', '1.0.0.3', '2.0.1.1', '3.0.0.1']) def testGetArnoldDisplayList_FilteredByOS_ReturnsFilteredSet(self): arnoldDisplayList = self.poolImages.getArnoldDisplayList(selectedOS = 'WindowsServer2016') - self.assertEqual(len(arnoldDisplayList), 1) - self.assertSetEqual(arnoldDisplayList, set(['1.0.0.3'])) + self.assertEqual(len(arnoldDisplayList), 2) + self.assertListEqual(arnoldDisplayList, ['1.0.0.3', '3.0.0.1']) def testGetArnoldDisplayList_FilteredByOS_AndMaya_ReturnsFilteredSet(self): arnoldDisplayList = self.poolImages.getArnoldDisplayList(selectedOS = 'CentOS 73', selectedMaya ='2018-Update1') self.assertEqual(len(arnoldDisplayList), 1) - self.assertSetEqual(arnoldDisplayList, set(['2.0.1.1'])) + self.assertListEqual(arnoldDisplayList, ['2.0.1.1']) def testGetArnoldDisplayList_FilteredByOS_AndMaya_AndArnold_ReturnsFilteredSet(self): arnoldDisplayList = self.poolImages.getArnoldDisplayList(selectedOS = 'CentOS 73', selectedMaya ='2017-Update5', selectedVRay = '1.0.0.2') self.assertEqual(len(arnoldDisplayList), 1) - self.assertSetEqual(arnoldDisplayList, set(['1.0.0.1'])) + self.assertListEqual(arnoldDisplayList, ['1.0.0.1']) #getSelectedImage def testGetSelectedImage_OSMayaOnly_ReturnsFirstValidEntry(self):