Support to accept terms for gallery images

Some gallery images need to accept term, then it can be deployed.
And when deploying, they need plan information. Implement progress
like LISAv2 to query and accept terms, if it's necessary. As
azure-mgmt-marketplaceordering isn't compatible with latest
azure-identity, so add cred_wrapper to workaround it.

1. Add azure-mgmt-marketplaceordering package to support terms.
2. Query plan for gallery image deployment.
3. add cred_wrapper for azure-identity compatibility.
4. Add plan in arm_template.
This commit is contained in:
Chi Song 2020-11-03 14:47:38 +08:00
Родитель 40d618d44f
Коммит 79aaee71bf
6 изменённых файлов: 245 добавлений и 15 удалений

Просмотреть файл

@ -387,6 +387,7 @@
"name": "[parameters('nodes')[copyIndex('vmCopy')]['name']]",
"location": "[parameters('nodes')[copyIndex('vmCopy')]['location']]",
"tags": { "RG": "[parameters('resourceGroupName')]" },
"plan": "[parameters('nodes')[copyIndex('vmCopy')]['purchasePlan']]",
"dependsOn": [
"[resourceId('Microsoft.Compute/availabilitySets', variables('availabilitySetName'))]",
"[resourceId('Microsoft.Compute/images', concat(parameters('nodes')[copyIndex('vmCopy')]['name'], '-image'))]",

Просмотреть файл

@ -2,10 +2,13 @@ from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
from azure.mgmt.compute import ComputeManagementClient # type: ignore
from azure.mgmt.marketplaceordering import MarketplaceOrderingAgreements # type: ignore
from lisa.environment import Environment
from lisa.node import Node
from .cred_wrapper import CredentialWrapper
if TYPE_CHECKING:
from .platform_ import AzurePlatform
@ -37,6 +40,14 @@ def get_compute_client(platform: Any) -> ComputeManagementClient:
)
def get_marketplace_ordering_client(platform: Any) -> MarketplaceOrderingAgreements:
azure_platform: AzurePlatform = platform
return MarketplaceOrderingAgreements(
credentials=CredentialWrapper(azure_platform.credential),
subscription_id=azure_platform.subscription_id,
)
def get_node_context(node: Node) -> NodeContext:
return node.get_context(NodeContext)

Просмотреть файл

@ -0,0 +1,72 @@
# copy from https://gist.github.com/lmazuel/cc683d82ea1d7b40208de7c9fc8de59d to
# be compatible with azure-mgmt-marketplaceordering=0.2.1. Once it's upgrade,
# the wrapper can be removed.
# Wrap credentials from azure-identity to be compatible with SDK that needs msrestazure
# or azure.common.credentials
# Need msrest >= 0.6.0
# See also https://pypi.org/project/azure-identity/
from typing import Any
from azure.core.pipeline import PipelineContext, PipelineRequest
from azure.core.pipeline.policies import BearerTokenCredentialPolicy
from azure.core.pipeline.transport import HttpRequest
from azure.identity import DefaultAzureCredential # type: ignore
from msrest.authentication import BasicTokenAuthentication
class CredentialWrapper(BasicTokenAuthentication):
def __init__(
self,
credential: Any = None,
resource_id: str = "https://management.azure.com/.default",
**kwargs: Any,
):
"""Wrap any azure-identity credential to work with SDK that needs
azure.common.credentials/msrestazure.
Default resource is ARM (syntax of endpoint v2)
:param credential: Any azure-identity credential (DefaultAzureCredential by
default)
:param str resource_id: The scope to use to get the token (default ARM)
"""
super(CredentialWrapper, self).__init__(dict())
if credential is None:
credential = DefaultAzureCredential()
self._policy = BearerTokenCredentialPolicy(credential, resource_id, **kwargs)
def _make_request(self) -> PipelineRequest: # type:ignore
return PipelineRequest(
HttpRequest("CredentialWrapper", "https://fakeurl"),
PipelineContext(None), # type:ignore
)
def set_token(self) -> None:
"""Ask the azure-core BearerTokenCredentialPolicy policy to get a token.
Using the policy gives us for free the caching system of azure-core.
We could make this code simpler by using private method, but by definition
I can't assure they will be there forever, so mocking a fake call to the policy
to extract the token, using 100% public API."""
request = self._make_request()
self._policy.on_request(request)
# Read Authorization, and get the second part after Bearer
token = request.http_request.headers["Authorization"].split(" ", 1)[1]
self.token = {"access_token": token}
def signed_session(self, session: Any = None) -> Any:
self.set_token()
return super(CredentialWrapper, self).signed_session(session)
if __name__ == "__main__":
import os
credentials = CredentialWrapper()
subscription_id = os.environ.get("AZURE_SUBSCRIPTION_ID", "<subscription_id>")
from azure.mgmt.resource import ResourceManagementClient # type:ignore
client = ResourceManagementClient(credentials, subscription_id)
for rg in client.resource_groups.list():
print(rg.name)

Просмотреть файл

@ -11,7 +11,12 @@ from typing import Any, Dict, List, Optional, Type, Union
from azure.core.exceptions import HttpResponseError
from azure.identity import DefaultAzureCredential # type: ignore
from azure.mgmt.compute.models import ResourceSku, VirtualMachine # type: ignore
from azure.mgmt.compute.models import ( # type: ignore
PurchasePlan,
ResourceSku,
VirtualMachine,
)
from azure.mgmt.marketplaceordering.models import AgreementTerms # type: ignore
from azure.mgmt.network import NetworkManagementClient # type: ignore
from azure.mgmt.network.models import NetworkInterface, PublicIPAddress # type: ignore
from azure.mgmt.resource import ( # type: ignore
@ -48,6 +53,7 @@ from .common import (
AZURE,
get_compute_client,
get_environment_context,
get_marketplace_ordering_client,
get_node_context,
wait_operation,
)
@ -117,6 +123,14 @@ class AzureVmGallerySchema:
version: str = "Latest"
@dataclass_json(letter_case=LetterCase.CAMEL)
@dataclass
class AzureVmPurchasePlanSchema:
name: str
product: str
publisher: str
@dataclass_json(letter_case=LetterCase.CAMEL)
@dataclass
class AzureNodeSchema:
@ -128,6 +142,8 @@ class AzureNodeSchema:
)
vhd: str = ""
nic_count: int = 1
# for gallery image, which need to accept terms
purchase_plan: Optional[AzureVmPurchasePlanSchema] = None
def __post_init__(self, *args: Any, **kwargs: Any) -> None:
add_secret(self.vhd)
@ -138,6 +154,8 @@ class AzureNodeSchema:
] = AzureVmGallerySchema.schema().load( # type: ignore
self.gallery_raw
)
# this step makes gallery_raw is validated, and filter out any unwanted
# content.
self.gallery_raw = self.gallery.to_dict() # type: ignore
elif self.gallery_raw:
assert isinstance(
@ -149,6 +167,7 @@ class AzureNodeSchema:
self.gallery = AzureVmGallerySchema(
gallery[0], gallery[1], gallery[2], gallery[3]
)
# gallery_raw is used
self.gallery_raw = self.gallery.to_dict() # type: ignore
else:
raise LisaException(
@ -742,6 +761,11 @@ class AzurePlatform(Platform):
elif not azure_node_runbook.gallery:
# set to default gallery, if nothing secified
azure_node_runbook.gallery = AzureVmGallerySchema()
if azure_node_runbook.gallery and not azure_node_runbook.purchase_plan:
azure_node_runbook.purchase_plan = self._process_gallery_image_plan(
azure_node_runbook.location, azure_node_runbook.gallery
)
nodes_parameters.append(azure_node_runbook)
if not arm_parameters.location:
@ -1000,3 +1024,57 @@ class AzurePlatform(Platform):
location_capabilities.extend(level_capabilities)
available_capabilities[location_name] = location_capabilities
self._eligible_capabilities = available_capabilities
def _process_gallery_image_plan(
self, location: str, gallery: AzureVmGallerySchema
) -> Optional[PurchasePlan]:
"""
this method to fill plan, if a VM needs it. If don't fill it, the deployment
will be failed.
1. Convert latest to a specified version, which is required by get API.
2. Get image_info to check if there is a plan.
3. If there is a plan, it may need to check and accept terms.
"""
compute_client = get_compute_client(self)
version = gallery.version.lower()
if version == "latest":
# latest doesn't work, it needs a specified version.
versioned_images = compute_client.virtual_machine_images.list(
location=location,
publisher_name=gallery.publisher,
offer=gallery.offer,
skus=gallery.sku,
)
# any one should be the same to get purchase plan
version = versioned_images[-1].name
image_info = compute_client.virtual_machine_images.get(
location=location,
publisher_name=gallery.publisher,
offer=gallery.offer,
skus=gallery.sku,
version=version,
)
plan: Optional[AzureVmPurchasePlanSchema] = None
if image_info.plan:
# if there is a plan, it may need to accept term.
marketplace_client = get_marketplace_ordering_client(self)
term: AgreementTerms = marketplace_client.marketplace_agreements.get(
publisher_id=gallery.publisher,
offer_id=gallery.offer,
plan_id=image_info.plan.name,
)
if term.accepted is False:
term.accepted = True
marketplace_client.marketplace_agreements.create(
publisher_id=gallery.publisher,
offer_id=gallery.offer,
plan_id=image_info.plan.name,
parameters=term,
)
plan = AzureVmPurchasePlanSchema(
name=image_info.plan.name,
product=image_info.plan.product,
publisher=image_info.plan.publisher,
)
return plan

95
poetry.lock сгенерированный
Просмотреть файл

@ -1,3 +1,17 @@
[[package]]
name = "adal"
version = "1.2.5"
description = "Note: This library is already replaced by MSAL Python, available here: https://pypi.org/project/msal/ .ADAL Python remains available here as a legacy. The ADAL for Python library makes it easy for python application to authenticate to Azure Active Directory (AAD) in order to access AAD protected web resources."
category = "main"
optional = false
python-versions = "*"
[package.dependencies]
cryptography = ">=1.1.0"
PyJWT = ">=1.0.0"
python-dateutil = ">=2.1.0"
requests = ">=2.0.0"
[[package]]
name = "appdirs"
version = "1.4.4"
@ -101,6 +115,19 @@ python-versions = "*"
[package.dependencies]
azure-core = ">=1.7.0.dev,<2.0.0"
[[package]]
name = "azure-mgmt-marketplaceordering"
version = "0.2.1"
description = "Microsoft Azure Market Place Ordering Client Library for Python"
category = "main"
optional = false
python-versions = "*"
[package.dependencies]
azure-common = ">=1.1,<2.0"
msrest = ">=0.5.0"
msrestazure = ">=0.4.32,<2.0.0"
[[package]]
name = "azure-mgmt-network"
version = "16.0.0"
@ -140,7 +167,7 @@ cffi = ">=1.1"
six = ">=1.4.1"
[package.extras]
tests = ["pytest (>=3.2.1,<3.3.0 || >3.3.0)"]
tests = ["pytest (>=3.2.1,!=3.3.0)"]
typecheck = ["mypy"]
[[package]]
@ -224,11 +251,11 @@ cffi = ">=1.8,<1.11.3 || >1.11.3"
six = ">=1.4.1"
[package.extras]
docs = ["sphinx (>=1.6.5,<1.8.0 || >1.8.0,<3.1.0 || >3.1.0,<3.1.1 || >3.1.1)", "sphinx-rtd-theme"]
docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"]
docstest = ["doc8", "pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"]
pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"]
ssh = ["bcrypt (>=3.1.5)"]
test = ["pytest (>=3.6.0,<3.9.0 || >3.9.0,<3.9.1 || >3.9.1,<3.9.2 || >3.9.2)", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,<3.79.2 || >3.79.2)"]
test = ["pytest (>=3.6.0,!=3.9.0,!=3.9.1,!=3.9.2)", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"]
[[package]]
name = "dataclasses-json"
@ -320,7 +347,7 @@ python-versions = "*"
asttokens = ">=2,<3"
[package.extras]
dev = ["mypy (0.750)", "pylint (2.3.1)", "yapf (0.20.2)", "tox (>=3.0.0)", "pydocstyle (>=2.1.1,<3)", "coverage (>=4.5.1,<5)", "docutils (>=0.14,<1)", "pygments (>=2.2.0,<3)", "dpcontracts (0.6.0)", "tabulate (>=0.8.7,<1)", "py-cpuinfo (>=5.0.0,<6)"]
dev = ["mypy (==0.750)", "pylint (==2.3.1)", "yapf (==0.20.2)", "tox (>=3.0.0)", "pydocstyle (>=2.1.1,<3)", "coverage (>=4.5.1,<5)", "docutils (>=0.14,<1)", "pygments (>=2.2.0,<3)", "dpcontracts (==0.6.0)", "tabulate (>=0.8.7,<1)", "py-cpuinfo (>=5.0.0,<6)"]
[[package]]
name = "idna"
@ -366,7 +393,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
parso = ">=0.7.0,<0.8.0"
[package.extras]
qa = ["flake8 (3.7.9)"]
qa = ["flake8 (==3.7.9)"]
testing = ["Django (<3.1)", "colorama", "docopt", "pytest (>=3.9.0,<5.0.0)"]
[[package]]
@ -378,9 +405,9 @@ optional = false
python-versions = ">=3.5"
[package.extras]
dev = ["pytest", "pytz", "simplejson", "mypy (0.782)", "flake8 (3.8.3)", "flake8-bugbear (20.1.4)", "pre-commit (>=2.4,<3.0)", "tox"]
docs = ["sphinx (3.2.1)", "sphinx-issues (1.2.0)", "alabaster (0.7.12)", "sphinx-version-warning (1.1.2)", "autodocsumm (0.2.0)"]
lint = ["mypy (0.782)", "flake8 (3.8.3)", "flake8-bugbear (20.1.4)", "pre-commit (>=2.4,<3.0)"]
dev = ["pytest", "pytz", "simplejson", "mypy (==0.782)", "flake8 (==3.8.3)", "flake8-bugbear (==20.1.4)", "pre-commit (>=2.4,<3.0)", "tox"]
docs = ["sphinx (==3.2.1)", "sphinx-issues (==1.2.0)", "alabaster (==0.7.12)", "sphinx-version-warning (==1.1.2)", "autodocsumm (==0.2.0)"]
lint = ["mypy (==0.782)", "flake8 (==3.8.3)", "flake8-bugbear (==20.1.4)", "pre-commit (>=2.4,<3.0)"]
tests = ["pytest", "pytz", "simplejson"]
[[package]]
@ -446,6 +473,19 @@ requests-oauthlib = ">=0.5.0"
[package.extras]
async = ["aiohttp (>=3.0)", "aiodns"]
[[package]]
name = "msrestazure"
version = "0.6.4"
description = "AutoRest swagger generator Python client runtime. Azure-specific module."
category = "main"
optional = false
python-versions = "*"
[package.dependencies]
adal = ">=0.6.0,<2.0.0"
msrest = ">=0.6.0,<2.0.0"
six = "*"
[[package]]
name = "mypy"
version = "0.782"
@ -663,7 +703,18 @@ six = "*"
[package.extras]
docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"]
tests = ["pytest (>=3.2.1,<3.3.0 || >3.3.0)", "hypothesis (>=3.27.0)"]
tests = ["pytest (>=3.2.1,!=3.3.0)", "hypothesis (>=3.27.0)"]
[[package]]
name = "python-dateutil"
version = "2.8.1"
description = "Extensions to the standard Python datetime module"
category = "main"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
[package.dependencies]
six = ">=1.5"
[[package]]
name = "python-jsonrpc-server"
@ -745,7 +796,7 @@ urllib3 = ">=1.21.1,<1.25.0 || >1.25.0,<1.25.1 || >1.25.1,<1.26"
[package.extras]
security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"]
socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"]
socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"]
[[package]]
name = "requests-oauthlib"
@ -819,7 +870,7 @@ temppathlib = ">=1.0.3,<2"
typing_extensions = ">=3.6.2.1"
[package.extras]
dev = ["mypy (0.620)", "pylint (1.8.2)", "yapf (0.20.2)", "tox (>=3.0.0)", "coverage (>=4.5.1,<5)", "pydocstyle (>=2.1.1,<3)"]
dev = ["mypy (==0.620)", "pylint (==1.8.2)", "yapf (==0.20.2)", "tox (>=3.0.0)", "coverage (>=4.5.1,<5)", "pydocstyle (>=2.1.1,<3)"]
[[package]]
name = "stringcase"
@ -838,7 +889,7 @@ optional = false
python-versions = "*"
[package.extras]
dev = ["mypy (0.570)", "pylint (1.8.2)", "yapf (0.20.2)", "tox (>=3.0.0)"]
dev = ["mypy (==0.570)", "pylint (==1.8.2)", "yapf (==0.20.2)", "tox (>=3.0.0)"]
test = ["tox (>=3.0.0)"]
[[package]]
@ -909,14 +960,18 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"
[package.extras]
brotli = ["brotlipy (>=0.6.0)"]
secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "pyOpenSSL (>=0.14)", "ipaddress"]
socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"]
socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
[metadata]
lock-version = "1.1"
python-versions = "^3.8"
content-hash = "e6a7b2166380656793d4410d58840a229ce8ddb3864d94f0237c99323ac4a488"
content-hash = "22e17ec78556ebdada79c492c967e90c6f4b3cec0489a4655f6a33d7066fbba8"
[metadata.files]
adal = [
{file = "adal-1.2.5-py2.py3-none-any.whl", hash = "sha256:7492aff8f0ba7dd4e1c477303295c645141540fff34c3ca6de0a0b0e6c1c122a"},
{file = "adal-1.2.5.tar.gz", hash = "sha256:8003ba03ef04170195b3eddda8a5ab43649ef2c5f0287023d515affb1ccfcfc3"},
]
appdirs = [
{file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"},
{file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"},
@ -952,6 +1007,10 @@ azure-mgmt-core = [
{file = "azure-mgmt-core-1.2.0.zip", hash = "sha256:8fe3b59446438f27e34f7b24ea692a982034d9e734617ca1320eedeee1939998"},
{file = "azure_mgmt_core-1.2.0-py2.py3-none-any.whl", hash = "sha256:6966226111e92dff26d984aa1c76f227ce0e8b2069c45c72cfb67f160c452444"},
]
azure-mgmt-marketplaceordering = [
{file = "azure-mgmt-marketplaceordering-0.2.1.zip", hash = "sha256:dc765cde7ec03efe456438c85c6207c2f77775a8ce8a7adb19b0df5c5dc513c2"},
{file = "azure_mgmt_marketplaceordering-0.2.1-py2.py3-none-any.whl", hash = "sha256:12d595f3dbda90de7cbc08ace99b925124ce675219b32bb3fde90e36d357c095"},
]
azure-mgmt-network = [
{file = "azure-mgmt-network-16.0.0.zip", hash = "sha256:6159a8c44590cc58841690c27c7d4acb0cd9ad0a1e5178c1d35e0f48e3c3c0e9"},
{file = "azure_mgmt_network-16.0.0-py2.py3-none-any.whl", hash = "sha256:c0e8358e9d530790dbf3efef6b31bce26e664de5096cbd84c62845067da815d1"},
@ -1148,6 +1207,10 @@ msrest = [
{file = "msrest-0.6.19-py2.py3-none-any.whl", hash = "sha256:87aa64948c3ef3dbf6f6956d2240493e68d714e4621b92b65b3c4d5808297929"},
{file = "msrest-0.6.19.tar.gz", hash = "sha256:55f8c3940bc5dc609f8cf9fcd639444716cc212a943606756272e0d0017bbb5b"},
]
msrestazure = [
{file = "msrestazure-0.6.4-py2.py3-none-any.whl", hash = "sha256:3de50f56147ef529b31e099a982496690468ecef33f0544cb0fa0cfe1e1de5b9"},
{file = "msrestazure-0.6.4.tar.gz", hash = "sha256:a06f0dabc9a6f5efe3b6add4bd8fb623aeadacf816b7a35b0f89107e0544d189"},
]
mypy = [
{file = "mypy-0.782-cp35-cp35m-macosx_10_6_x86_64.whl", hash = "sha256:2c6cde8aa3426c1682d35190b59b71f661237d74b053822ea3d748e2c9578a7c"},
{file = "mypy-0.782-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9c7a9a7ceb2871ba4bac1cf7217a7dd9ccd44c27c2950edbc6dc08530f32ad4e"},
@ -1253,6 +1316,10 @@ pynacl = [
{file = "PyNaCl-1.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:7c6092102219f59ff29788860ccb021e80fffd953920c4a8653889c029b2d420"},
{file = "PyNaCl-1.4.0.tar.gz", hash = "sha256:54e9a2c849c742006516ad56a88f5c74bf2ce92c9f67435187c3c5953b346505"},
]
python-dateutil = [
{file = "python-dateutil-2.8.1.tar.gz", hash = "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c"},
{file = "python_dateutil-2.8.1-py2.py3-none-any.whl", hash = "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"},
]
python-jsonrpc-server = [
{file = "python-jsonrpc-server-0.4.0.tar.gz", hash = "sha256:62c543e541f101ec5b57dc654efc212d2c2e3ea47ff6f54b2e7dcb36ecf20595"},
{file = "python_jsonrpc_server-0.4.0-py3-none-any.whl", hash = "sha256:e5a908ff182e620aac07db5f57887eeb0afe33993008f57dc1b85b594cea250c"},

Просмотреть файл

@ -18,6 +18,7 @@ portalocker = "^1.7.1"
azure-identity = {version = "^1.4.0", allow-prereleases = true}
azure-mgmt-resource = {version = "^15.0.0-beta.1", allow-prereleases = true}
azure-mgmt-compute = {version = "^17.0.0-beta.1", allow-prereleases = true}
azure-mgmt-marketplaceordering = {version = "^0.2.1", allow-prereleases = true}
azure-mgmt-network = {version = "^16.0.0-beta.1", allow-prereleases = true}
asserts = "^0.11.0"