[Extensions] Enable CLI extensions to include packages in the 'azure' namespace (#13163)

doc/extensions/authoring.md

@@ -120,8 +120,7 @@ See [Extension Metadata](metadata.md) for more information.
 ### Limit dependencies in setup.py
 
 - Before adding a dependency to your setup.py, check that it's not already available in [azure-cli-core setup.py](https://github.com/Azure/azure-cli/blob/master/src/azure-cli-core/setup.py).
-- For Azure SDKs, use autorest to generate your SDK into a package that isn't under the `azure` directory.
-- You can verify that your extension doesn't use the `azure` directory by opening your `.whl` and opening the `top_level.txt` file in the `*.dist-info` directory. It should not contain `azure`.
+- Azure SDK or Azure Management SDK dependencies may be overridden by the versions installed as requirements of azure-cli-core. If you use any, test carefully, gracefully handle API changes, and be prepared to release updates. You might also consider rebasing the libraries under a different namespace (besides `azure`) to avoid conflicting with core CLI functionality.
 
 ### How do I know I'm using my dev extension(s)?
 
@@ -134,12 +133,6 @@ See [Extension Metadata](metadata.md) for more information.
 - e.g. `python3.6 -m venv env36` and `python3.8 -m venv env38`.
 
 
-:zap: IMPORTANT :zap:
-- Since azure-cli uses the `azure` directory, no extension can use this.
-- This applies to all other dependencies used by azure-cli-core.
-- See [this Stack Overflow question](https://stackoverflow.com/questions/8936884/python-import-path-packages-with-the-same-name-in-different-folders).
-
-
 Also, see the [FAQ](faq.md).
 
 ### Differences between hosting and not hosting source code in Azure/azure-cli-extensions

src/azure-cli-core/azure/cli/core/extension/operations.py

@@ -357,6 +357,24 @@ def reload_extension(extension_name, extension_module=None):
 def add_extension_to_path(extension_name, ext_dir=None):
     ext_dir = ext_dir or get_extension(extension_name).path
     sys.path.append(ext_dir)
+    # If this path update should have made a new "azure" module available,
+    # extend the existing module with its path. This allows extensions to
+    # include (or depend on) Azure SDK modules that are not yet part of
+    # the CLI. This applies to both the "azure" and "azure.mgmt" namespaces,
+    # but ensures that modules installed by the CLI take priority.
+    azure_dir = os.path.join(ext_dir, "azure")
+    if os.path.isdir(azure_dir):
+        import azure
+        azure.__path__.append(azure_dir)
+        azure_mgmt_dir = os.path.join(azure_dir, "mgmt")
+        if os.path.isdir(azure_mgmt_dir):
+            try:
+                # Should have been imported already, so this will be quick
+                import azure.mgmt
+            except ImportError:
+                pass
+            else:
+                azure.mgmt.__path__.append(azure_mgmt_dir)
 
 
 def get_lsb_release():

src/azure-cli-core/azure/cli/core/extension/tests/latest/test_extension_commands.py

@@ -8,11 +8,12 @@ import unittest
 import shutil
 import hashlib
 import mock
+import sys
 
 from azure.cli.core.util import CLIError
-from azure.cli.core.extension import build_extension_path
-from azure.cli.core.extension.operations import (list_extensions, add_extension, show_extension,
-                                                 remove_extension, update_extension,
+from azure.cli.core.extension import get_extension, build_extension_path
+from azure.cli.core.extension.operations import (add_extension_to_path, list_extensions, add_extension,
+                                                 show_extension, remove_extension, update_extension,
                                                  list_available_extensions, OUT_KEY_NAME, OUT_KEY_VERSION,
                                                  OUT_KEY_METADATA, OUT_KEY_PATH)
 from azure.cli.core.extension._resolve import NoExtensionCandidatesError
@@ -424,6 +425,56 @@ class TestExtensionCommands(unittest.TestCase):
         ext = show_extension(MY_EXT_NAME)
         self.assertEqual(ext[OUT_KEY_VERSION], '0.0.4+dev')
 
+    def test_add_extension_to_path(self):
+        add_extension(cmd=self.cmd, source=MY_EXT_SOURCE)
+        num_exts = len(list_extensions())
+        self.assertEqual(num_exts, 1)
+        ext = get_extension('myfirstcliextension')
+        old_path = sys.path[:]
+        try:
+            add_extension_to_path(ext.name)
+            self.assertSequenceEqual(old_path, sys.path[:-1])
+            self.assertEqual(ext.path, sys.path[-1])
+        finally:
+            sys.path[:] = old_path
+
+    def test_add_extension_azure_to_path(self):
+        import azure
+        import azure.mgmt
+        old_path_0 = list(sys.path)
+        old_path_1 = list(azure.__path__)
+        old_path_2 = list(azure.mgmt.__path__)
+
+        add_extension(cmd=self.cmd, source=MY_EXT_SOURCE)
+        ext = get_extension('myfirstcliextension')
+        azure_dir = os.path.join(ext.path, "azure")
+        azure_mgmt_dir = os.path.join(azure_dir, "mgmt")
+        os.mkdir(azure_dir)
+        os.mkdir(azure_mgmt_dir)
+
+        try:
+            add_extension_to_path(ext.name)
+            new_path_1 = list(azure.__path__)
+            new_path_2 = list(azure.mgmt.__path__)
+        finally:
+            sys.path.remove(ext.path)
+            remove_extension(ext.name)
+            if isinstance(azure.__path__, list):
+                azure.__path__[:] = old_path_1
+            else:
+                list(azure.__path__)
+            if isinstance(azure.mgmt.__path__, list):
+                azure.mgmt.__path__[:] = old_path_2
+            else:
+                list(azure.mgmt.__path__)
+        self.assertSequenceEqual(old_path_1, new_path_1[:-1])
+        self.assertSequenceEqual(old_path_2, new_path_2[:-1])
+        self.assertEqual(azure_dir, new_path_1[-1])
+        self.assertEqual(azure_mgmt_dir, new_path_2[-1])
+        self.assertSequenceEqual(old_path_0, list(sys.path))
+        self.assertSequenceEqual(old_path_1, list(azure.__path__))
+        self.assertSequenceEqual(old_path_2, list(azure.mgmt.__path__))
+
     def _setup_cmd(self):
         cmd = mock.MagicMock()
         cmd.cli_ctx = DummyCli()