added automated build

fixed service principal

python/pipelines.yaml

@@ -0,0 +1,22 @@
+pool:
+  vmImage: 'ubuntu-16.04' 
+
+steps:
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: '3.6'
+
+- script: |
+    cd python
+    python install -e .[extra]
+  displayName: 'Install Package'
+
+- script: |
+    cd python
+    pytest --junitxml=junit/test-unitttest.xml
+  displayName: 'Python Unit Tests'
+
+- task: PublishTestResults@2
+    inputs:
+      testResultsFiles: '**/test-unitttest.xml'
+      testRunTitle: 'Test results for PyTest'

python/pyworkspace/azure/auth/serviceprincipal.py

@@ -1,8 +1,9 @@
 from ...base import Resource
 
+
 class AzureServicePrincipal(Resource):
     yaml_tag = u'!azure.serviceprincipal'
-    
+
     def __init__(self, clientid, tenantid):
         self.clientid = clientid
         self.tenantid = tenantid

python/pyworkspace/azure/keyvault.py

@@ -2,26 +2,24 @@ import yaml
 from azure.keyvault import KeyVaultClient, KeyVaultAuthentication, KeyVaultId
 from ..credentialprovider import CredentialProvider
 
+
 class AzureKeyVault(CredentialProvider):
     yaml_tag = u'!azure.keyvault'
-    
+
     def __init__(self, dnsname, tenantid):
         self.dnsname = dnsname
         self.tenantid = tenantid
 
-    def get_client(self):
-        # cache the client
-        if not hasattr(self, 'client'):
-            # create a KeyVaultAuthentication instance which will callback to the supplied adal_callback
-            credential_client = self.credential.get_client()
+    def get_client_lazy(self):
+        # create a KeyVaultAuthentication instance which will callback to the supplied adal_callback
+        credential_client = self.credential.get_client()
 
-            # distinguish between serviceprincipal and adal callback auth
-            auth = KeyVaultAuthentication(credential_client) if callable(credential_client) else credential_client 
+        # distinguish between serviceprincipal and adal callback auth
+        auth = KeyVaultAuthentication(credential_client) if callable(
+            credential_client) else credential_client
+
+        return KeyVaultClient(auth)
 
-            self.client = KeyVaultClient(auth)
-        
-        return self.client
-        
     def get_secret(self, key, secret_version=KeyVaultId.version_none):
         # TODO: catch exception and rethrow CredentialProviderKeyNotFound
-        return self.get_client().get_secret(self.dnsname, key, secret_version=secret_version).value
+        return self.get_client().get_secret(self.dnsname, key, secret_version=secret_version).value

python/pyworkspace/workspace.py

@@ -11,8 +11,9 @@ from .datasource import *
 from .credentialprovider import *
 from .python import *
 
+
 class Workspace:
-    def __init__(self, path: str=None, content: str=None):
+    def __init__(self, path: str = None, content: str = None):
         # TODO: think about multiple yamls and when to actually stop
         # force user to supply path
         # stop at .git directory (what about .svn?)
@@ -21,13 +22,13 @@ class Workspace:
         if content is None:
             if path is None:
                 path = self.__find_yaml('.')
-            
+
             with open(path, 'r') as f:
                 content = f.read()
 
         self.__parse(content)
 
-    def __find_yaml(self, path) -> str:   
+    def __find_yaml(self, path) -> str:
         path = os.path.realpath(path)
 
         for name in os.listdir(path):
@@ -37,7 +38,7 @@ class Workspace:
 
         # going up the directory structure
         new_path = os.path.realpath(os.path.join(path, '..'))
-        if path == new_path: # hit the root
+        if path == new_path:  # hit the root
             raise Exception("Unable to find resources.yaml")
 
         return self.__find_yaml(new_path)
@@ -45,7 +46,7 @@ class Workspace:
     def __parse(self, content: str):
         # TODO: don't fail for future types. the safe_load thing is the right approach, but
         #       this will lead to failures ones new types are introduced and the workspace library
-        #       is still old... 
+        #       is still old...
 
         # Still loading using SafeLoader, but making sure unknown tags are ignored
         # https://security.openstack.org/guidelines/dg_avoid-dangerous-input-parsing-libraries.html
@@ -53,37 +54,42 @@ class Workspace:
 
         class SafeLoaderIgnoreUnknown(yaml.SafeLoader):
             def ignore_unknown(self, node):
-                return None 
-        
-        SafeLoaderIgnoreUnknown.add_constructor(None, SafeLoaderIgnoreUnknown.ignore_unknown)
+                return None
+
+        SafeLoaderIgnoreUnknown.add_constructor(
+            None, SafeLoaderIgnoreUnknown.ignore_unknown)
         self.resources = yaml.load(content, Loader=SafeLoaderIgnoreUnknown)
 
         # visit all nodes and setup links
         def setup_links(node, path, name):
-            node.__workspace = self
-            node.__path = path
-            node.__name = name        
+            # make sure we don't overwrite path/name from a reference usage (e.g. *foo)
+            # &foo needs to come before *foo
+            if not hasattr(node, '_Workspace__workspace'):
+                node.__workspace = self
+                node.__path = path
+                node.__name = name
 
         # setup root links to avoid back reference to credential provider
         self.visit(setup_links)
 
-    def visit_resource(self, 
-              action: Callable[[yaml.YAMLObject, List[str], str], Any],
-              path: List[str],
-              node: Any,
-              name: str) -> List:
+    def visit_resource(self,
+                       action: Callable[[yaml.YAMLObject, List[str], str], Any],
+                       path: List[str],
+                       node: Any,
+                       name: str) -> List:
 
         ret = []
 
         # execute action for the reousrce
-        v = action(node, path, name) 
+        v = action(node, path, name)
         if v is not None:
             ret.append(v)
 
         # recurse into yaml objects to support nested data defs
         for k, n in node.__dict__.items():
             if isinstance(n, yaml.YAMLObject):
-                ret.extend(self.visit_resource(action, [*path, name], node=n, name=k))
+                ret.extend(self.visit_resource(
+                    action, [*path, name], node=n, name=k))
 
         return ret
 
@@ -113,7 +119,8 @@ class Workspace:
 
         if len(path) == 1:
             # search all
-            ret = self.visit(lambda node, _, name: node if name == key else None)
+            ret = self.visit(
+                lambda node, _, name: node if name == key else None)
             ret_len = len(ret)
 
             # make it convenient
@@ -128,4 +135,4 @@ class Workspace:
             node = self.resources
             for name in path:
                 node = node[name]
-            return node
+            return node

python/setup.py

@@ -6,34 +6,35 @@ from io import open
 here = path.abspath(path.dirname(__file__))
 
 # Get the long description from the README file
-with open(path.join(here, 'README.md'), encoding='utf-8') as f:
+with open(path.join(here, '..', 'README.md'), encoding='utf-8') as f:
     long_description = f.read()
 
-setup(name='pyworkspace', 
+setup(name='pyworkspace',
       version='0.1',
       description='Manages your cloud resources across multiple executing environments.',
       url='http://github.com/Microsoft/Workspace',
       author='Markus Cozowicz',
       author_email='marcozo@microsoft.com',
-      classifiers=[ 
-        'Development Status :: 3 - Alpha',
-        'Intended Audience :: Developers',
-        'Intended Audience :: Science/Research'
-        'Topic :: Software Development',
-        'License :: OSI Approved :: MIT License',
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.4',
-        'Programming Language :: Python :: 3.5',
-        'Programming Language :: Python :: 3.6',
-        'Programming Language :: Python :: 3.7',
+      classifiers=[
+          'Development Status :: 3 - Alpha',
+          'Intended Audience :: Developers',
+          'Intended Audience :: Science/Research'
+          'Topic :: Software Development',
+          'License :: OSI Approved :: MIT License',
+          'Programming Language :: Python :: 3',
+          'Programming Language :: Python :: 3.4',
+          'Programming Language :: Python :: 3.5',
+          'Programming Language :: Python :: 3.6',
+          'Programming Language :: Python :: 3.7',
       ],
       keywords='cloud security resources',
       python_requires='>=3',
       install_requires=['pyyaml'],
-      extras_require={ 
-        'test': ['azureml-dataprep[pandas]', 
-                 'azure-keyvault', 
-                 'sqlalchemy', 
-                 'keyring'],
+      extras_require={
+          'test': ['azureml-dataprep[pandas]',
+                   'azure-keyvault',
+                   'azure-storage-blob',
+                   'sqlalchemy',
+                   'keyring'],
       },
-      packages=find_packages())
+      packages=find_packages())

python/tests/azure/storage/test_azure_storage.py

@@ -3,20 +3,23 @@ import keyring
 import os
 import pytest
 
+
 @pytest.fixture
 def test_subdir():
     # change to tests/ subdir so we can resolve the yaml
     os.chdir(os.path.dirname(os.path.abspath(__file__)))
 
-def test_service_prinicpal(test_subdir):
+
+def test_service_principal(test_subdir):
     ws = pyworkspace.Workspace()
 
     svc1 = ws['myvault1']
     assert len(svc1.get_secret("workspacetest1")) > 0
 
+
 def test_service_csv1(test_subdir):
     ws = pyworkspace.Workspace()
 
     df = ws['csv1'].to_pandas_dataframe()
     assert (df.columns == ['a', 'b', 'c']).all()
-    assert df.shape == (2, 3)
+    assert df.shape == (2, 3)