Merge pull request #416 from zezha-msft/path-style-fix

Fixed the handling of path style host for emulator

azure-storage-blob/azure/storage/blob/baseblobservice.py

@@ -200,6 +200,7 @@ class BaseBlobService(StorageClient):
             self.authentication = _StorageSharedKeyAuthentication(
                 self.account_name,
                 self.account_key,
+                self.is_emulated
             )
         elif self.sas_token:
             self.authentication = _StorageSASAuthentication(self.sas_token)

azure-storage-common/ChangeLog.md

@@ -6,6 +6,7 @@
 - Added back the ability to generate account SAS for table service.
 - Fixed bug where a question mark prefix on SAS tokens causes failures.
 - The package has switched from Apache 2.0 to the MIT license.
+- Fixed the handling of path style host for the Storage Emulator, specifically the location lock and retry to secondary location.
 
 ## Version 0.37.1:
 - Fixed the return type of __add__ and __or__ methods on the AccountPermissions class

azure-storage-common/azure/storage/common/_auth.py

@@ -6,15 +6,20 @@
 from ._common_conversion import (
     _sign_string,
 )
+from ._constants import (
+    DEV_ACCOUNT_NAME,
+    DEV_ACCOUNT_SECONDARY_NAME
+)
 
 import logging
 logger = logging.getLogger(__name__)
 
 
 class _StorageSharedKeyAuthentication(object):
-    def __init__(self, account_name, account_key):
+    def __init__(self, account_name, account_key, is_emulated=False):
         self.account_name = account_name
         self.account_key = account_key
+        self.is_emulated = is_emulated
 
     def _get_headers(self, request, headers_to_sign):
         headers = dict((name.lower(), value) for name, value in request.headers.items() if value)
@@ -27,6 +32,13 @@ class _StorageSharedKeyAuthentication(object):
 
     def _get_canonicalized_resource(self, request):
         uri_path = request.path.split('?')[0]
+
+        # for emulator, use the DEV_ACCOUNT_NAME instead of DEV_ACCOUNT_SECONDARY_NAME
+        # as this is how the emulator works
+        if self.is_emulated and uri_path.find(DEV_ACCOUNT_SECONDARY_NAME) == 1:
+            # only replace the first instance
+            uri_path = uri_path.replace(DEV_ACCOUNT_SECONDARY_NAME, DEV_ACCOUNT_NAME, 1)
+
         return '/' + self.account_name + uri_path
 
     def _get_canonicalized_headers(self, request):

azure-storage-common/azure/storage/common/_connection.py

@@ -14,6 +14,7 @@ from ._constants import (
     SERVICE_HOST_BASE,
     DEFAULT_PROTOCOL,
     DEV_ACCOUNT_NAME,
+    DEV_ACCOUNT_SECONDARY_NAME,
     DEV_ACCOUNT_KEY,
     DEV_BLOB_HOST,
     DEV_QUEUE_HOST,
@@ -58,8 +59,8 @@ class _ServiceParameters(object):
             # Only set the account key if a sas_token is not present to allow sas to be used with the emulator
             self.account_key = DEV_ACCOUNT_KEY if not self.sas_token else None
 
-            self.primary_endpoint = '{}/{}'.format(_EMULATOR_ENDPOINTS[service], self.account_name)
-            self.secondary_endpoint = '{}/{}-secondary'.format(_EMULATOR_ENDPOINTS[service], self.account_name)
+            self.primary_endpoint = '{}/{}'.format(_EMULATOR_ENDPOINTS[service], DEV_ACCOUNT_NAME)
+            self.secondary_endpoint = '{}/{}'.format(_EMULATOR_ENDPOINTS[service], DEV_ACCOUNT_SECONDARY_NAME)
         else:
             # Strip whitespace from the key
             if self.account_key:

azure-storage-common/azure/storage/common/_constants.py

@@ -29,6 +29,7 @@ DEV_QUEUE_HOST = '127.0.0.1:10001'
 
 # Default credentials for Development Storage Service
 DEV_ACCOUNT_NAME = 'devstoreaccount1'
+DEV_ACCOUNT_SECONDARY_NAME = 'devstoreaccount1-secondary'
 DEV_ACCOUNT_KEY = 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=='
 
 # Socket timeout in seconds

azure-storage-common/azure/storage/common/models.py

@@ -120,6 +120,8 @@ class RetryContext(object):
     :ivar Exception exception:
         The exception that just occurred. The type could either be AzureException (for HTTP errors),
         or other Exception types from lower layers, which are kept unwrapped for easier processing.
+    :ivar bool is_emulated:
+        Whether retry is targeting the emulator. The default value is False.
     '''
 
     def __init__(self):
@@ -127,6 +129,7 @@ class RetryContext(object):
         self.response = None
         self.location_mode = None
         self.exception = None
+        self.is_emulated = False
 
 
 class LocationMode(object):

azure-storage-common/azure/storage/common/retry.py

@@ -8,6 +8,10 @@ from math import pow
 import random
 
 from .models import LocationMode
+from ._constants import (
+    DEV_ACCOUNT_NAME,
+    DEV_ACCOUNT_SECONDARY_NAME
+)
 
 
 class _Retry(object):
@@ -98,10 +102,22 @@ class _Retry(object):
             # If there's more than one possible location, retry to the alternative
             if context.location_mode == LocationMode.PRIMARY:
                 context.location_mode = LocationMode.SECONDARY
+
+                # if targeting the emulator (with path style), change path instead of host
+                if context.is_emulated:
+                    # replace the first instance of primary account name with the secondary account name
+                    context.request.path = context.request.path.replace(DEV_ACCOUNT_NAME, DEV_ACCOUNT_SECONDARY_NAME, 1)
+                else:
+                    context.request.host = context.request.host_locations.get(context.location_mode)
             else:
                 context.location_mode = LocationMode.PRIMARY
 
-            context.request.host = context.request.host_locations.get(context.location_mode)
+                # if targeting the emulator (with path style), change path instead of host
+                if context.is_emulated:
+                    # replace the first instance of secondary account name with the primary account name
+                    context.request.path = context.request.path.replace(DEV_ACCOUNT_SECONDARY_NAME, DEV_ACCOUNT_NAME, 1)
+                else:
+                    context.request.host = context.request.host_locations.get(context.location_mode)
 
     def _retry(self, context, backoff):
         '''

azure-storage-common/azure/storage/common/storageclient.py

@@ -214,6 +214,7 @@ class StorageClient(object):
         '''
         operation_context = operation_context or _OperationContext()
         retry_context = RetryContext()
+        retry_context.is_emulated = self.is_emulated
 
         # Apply the appropriate host based on the location mode
         self._apply_host(request, operation_context, retry_context)
@@ -356,4 +357,7 @@ class StorageClient(object):
                 # this is the first request of that operation. Set the location to 
                 # be used for subsequent requests in the operation.
                 if operation_context.location_lock and not operation_context.host_location:
-                    operation_context.host_location = {retry_context.location_mode: request.host}
+                    # note: to cover the emulator scenario, the host_location is grabbed
+                    # from request.host_locations(which includes the dev account name)
+                    # instead of request.host(which at this point no longer includes the dev account name)
+                    operation_context.host_location = {retry_context.location_mode: request.host_locations[retry_context.location_mode]}

azure-storage-queue/azure/storage/queue/queueservice.py

@@ -178,6 +178,7 @@ class QueueService(StorageClient):
             self.authentication = _StorageSharedKeyAuthentication(
                 self.account_name,
                 self.account_key,
+                self.is_emulated
             )
         elif self.sas_token:
             self.authentication = _StorageSASAuthentication(self.sas_token)

tests/common/test_account.py

@@ -19,12 +19,17 @@ from azure.storage.common import (
     AccountPermissions,
     Services,
 )
+from azure.storage.common._constants import (
+    DEV_ACCOUNT_NAME,
+    DEV_ACCOUNT_KEY
+)
 from azure.storage.file import FileService
 from azure.storage.queue import QueueService
 from tests.testcase import (
     StorageTestCase,
     TestMode,
     record,
+    not_for_emulator,
 )
 
 
@@ -34,10 +39,18 @@ from tests.testcase import (
 class StorageAccountTest(StorageTestCase):
     def setUp(self):
         super(StorageAccountTest, self).setUp()
-        self.account_name = self.settings.STORAGE_ACCOUNT_NAME
-        self.account_key = self.settings.STORAGE_ACCOUNT_KEY
         self.sas_token = '?sv=2015-04-05&st=2015-04-29T22%3A18%3A26Z&se=2015-04-30T02%3A23%3A26Z&sr=b&sp=rw&sip=168.1.5.60-168.1.5.70&spr=https&sig=Z%2FRHIX5Xcg0Mq2rqI3OlWTjEg2tYkboXr1P9ZUXDtkk%3D'
-        self.account = CloudStorageAccount(self.account_name, self.account_key)
+
+        if self.settings.IS_EMULATED:
+            self.account_name = DEV_ACCOUNT_NAME
+            self.account_key = DEV_ACCOUNT_KEY
+            self.protocol = "http"
+            self.account = CloudStorageAccount(DEV_ACCOUNT_NAME, DEV_ACCOUNT_KEY, is_emulated=True)
+        else:
+            self.account_name = self.settings.STORAGE_ACCOUNT_NAME
+            self.account_key = self.settings.STORAGE_ACCOUNT_KEY
+            self.account = CloudStorageAccount(self.account_name, self.account_key)
+            self.protocol = self.settings.PROTOCOL
 
     # --Helpers-----------------------------------------------------------------
     def validate_service(self, service, type):
@@ -83,6 +96,7 @@ class StorageAccountTest(StorageTestCase):
         # Assert
         self.validate_service(service, QueueService)
 
+    @not_for_emulator
     def test_create_file_service(self):
         # Arrange
 
@@ -188,7 +202,7 @@ class StorageAccountTest(StorageTestCase):
             datetime.utcnow() + timedelta(hours=1),
         )
 
-        service = BlockBlobService(self.account_name, sas_token=token)
+        service = BlockBlobService(self.account_name, sas_token=token, is_emulated=self.settings.IS_EMULATED)
         data = b'shared access signature with read/write permission on blob'
         container_name = 'container1'
         blob_name = 'blob1.txt'
@@ -225,7 +239,7 @@ class StorageAccountTest(StorageTestCase):
         )
 
         service_with_key = self.account.create_block_blob_service()
-        service_with_sas = BlockBlobService(account_name=self.account_name, sas_token=token)
+        service_with_sas = BlockBlobService(account_name=self.account_name, sas_token=token, is_emulated=self.settings.IS_EMULATED)
         data = b'shared access signature with read/write permission on blob'
         container_name = 'container2'
         blob_name = 'blob1.txt'
@@ -265,9 +279,10 @@ class StorageAccountTest(StorageTestCase):
         self.assertTrue('ss=bt' in token)
 
         # Act Table
-        url = '{}://{}.table.core.windows.net/?restype=service&comp=properties&{}'.format(
-            self.settings.PROTOCOL,
-            self.account_name,
+        # this needs to be hard coded as the table package is no longer maintained here
+        url = '{}://{}/?restype=service&comp=properties&{}'.format(
+            self.protocol,
+            self.account_name + ".table.core.windows.net" if not self.settings.IS_EMULATED else "127.0.0.1:10002/" + DEV_ACCOUNT_NAME,
             token,
         )
         response = requests.get(url)
@@ -277,7 +292,7 @@ class StorageAccountTest(StorageTestCase):
 
         # Act Blob
         service_with_key = self.account.create_block_blob_service()
-        service_with_sas = BlockBlobService(account_name=self.account_name, sas_token=token)
+        service_with_sas = BlockBlobService(account_name=self.account_name, sas_token=token, is_emulated=self.settings.IS_EMULATED)
         data = b'shared access signature with read/write permission on blob'
         container_name = 'container2'
         blob_name = 'blob1.txt'

tests/common/test_client.py

@@ -330,7 +330,7 @@ class StorageClientTest(StorageTestCase):
     @record
     def test_request_callback_signed_header(self):
         # Arrange
-        service = BlockBlobService(self.account_name, self.account_key)
+        service = BlockBlobService(self.account_name, self.account_key, is_emulated=self.settings.IS_EMULATED)
         name = self.get_resource_name('cont')
 
         # Act
@@ -351,7 +351,7 @@ class StorageClientTest(StorageTestCase):
     @record
     def test_response_callback(self):
         # Arrange
-        service = BlockBlobService(self.account_name, self.account_key)
+        service = BlockBlobService(self.account_name, self.account_key, is_emulated=self.settings.IS_EMULATED)
         name = self.get_resource_name('cont')
 
         # Act

tests/common/test_retry.py

@@ -193,7 +193,7 @@ class StorageRetryTest(StorageTestCase):
         container_name = self.get_resource_name()
         service = self._create_storage_service(BlockBlobService, self.settings)
         service.create_container(container_name)
-        service.retry = ExponentialRetry(max_attempts=3).retry
+        service.retry = ExponentialRetry(initial_backoff=1, increment_power=3, max_attempts=3).retry
 
         # Force the create call to 'timeout' with a 408
         response_callback = ResponseCallback(status=200, new_status=408)
@@ -330,7 +330,10 @@ class StorageRetryTest(StorageTestCase):
 
             # Assert
             def request_callback(request):
-                self.assertNotEqual(-1, request.host.find('-secondary'))
+                if self.settings.IS_EMULATED:
+                    self.assertNotEqual(-1, request.path.find('-secondary'))
+                else:
+                    self.assertNotEqual(-1, request.host.find('-secondary'))
 
             service.request_callback = request_callback
             service.get_container_metadata(container_name)
@@ -415,7 +418,10 @@ class StorageRetryTest(StorageTestCase):
         # to the final location of the first list request (aka secondary) despite 
         # the client normally trying primary first
         def request_callback(request):
-            self.assertNotEqual(-1, request.host.find('-secondary'))
+            if self.settings.IS_EMULATED:
+                self.assertNotEqual(-1, request.path.find('-secondary'))
+            else:
+                self.assertNotEqual(-1, request.host.find('-secondary'))
 
         service.request_callback = request_callback
         service._list_containers(prefix='lock', _context=context)

tests/common/test_service_properties.py

@@ -21,6 +21,7 @@ from azure.storage.queue import QueueService
 from tests.testcase import (
     StorageTestCase,
     record,
+    not_for_emulator,
 )
 
 
@@ -33,7 +34,10 @@ class ServicePropertiesTest(StorageTestCase):
 
         self.bs = self._create_storage_service(BlockBlobService, self.settings)
         self.qs = self._create_storage_service(QueueService, self.settings)
-        self.fs = self._create_storage_service(FileService, self.settings)
+
+        # file service is not supported on the emulator
+        if not self.settings.IS_EMULATED:
+            self.fs = self._create_storage_service(FileService, self.settings)
 
     # --Helpers-----------------------------------------------------------------
     def _assert_properties_default(self, prop):
@@ -113,6 +117,7 @@ class ServicePropertiesTest(StorageTestCase):
         self.assertIsNone(resp)
         self._assert_properties_default(self.qs.get_queue_service_properties())
 
+    @not_for_emulator
     @record
     def test_file_service_properties(self):
         # Arrange

tests/common/test_service_stats.py

@@ -27,8 +27,9 @@ class ServiceStatsTest(StorageTestCase):
         self.assertIsNotNone(stats)
         self.assertIsNotNone(stats.geo_replication)
 
-        self.assertEqual(stats.geo_replication.status, 'live')
-        self.assertIsNotNone(stats.geo_replication.last_sync_time)
+        if not self.settings.IS_EMULATED:
+            self.assertEqual(stats.geo_replication.status, 'live')
+            self.assertIsNotNone(stats.geo_replication.last_sync_time)
 
     def _assert_stats_unavailable(self, stats):
         self.assertIsNotNone(stats)

tests/testcase.py

@@ -378,3 +378,12 @@ def record(test):
             test(self)
     recording_test.__name__ = test.__name__
     return recording_test
+
+
+def not_for_emulator(test):
+    def skip_test_if_targeting_emulator(self):
+        if self.settings.IS_EMULATED:
+            return
+        else:
+            test(self)
+    return skip_test_if_targeting_emulator