From 8b460128c5e7af0ca6504e8052a60bdbb248e9b4 Mon Sep 17 00:00:00 2001 From: Zunli Hu Date: Mon, 21 Sep 2020 10:35:47 +0800 Subject: [PATCH] [Storage] Change datetime for copy (#15195) * change datetime * add incremental * suppress credential * refine help * fix style --- scripts/ci/credscan/CredScanSuppressions.json | 3 +- .../cli/command_modules/storage/_params.py | 24 +++--- .../test_storage_blob_incremental_copy.yaml | 74 ++++++++++--------- .../test_storage_blob_copy_scenarios.py | 5 +- .../latest/test_storage_blob_scenarios.py | 3 +- 5 files changed, 59 insertions(+), 50 deletions(-) diff --git a/scripts/ci/credscan/CredScanSuppressions.json b/scripts/ci/credscan/CredScanSuppressions.json index 12b065409..0a36657f6 100644 --- a/scripts/ci/credscan/CredScanSuppressions.json +++ b/scripts/ci/credscan/CredScanSuppressions.json @@ -190,7 +190,8 @@ "src\\azure-cli\\azure\\cli\\command_modules\\storage\\tests\\hybrid_2018_03_01\\test_storage_url_helpers.py", "src\\azure-cli\\azure\\cli\\command_modules\\storage\\tests\\latest\\test_storage_url_helpers.py", "src\\azure-cli\\azure\\cli\\command_modules\\storage\\tests\\latest\\recordings\\test_renew_account_key.yaml", - "src\\azure-cli\\azure\\cli\\command_modules\\storage\\tests\\latest\\recordings\\test_renew_account_kerb_key.yaml" + "src\\azure-cli\\azure\\cli\\command_modules\\storage\\tests\\latest\\recordings\\test_renew_account_kerb_key.yaml", + "src\\azure-cli\\azure\\cli\\command_modules\\storage\\tests\\latest\\recordings\\test_storage_blob_incremental_copy.yaml" ], "_justification": "[Storage] response body contains random value recognized as secret in outdated recoding files of storage may remove in the future" }, diff --git a/src/azure-cli/azure/cli/command_modules/storage/_params.py b/src/azure-cli/azure/cli/command_modules/storage/_params.py index 0b84741a9..6b2870b13 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/_params.py +++ b/src/azure-cli/azure/cli/command_modules/storage/_params.py @@ -166,6 +166,13 @@ def load_arguments(self, _): # pylint: disable=too-many-locals, too-many-statem 'for retrieving the remaining of the results. Provide "*" to return all.' ) + if_modified_since_type = CLIArgumentType( + help='Commence only if modified since supplied UTC datetime (Y-m-d\'T\'H:M\'Z\')', + type=get_datetime_type(False)) + if_unmodified_since_type = CLIArgumentType( + help='Commence only if unmodified since supplied UTC datetime (Y-m-d\'T\'H:M\'Z\')', + type=get_datetime_type(False)) + with self.argument_context('storage') as c: c.argument('container_name', container_name_type) c.argument('directory_name', directory_type) @@ -179,12 +186,8 @@ def load_arguments(self, _): # pylint: disable=too-many-locals, too-many-statem c.argument('timeout', help='Request timeout in seconds. Applies to each call to the service.', type=int) with self.argument_context('storage', arg_group='Precondition') as c: - c.argument('if_modified_since', - help='Commence only if modified since supplied UTC datetime (Y-m-d\'T\'H:M\'Z\')', - type=get_datetime_type(False)) - c.argument('if_unmodified_since', - help='Commence only if unmodified since supplied UTC datetime (Y-m-d\'T\'H:M\'Z\')', - type=get_datetime_type(False)) + c.argument('if_modified_since', if_modified_since_type) + c.argument('if_unmodified_since', if_unmodified_since_type) c.argument('if_match') c.argument('if_none_match') @@ -832,8 +835,9 @@ def load_arguments(self, _): # pylint: disable=too-many-locals, too-many-statem with self.argument_context('storage blob copy') as c: for item in ['destination', 'source']: - c.argument('{}_if_modified_since'.format(item), arg_group='Pre-condition') - c.argument('{}_if_unmodified_since'.format(item), arg_group='Pre-condition') + c.argument('{}_if_modified_since'.format(item), arg_group='Pre-condition', arg_type=if_modified_since_type) + c.argument('{}_if_unmodified_since'.format(item), arg_group='Pre-condition', + arg_type=if_unmodified_since_type) c.argument('{}_if_match'.format(item), arg_group='Pre-condition') c.argument('{}_if_none_match'.format(item), arg_group='Pre-condition') c.argument('container_name', container_name_type, options_list=('--destination-container', '-c')) @@ -862,8 +866,8 @@ def load_arguments(self, _): # pylint: disable=too-many-locals, too-many-statem from azure.cli.command_modules.storage._validators import process_blob_source_uri c.register_source_uri_arguments(validator=process_blob_source_uri, blob_only=True) - c.argument('destination_if_modified_since', arg_group='Pre-condition') - c.argument('destination_if_unmodified_since', arg_group='Pre-condition') + c.argument('destination_if_modified_since', arg_group='Pre-condition', arg_type=if_modified_since_type) + c.argument('destination_if_unmodified_since', arg_group='Pre-condition', arg_type=if_unmodified_since_type) c.argument('destination_if_match', arg_group='Pre-condition') c.argument('destination_if_none_match', arg_group='Pre-condition') c.argument('container_name', container_name_type, options_list=('--destination-container', '-c')) diff --git a/src/azure-cli/azure/cli/command_modules/storage/tests/latest/recordings/test_storage_blob_incremental_copy.yaml b/src/azure-cli/azure/cli/command_modules/storage/tests/latest/recordings/test_storage_blob_incremental_copy.yaml index 931975724..26d821121 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/tests/latest/recordings/test_storage_blob_incremental_copy.yaml +++ b/src/azure-cli/azure/cli/command_modules/storage/tests/latest/recordings/test_storage_blob_incremental_copy.yaml @@ -15,8 +15,8 @@ interactions: ParameterSetName: - -n -g --query -o User-Agent: - - python/3.7.7 (Windows-10-10.0.18362-SP0) msrest/0.6.13 msrest_azure/0.6.3 - azure-mgmt-storage/9.0.0 Azure-SDK-For-Python AZURECLI/2.7.0 + - python/3.7.7 (Windows-10-10.0.19041-SP0) msrest/0.6.18 msrest_azure/0.6.3 + azure-mgmt-storage/11.2.0 Azure-SDK-For-Python AZURECLI/2.12.0 accept-language: - en-US method: POST @@ -32,7 +32,7 @@ interactions: content-type: - application/json date: - - Thu, 11 Jun 2020 09:21:00 GMT + - Thu, 17 Sep 2020 06:53:50 GMT expires: - '-1' pragma: @@ -48,7 +48,7 @@ interactions: x-content-type-options: - nosniff x-ms-ratelimit-remaining-subscription-resource-requests: - - '11998' + - '11999' status: code: 200 message: OK @@ -60,9 +60,9 @@ interactions: Content-Length: - '0' User-Agent: - - Azure-Storage/2.0.0-2.0.1 (Python CPython 3.7.7; Windows 10) AZURECLI/2.7.0 + - Azure-Storage/2.0.0-2.0.1 (Python CPython 3.7.7; Windows 10) AZURECLI/2.12.0 x-ms-date: - - Thu, 11 Jun 2020 09:21:00 GMT + - Thu, 17 Sep 2020 06:53:51 GMT x-ms-version: - '2018-11-09' method: PUT @@ -74,11 +74,11 @@ interactions: content-length: - '0' date: - - Thu, 11 Jun 2020 09:21:02 GMT + - Thu, 17 Sep 2020 06:53:53 GMT etag: - - '"0x8D80DE8C366F4BD"' + - '"0x8D85AD671384D6D"' last-modified: - - Thu, 11 Jun 2020 09:21:03 GMT + - Thu, 17 Sep 2020 06:53:54 GMT server: - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 x-ms-version: @@ -94,13 +94,13 @@ interactions: Content-Length: - '0' User-Agent: - - Azure-Storage/2.0.0-2.0.1 (Python CPython 3.7.7; Windows 10) AZURECLI/2.7.0 + - Azure-Storage/2.0.0-2.0.1 (Python CPython 3.7.7; Windows 10) AZURECLI/2.12.0 x-ms-blob-content-length: - '16384' x-ms-blob-type: - PageBlob x-ms-date: - - Thu, 11 Jun 2020 09:21:03 GMT + - Thu, 17 Sep 2020 06:53:54 GMT x-ms-version: - '2018-11-09' method: PUT @@ -112,11 +112,11 @@ interactions: content-length: - '0' date: - - Thu, 11 Jun 2020 09:21:04 GMT + - Thu, 17 Sep 2020 06:53:54 GMT etag: - - '"0x8D80DE8C4419CA0"' + - '"0x8D85AD671FE6357"' last-modified: - - Thu, 11 Jun 2020 09:21:04 GMT + - Thu, 17 Sep 2020 06:53:55 GMT server: - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 x-ms-request-server-encrypted: @@ -134,9 +134,9 @@ interactions: Content-Length: - '0' User-Agent: - - Azure-Storage/2.0.0-2.0.1 (Python CPython 3.7.7; Windows 10) AZURECLI/2.7.0 + - Azure-Storage/2.0.0-2.0.1 (Python CPython 3.7.7; Windows 10) AZURECLI/2.12.0 x-ms-date: - - Thu, 11 Jun 2020 09:21:04 GMT + - Thu, 17 Sep 2020 06:53:56 GMT x-ms-version: - '2018-11-09' method: PUT @@ -148,17 +148,17 @@ interactions: content-length: - '0' date: - - Thu, 11 Jun 2020 09:21:05 GMT + - Thu, 17 Sep 2020 06:53:56 GMT etag: - - '"0x8D80DE8C4419CA0"' + - '"0x8D85AD671FE6357"' last-modified: - - Thu, 11 Jun 2020 09:21:04 GMT + - Thu, 17 Sep 2020 06:53:55 GMT server: - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 x-ms-request-server-encrypted: - 'false' x-ms-snapshot: - - '2020-06-11T09:21:06.1341828Z' + - '2020-09-17T06:53:57.0164175Z' x-ms-version: - '2018-11-09' status: @@ -180,8 +180,8 @@ interactions: ParameterSetName: - -n -g --query -o User-Agent: - - python/3.7.7 (Windows-10-10.0.18362-SP0) msrest/0.6.13 msrest_azure/0.6.3 - azure-mgmt-storage/9.0.0 Azure-SDK-For-Python AZURECLI/2.7.0 + - python/3.7.7 (Windows-10-10.0.19041-SP0) msrest/0.6.18 msrest_azure/0.6.3 + azure-mgmt-storage/11.2.0 Azure-SDK-For-Python AZURECLI/2.12.0 accept-language: - en-US method: POST @@ -197,7 +197,7 @@ interactions: content-type: - application/json date: - - Thu, 11 Jun 2020 09:21:06 GMT + - Thu, 17 Sep 2020 06:53:58 GMT expires: - '-1' pragma: @@ -213,7 +213,7 @@ interactions: x-content-type-options: - nosniff x-ms-ratelimit-remaining-subscription-resource-requests: - - '11998' + - '11999' status: code: 200 message: OK @@ -225,9 +225,9 @@ interactions: Content-Length: - '0' User-Agent: - - Azure-Storage/2.0.0-2.0.1 (Python CPython 3.7.7; Windows 10) AZURECLI/2.7.0 + - Azure-Storage/2.0.0-2.0.1 (Python CPython 3.7.7; Windows 10) AZURECLI/2.12.0 x-ms-date: - - Thu, 11 Jun 2020 09:21:07 GMT + - Thu, 17 Sep 2020 06:53:58 GMT x-ms-version: - '2018-11-09' method: PUT @@ -239,11 +239,11 @@ interactions: content-length: - '0' date: - - Thu, 11 Jun 2020 09:21:10 GMT + - Thu, 17 Sep 2020 06:54:01 GMT etag: - - '"0x8D80DE8C76F9E10"' + - '"0x8D85AD67590445A"' last-modified: - - Thu, 11 Jun 2020 09:21:10 GMT + - Thu, 17 Sep 2020 06:54:01 GMT server: - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 x-ms-version: @@ -258,12 +258,14 @@ interactions: - keep-alive Content-Length: - '0' + If-Modified-Since: + - Mon, 29 Jun 2020 06:32:00 GMT User-Agent: - - Azure-Storage/2.0.0-2.0.1 (Python CPython 3.7.7; Windows 10) AZURECLI/2.7.0 + - Azure-Storage/2.0.0-2.0.1 (Python CPython 3.7.7; Windows 10) AZURECLI/2.12.0 x-ms-copy-source: - - https://clitestrahgamwkwbwtkykfm.blob.core.windows.net/contqvllunkl3q3am2gquupp/src?se=2020-06-12T09%3A21%3A10Z&sp=r&spr=https&sv=2018-11-09&sr=b&sig=DQI/Vu4xLAF7uBO14WXbhn%2BD5xHh26mXJkY7d1QYA94%3D&snapshot=2020-06-11T09:21:06.1341828Z + - https://clitestrjptjxuxfis4hef3e.blob.core.windows.net/contfazeuvmfwt7bgddmwk6u/src?se=2020-09-18T06%3A54%3A01Z&sp=r&spr=https&sv=2018-11-09&sr=b&sig=gjNq3qPzTlXLmNCqLFWWc3xim3Th0FLWdh2A7slKb6k%3D&snapshot=2020-09-17T06:53:57.0164175Z x-ms-date: - - Thu, 11 Jun 2020 09:21:10 GMT + - Thu, 17 Sep 2020 06:54:01 GMT x-ms-version: - '2018-11-09' method: PUT @@ -275,15 +277,15 @@ interactions: content-length: - '0' date: - - Thu, 11 Jun 2020 09:21:11 GMT + - Thu, 17 Sep 2020 06:54:02 GMT etag: - - '"0x8D80DE8C846270F"' + - '"0x8D85AD6766005B6"' last-modified: - - Thu, 11 Jun 2020 09:21:11 GMT + - Thu, 17 Sep 2020 06:54:02 GMT server: - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 x-ms-copy-id: - - 4ea2ad4e-1335-4900-ab1a-d281d209182e + - c4b8603c-bbbe-482a-a2f1-2e18719949cd x-ms-copy-status: - pending x-ms-version: diff --git a/src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_blob_copy_scenarios.py b/src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_blob_copy_scenarios.py index 23315575c..b955ffd3f 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_blob_copy_scenarios.py +++ b/src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_blob_copy_scenarios.py @@ -85,8 +85,9 @@ class StorageBlobCopyTests(StorageScenarioMixin, LiveScenarioTest): ' --expiry {}', account_info, source_container, start, expiry).output.strip() - self.storage_cmd('storage blob copy start -b dst -c {} --source-blob src --sas-token {} --source-container {}', - account_info, target_container, sas, source_container) + self.storage_cmd('storage blob copy start -b dst -c {} --source-blob src --sas-token {} --source-container {} ' + '--source-if-unmodified-since "2020-06-29T06:32Z" --destination-if-modified-since ' + '"2020-06-29T06:32Z" ', account_info, target_container, sas, source_container) from time import sleep, time start = time() diff --git a/src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_blob_scenarios.py b/src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_blob_scenarios.py index c83eb487e..dbae07214 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_blob_scenarios.py +++ b/src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_blob_scenarios.py @@ -36,7 +36,8 @@ class StorageBlobUploadTests(StorageScenarioMixin, ScenarioTest): target_container = self.create_container(target_account_info) self.storage_cmd('storage blob incremental-copy start --source-container {} --source-blob ' 'src --source-account-name {} --source-account-key {} --source-snapshot ' - '{} --destination-container {} --destination-blob backup', + '{} --destination-container {} --destination-blob backup ' + '--destination-if-modified-since "2020-06-29T06:32Z" ', target_account_info, source_container, source_account, source_account_info[1], snapshot, target_container)