From ac2b1d5830a3fce7eaffdaa53b1966d8971f0c84 Mon Sep 17 00:00:00 2001 From: Fred Park Date: Mon, 22 Jul 2019 16:49:44 +0000 Subject: [PATCH] Support strip components for synccopy - Resolves #103 --- blobxfer/models/options.py | 1 + blobxfer/operations/progress.py | 6 +- blobxfer/operations/synccopy.py | 8 + cli/cli.py | 660 ++++++++++----------- cli/settings.py | 4 + tests/test_blobxfer_models_synccopy.py | 1 + tests/test_blobxfer_operations_azure.py | 1 + tests/test_blobxfer_operations_progress.py | 1 + tests/test_blobxfer_operations_synccopy.py | 17 + 9 files changed, 350 insertions(+), 349 deletions(-) diff --git a/blobxfer/models/options.py b/blobxfer/models/options.py index be50d41..a15a91e 100644 --- a/blobxfer/models/options.py +++ b/blobxfer/models/options.py @@ -126,6 +126,7 @@ SyncCopy = collections.namedtuple( 'recursive', 'rename', 'server_side_copy', + 'strip_components', ] ) diff --git a/blobxfer/operations/progress.py b/blobxfer/operations/progress.py index 4e3d1ad..24f0d69 100644 --- a/blobxfer/operations/progress.py +++ b/blobxfer/operations/progress.py @@ -196,12 +196,12 @@ def output_parameters(general_options, spec): spec.options.recursive)) log.append(' rename single: {}'.format( spec.options.rename)) + log.append(' strip components: {}'.format( + spec.options.strip_components)) # specific epilog if isinstance(spec, blobxfer.models.download.Specification): log.append(' chunk size bytes: {}'.format( spec.options.chunk_size_bytes)) - log.append(' strip components: {}'.format( - spec.options.strip_components)) log.append(' compute file md5: {}'.format( spec.options.check_file_md5)) log.append(' restore properties: attr={} lmt={}'.format( @@ -218,8 +218,6 @@ def output_parameters(general_options, spec): spec.options.chunk_size_bytes)) log.append(' one shot bytes: {}'.format( spec.options.one_shot_bytes)) - log.append(' strip components: {}'.format( - spec.options.strip_components)) if spec.options.store_file_properties.content_type is not None: ct = '\'{}\''.format( spec.options.store_file_properties.content_type) diff --git a/blobxfer/operations/synccopy.py b/blobxfer/operations/synccopy.py index b4063e2..1a08c26 100644 --- a/blobxfer/operations/synccopy.py +++ b/blobxfer/operations/synccopy.py @@ -720,6 +720,14 @@ class SyncCopy(object): name = str(pathlib.Path(name) / tmp) else: name = str(pathlib.Path(name) / src_ase.name) + # apply strip components + if self._spec.options.strip_components > 0: + _rparts = pathlib.Path(name).parts + _strip = min( + (len(_rparts) - 1, self._spec.options.strip_components) + ) + if _strip > 0: + name = str(pathlib.Path(*_rparts[_strip:])) # translate source mode to dest mode dst_mode = self._translate_src_mode_to_dst_mode(src_ase.mode) dst_ase = self._check_for_existing_remote(sa, cont, name, dst_mode) diff --git a/cli/cli.py b/cli/cli.py index 5a36950..d23dec5 100644 --- a/cli/cli.py +++ b/cli/cli.py @@ -137,6 +137,35 @@ class CliContext(object): pass_cli_context = click.make_pass_decorator(CliContext, ensure=True) +def _access_key_option(f): + def callback(ctx, param, value): + clictx = ctx.ensure_object(CliContext) + clictx.cli_options['access_key'] = value + return value + return click.option( + '--storage-account-key', + expose_value=False, + default=None, + help='Storage account access key', + envvar='BLOBXFER_STORAGE_ACCOUNT_KEY', + callback=callback)(f) + + +def _chunk_size_bytes_option(f): + def callback(ctx, param, value): + clictx = ctx.ensure_object(CliContext) + clictx.cli_options['chunk_size_bytes'] = value + return value + return click.option( + '--chunk-size-bytes', + expose_value=False, + type=int, + default=None, + help='Block or chunk size in bytes; set to 0 for auto-select ' + 'on upload [0]', + callback=callback)(f) + + def _config_option(f): def callback(ctx, param, value): clictx = ctx.ensure_object(CliContext) @@ -221,6 +250,47 @@ def _enable_azure_storage_logger_option(f): callback=callback)(f) +def _endpoint_option(f): + def callback(ctx, param, value): + clictx = ctx.ensure_object(CliContext) + clictx.cli_options['endpoint'] = value + return value + return click.option( + '--endpoint', + expose_value=False, + default=None, + help='Azure Storage endpoint [core.windows.net]', + callback=callback)(f) + + +def _exclude_option(f): + def callback(ctx, param, value): + clictx = ctx.ensure_object(CliContext) + clictx.cli_options['exclude'] = value + return value + return click.option( + '--exclude', + expose_value=False, + multiple=True, + default=None, + help='Exclude pattern', + callback=callback)(f) + + +def _include_option(f): + def callback(ctx, param, value): + clictx = ctx.ensure_object(CliContext) + clictx.cli_options['include'] = value + return value + return click.option( + '--include', + expose_value=False, + multiple=True, + default=None, + help='Include pattern', + callback=callback)(f) + + def _log_file_option(f): def callback(ctx, param, value): clictx = ctx.ensure_object(CliContext) @@ -264,6 +334,33 @@ def _md5_processes_option(f): callback=callback)(f) +def _mode_option(f): + def callback(ctx, param, value): + clictx = ctx.ensure_object(CliContext) + clictx.cli_options['mode'] = value + return value + return click.option( + '--mode', + expose_value=False, + default=None, + help='Transfer mode: auto, append, block, file, page [auto]', + callback=callback)(f) + + +def _overwrite_option(f): + def callback(ctx, param, value): + clictx = ctx.ensure_object(CliContext) + clictx.cli_options['overwrite'] = value + return value + return click.option( + '--overwrite/--no-overwrite', + expose_value=False, + default=None, + help='Overwrite destination if exists. For append blobs, ' + '--no-overwrite will append to any existing blob. [True]', + callback=callback)(f) + + def _progress_bar_option(f): def callback(ctx, param, value): clictx = ctx.ensure_object(CliContext) @@ -320,6 +417,20 @@ def _proxy_username_option(f): callback=callback)(f) +def _quiet_option(f): + def callback(ctx, param, value): + clictx = ctx.ensure_object(CliContext) + clictx.cli_options['quiet'] = value + return value + return click.option( + '-q', '--quiet', + expose_value=False, + is_flag=True, + default=None, + help='Quiet mode', + callback=callback)(f) + + def _read_timeout_option(f): def callback(ctx, param, value): clictx = ctx.ensure_object(CliContext) @@ -334,6 +445,21 @@ def _read_timeout_option(f): callback=callback)(f) +def _rename_option(f): + def callback(ctx, param, value): + clictx = ctx.ensure_object(CliContext) + clictx.cli_options['rename'] = value + return value + return click.option( + '--rename', + expose_value=False, + is_flag=True, + default=None, + help='Rename to specified destination for a single object. ' + 'Automatically enabled with stdin source. [False]', + callback=callback)(f) + + def _resume_file_option(f): def callback(ctx, param, value): clictx = ctx.ensure_object(CliContext) @@ -347,6 +473,20 @@ def _resume_file_option(f): callback=callback)(f) +def _sas_option(f): + def callback(ctx, param, value): + clictx = ctx.ensure_object(CliContext) + clictx.cli_options['sas'] = value + return value + return click.option( + '--sas', + expose_value=False, + default=None, + help='Shared access signature', + envvar='BLOBXFER_SAS', + callback=callback)(f) + + def _show_config_option(f): def callback(ctx, param, value): clictx = ctx.ensure_object(CliContext) @@ -360,6 +500,63 @@ def _show_config_option(f): callback=callback)(f) +def _skip_on_filesize_match_option(f): + def callback(ctx, param, value): + clictx = ctx.ensure_object(CliContext) + clictx.cli_options['skip_on_filesize_match'] = value + return value + return click.option( + '--skip-on-filesize-match', + expose_value=False, + is_flag=True, + default=None, + help='Skip on equivalent file size [False]', + callback=callback)(f) + + +def _skip_on_lmt_ge_option(f): + def callback(ctx, param, value): + clictx = ctx.ensure_object(CliContext) + clictx.cli_options['skip_on_lmt_ge'] = value + return value + return click.option( + '--skip-on-lmt-ge', + expose_value=False, + is_flag=True, + default=None, + help='Skip on last modified time greater than or equal to [False]', + callback=callback)(f) + + +def _skip_on_md5_match_option(f): + def callback(ctx, param, value): + clictx = ctx.ensure_object(CliContext) + clictx.cli_options['skip_on_md5_match'] = value + return value + return click.option( + '--skip-on-md5-match', + expose_value=False, + is_flag=True, + default=None, + help='Skip on MD5 match [False]', + callback=callback)(f) + + +def _strip_components_option(f): + def callback(ctx, param, value): + clictx = ctx.ensure_object(CliContext) + clictx.cli_options['strip_components'] = value + return value + return click.option( + '--strip-components', + expose_value=False, + type=int, + default=None, + help='Strip leading file path components: local path for upload, ' + 'remote path for download, or source path for synccopy [0]', + callback=callback)(f) + + def _timeout_option(f): def callback(ctx, param, value): clictx = ctx.ensure_object(CliContext) @@ -403,17 +600,68 @@ def _verbose_option(f): callback=callback)(f) -def _quiet_option(f): +def common_options(f): + f = _verbose_option(f) + f = _transfer_threads_option(f) + f = _timeout_option(f) + f = _strip_components_option(f) + f = _skip_on_md5_match_option(f) + f = _skip_on_lmt_ge_option(f) + f = _skip_on_filesize_match_option(f) + f = _show_config_option(f) + f = _sas_option(f) + f = _resume_file_option(f) + f = _rename_option(f) + f = _read_timeout_option(f) + f = _quiet_option(f) + f = _proxy_username_option(f) + f = _proxy_password_option(f) + f = _proxy_host_option(f) + f = _progress_bar_option(f) + f = _overwrite_option(f) + f = _mode_option(f) + f = _md5_processes_option(f) + f = _max_retries_option(f) + f = _log_file_option(f) + f = _include_option(f) + f = _exclude_option(f) + f = _endpoint_option(f) + f = _enable_azure_storage_logger_option(f) + f = _dry_run_option(f) + f = _disk_threads_option(f) + f = _delete_option(f) + f = _delete_only_option(f) + f = _crypto_processes_option(f) + f = _connect_timeout_option(f) + f = _config_option(f) + f = _chunk_size_bytes_option(f) + f = _access_key_option(f) + return f + + +def _file_attributes(f): def callback(ctx, param, value): clictx = ctx.ensure_object(CliContext) - clictx.cli_options['quiet'] = value + clictx.cli_options['file_attributes'] = value return value return click.option( - '-q', '--quiet', + '--file-attributes/--no-file-attributes', expose_value=False, - is_flag=True, default=None, - help='Quiet mode', + help='Store or restore file attributes [False]', + callback=callback)(f) + + +def _file_md5_option(f): + def callback(ctx, param, value): + clictx = ctx.ensure_object(CliContext) + clictx.cli_options['file_md5'] = value + return value + return click.option( + '--file-md5/--no-file-md5', + expose_value=False, + default=None, + help='Compute file MD5 [False]', callback=callback)(f) @@ -430,6 +678,60 @@ def _local_resource_option(f): callback=callback)(f) +def _recursive_option(f): + def callback(ctx, param, value): + clictx = ctx.ensure_object(CliContext) + clictx.cli_options['recursive'] = value + return value + return click.option( + '--recursive/--no-recursive', + expose_value=False, + default=None, + help='Recursive [True]', + callback=callback)(f) + + +def _remote_path_option(f): + def callback(ctx, param, value): + clictx = ctx.ensure_object(CliContext) + clictx.cli_options['remote_path'] = value + return value + return click.option( + '--remote-path', + expose_value=False, + default=None, + help='Remote path on Azure Storage', + callback=callback)(f) + + +def _rsa_private_key_option(f): + def callback(ctx, param, value): + clictx = ctx.ensure_object(CliContext) + clictx.cli_options['rsa_private_key'] = value + return value + return click.option( + '--rsa-private-key', + expose_value=False, + default=None, + help='RSA private key PEM file', + envvar='BLOBXFER_RSA_PRIVATE_KEY', + callback=callback)(f) + + +def _rsa_private_key_passphrase_option(f): + def callback(ctx, param, value): + clictx = ctx.ensure_object(CliContext) + clictx.cli_options['rsa_private_key_passphrase'] = value + return value + return click.option( + '--rsa-private-key-passphrase', + expose_value=False, + default=None, + help='RSA private key passphrase', + envvar='BLOBXFER_RSA_PRIVATE_KEY_PASSPHRASE', + callback=callback)(f) + + def _storage_account_option(f): def callback(ctx, param, value): clictx = ctx.ensure_object(CliContext) @@ -444,19 +746,6 @@ def _storage_account_option(f): callback=callback)(f) -def _remote_path_option(f): - def callback(ctx, param, value): - clictx = ctx.ensure_object(CliContext) - clictx.cli_options['remote_path'] = value - return value - return click.option( - '--remote-path', - expose_value=False, - default=None, - help='Remote path on Azure Storage', - callback=callback)(f) - - def _storage_url_option(f): def callback(ctx, param, value): clictx = ctx.ensure_object(CliContext) @@ -471,54 +760,19 @@ def _storage_url_option(f): callback=callback)(f) -def common_options(f): - f = _verbose_option(f) - f = _transfer_threads_option(f) - f = _timeout_option(f) - f = _show_config_option(f) - f = _resume_file_option(f) - f = _read_timeout_option(f) - f = _quiet_option(f) - f = _proxy_username_option(f) - f = _proxy_password_option(f) - f = _proxy_host_option(f) - f = _progress_bar_option(f) - f = _md5_processes_option(f) - f = _max_retries_option(f) - f = _log_file_option(f) - f = _enable_azure_storage_logger_option(f) - f = _dry_run_option(f) - f = _disk_threads_option(f) - f = _delete_option(f) - f = _delete_only_option(f) - f = _crypto_processes_option(f) - f = _connect_timeout_option(f) - f = _config_option(f) - return f - - def upload_download_options(f): - f = _remote_path_option(f) f = _storage_url_option(f) f = _storage_account_option(f) + f = _rsa_private_key_passphrase_option(f) + f = _rsa_private_key_option(f) + f = _remote_path_option(f) + f = _recursive_option(f) f = _local_resource_option(f) + f = _file_md5_option(f) + f = _file_attributes(f) return f -def _access_key_option(f): - def callback(ctx, param, value): - clictx = ctx.ensure_object(CliContext) - clictx.cli_options['access_key'] = value - return value - return click.option( - '--storage-account-key', - expose_value=False, - default=None, - help='Storage account access key', - envvar='BLOBXFER_STORAGE_ACCOUNT_KEY', - callback=callback)(f) - - def _access_tier_option(f): def callback(ctx, param, value): clictx = ctx.ensure_object(CliContext) @@ -532,21 +786,6 @@ def _access_tier_option(f): callback=callback)(f) -def _chunk_size_bytes_option(f): - def callback(ctx, param, value): - clictx = ctx.ensure_object(CliContext) - clictx.cli_options['chunk_size_bytes'] = value - return value - return click.option( - '--chunk-size-bytes', - expose_value=False, - type=int, - default=None, - help='Block or chunk size in bytes; set to 0 for auto-select ' - 'on upload [0]', - callback=callback)(f) - - def _delete_option(f): def callback(ctx, param, value): clictx = ctx.ensure_object(CliContext) @@ -590,46 +829,6 @@ def _distribution_mode(f): callback=callback)(f) -def _endpoint_option(f): - def callback(ctx, param, value): - clictx = ctx.ensure_object(CliContext) - clictx.cli_options['endpoint'] = value - return value - return click.option( - '--endpoint', - expose_value=False, - default=None, - help='Azure Storage endpoint [core.windows.net]', - callback=callback)(f) - - -def _exclude_option(f): - def callback(ctx, param, value): - clictx = ctx.ensure_object(CliContext) - clictx.cli_options['exclude'] = value - return value - return click.option( - '--exclude', - expose_value=False, - multiple=True, - default=None, - help='Exclude pattern', - callback=callback)(f) - - -def _file_attributes(f): - def callback(ctx, param, value): - clictx = ctx.ensure_object(CliContext) - clictx.cli_options['file_attributes'] = value - return value - return click.option( - '--file-attributes/--no-file-attributes', - expose_value=False, - default=None, - help='Store or restore file attributes [False]', - callback=callback)(f) - - def _file_cache_control_option(f): def callback(ctx, param, value): clictx = ctx.ensure_object(CliContext) @@ -656,33 +855,6 @@ def _file_content_type_option(f): callback=callback)(f) -def _file_md5_option(f): - def callback(ctx, param, value): - clictx = ctx.ensure_object(CliContext) - clictx.cli_options['file_md5'] = value - return value - return click.option( - '--file-md5/--no-file-md5', - expose_value=False, - default=None, - help='Compute file MD5 [False]', - callback=callback)(f) - - -def _include_option(f): - def callback(ctx, param, value): - clictx = ctx.ensure_object(CliContext) - clictx.cli_options['include'] = value - return value - return click.option( - '--include', - expose_value=False, - multiple=True, - default=None, - help='Include pattern', - callback=callback)(f) - - def _max_single_object_concurrency(f): def callback(ctx, param, value): clictx = ctx.ensure_object(CliContext) @@ -697,19 +869,6 @@ def _max_single_object_concurrency(f): callback=callback)(f) -def _mode_option(f): - def callback(ctx, param, value): - clictx = ctx.ensure_object(CliContext) - clictx.cli_options['mode'] = value - return value - return click.option( - '--mode', - expose_value=False, - default=None, - help='Transfer mode: auto, append, block, file, page [auto]', - callback=callback)(f) - - def _one_shot_bytes_option(f): def callback(ctx, param, value): clictx = ctx.ensure_object(CliContext) @@ -726,48 +885,6 @@ def _one_shot_bytes_option(f): callback=callback)(f) -def _overwrite_option(f): - def callback(ctx, param, value): - clictx = ctx.ensure_object(CliContext) - clictx.cli_options['overwrite'] = value - return value - return click.option( - '--overwrite/--no-overwrite', - expose_value=False, - default=None, - help='Overwrite destination if exists. For append blobs, ' - '--no-overwrite will append to any existing blob. [True]', - callback=callback)(f) - - -def _recursive_option(f): - def callback(ctx, param, value): - clictx = ctx.ensure_object(CliContext) - clictx.cli_options['recursive'] = value - return value - return click.option( - '--recursive/--no-recursive', - expose_value=False, - default=None, - help='Recursive [True]', - callback=callback)(f) - - -def _rename_option(f): - def callback(ctx, param, value): - clictx = ctx.ensure_object(CliContext) - clictx.cli_options['rename'] = value - return value - return click.option( - '--rename', - expose_value=False, - is_flag=True, - default=None, - help='Rename to specified destination for a single object. ' - 'Automatically enabled with stdin source. [False]', - callback=callback)(f) - - def _restore_file_lmt_option(f): def callback(ctx, param, value): clictx = ctx.ensure_object(CliContext) @@ -783,34 +900,6 @@ def _restore_file_lmt_option(f): callback=callback)(f) -def _rsa_private_key_option(f): - def callback(ctx, param, value): - clictx = ctx.ensure_object(CliContext) - clictx.cli_options['rsa_private_key'] = value - return value - return click.option( - '--rsa-private-key', - expose_value=False, - default=None, - help='RSA private key PEM file', - envvar='BLOBXFER_RSA_PRIVATE_KEY', - callback=callback)(f) - - -def _rsa_private_key_passphrase_option(f): - def callback(ctx, param, value): - clictx = ctx.ensure_object(CliContext) - clictx.cli_options['rsa_private_key_passphrase'] = value - return value - return click.option( - '--rsa-private-key-passphrase', - expose_value=False, - default=None, - help='RSA private key passphrase', - envvar='BLOBXFER_RSA_PRIVATE_KEY_PASSPHRASE', - callback=callback)(f) - - def _rsa_public_key_option(f): def callback(ctx, param, value): clictx = ctx.ensure_object(CliContext) @@ -825,20 +914,6 @@ def _rsa_public_key_option(f): callback=callback)(f) -def _sas_option(f): - def callback(ctx, param, value): - clictx = ctx.ensure_object(CliContext) - clictx.cli_options['sas'] = value - return value - return click.option( - '--sas', - expose_value=False, - default=None, - help='Shared access signature', - envvar='BLOBXFER_SAS', - callback=callback)(f) - - def _server_side_copy_option(f): def callback(ctx, param, value): clictx = ctx.ensure_object(CliContext) @@ -852,48 +927,6 @@ def _server_side_copy_option(f): callback=callback)(f) -def _skip_on_filesize_match_option(f): - def callback(ctx, param, value): - clictx = ctx.ensure_object(CliContext) - clictx.cli_options['skip_on_filesize_match'] = value - return value - return click.option( - '--skip-on-filesize-match', - expose_value=False, - is_flag=True, - default=None, - help='Skip on equivalent file size [False]', - callback=callback)(f) - - -def _skip_on_lmt_ge_option(f): - def callback(ctx, param, value): - clictx = ctx.ensure_object(CliContext) - clictx.cli_options['skip_on_lmt_ge'] = value - return value - return click.option( - '--skip-on-lmt-ge', - expose_value=False, - is_flag=True, - default=None, - help='Skip on last modified time greater than or equal to [False]', - callback=callback)(f) - - -def _skip_on_md5_match_option(f): - def callback(ctx, param, value): - clictx = ctx.ensure_object(CliContext) - clictx.cli_options['skip_on_md5_match'] = value - return value - return click.option( - '--skip-on-md5-match', - expose_value=False, - is_flag=True, - default=None, - help='Skip on MD5 match [False]', - callback=callback)(f) - - def _stdin_as_page_blob_size_option(f): def callback(ctx, param, value): clictx = ctx.ensure_object(CliContext) @@ -908,21 +941,6 @@ def _stdin_as_page_blob_size_option(f): callback=callback)(f) -def _strip_components_option(f): - def callback(ctx, param, value): - clictx = ctx.ensure_object(CliContext) - clictx.cli_options['strip_components'] = value - return value - return click.option( - '--strip-components', - expose_value=False, - type=int, - default=None, - help='Strip leading file path components: local path for upload ' - 'or remote path for download [0]', - callback=callback)(f) - - def _stripe_chunk_size_bytes_option(f): def callback(ctx, param, value): clictx = ctx.ensure_object(CliContext) @@ -1034,55 +1052,19 @@ def _sync_copy_source_url(f): def upload_options(f): f = _stripe_chunk_size_bytes_option(f) - f = _strip_components_option(f) f = _stdin_as_page_blob_size_option(f) - f = _skip_on_md5_match_option(f) - f = _skip_on_lmt_ge_option(f) - f = _skip_on_filesize_match_option(f) - f = _sas_option(f) f = _rsa_public_key_option(f) - f = _rsa_private_key_passphrase_option(f) - f = _rsa_private_key_option(f) - f = _rename_option(f) - f = _recursive_option(f) - f = _overwrite_option(f) f = _one_shot_bytes_option(f) - f = _mode_option(f) - f = _include_option(f) - f = _file_md5_option(f) f = _file_content_type_option(f) f = _file_cache_control_option(f) - f = _file_attributes(f) - f = _exclude_option(f) - f = _endpoint_option(f) f = _distribution_mode(f) - f = _chunk_size_bytes_option(f) f = _access_tier_option(f) - f = _access_key_option(f) return f def download_options(f): - f = _strip_components_option(f) - f = _skip_on_md5_match_option(f) - f = _skip_on_lmt_ge_option(f) - f = _skip_on_filesize_match_option(f) - f = _sas_option(f) - f = _rsa_private_key_passphrase_option(f) - f = _rsa_private_key_option(f) f = _restore_file_lmt_option(f) - f = _rename_option(f) - f = _recursive_option(f) - f = _overwrite_option(f) - f = _mode_option(f) f = _max_single_object_concurrency(f) - f = _include_option(f) - f = _file_md5_option(f) - f = _file_attributes(f) - f = _exclude_option(f) - f = _endpoint_option(f) - f = _chunk_size_bytes_option(f) - f = _access_key_option(f) return f @@ -1095,21 +1077,9 @@ def sync_copy_options(f): f = _sync_copy_dest_mode_option(f) f = _sync_copy_dest_access_key_option(f) f = _storage_account_option(f) - f = _skip_on_md5_match_option(f) - f = _skip_on_lmt_ge_option(f) - f = _skip_on_filesize_match_option(f) f = _server_side_copy_option(f) - f = _sas_option(f) - f = _rename_option(f) f = _remote_path_option(f) - f = _overwrite_option(f) - f = _mode_option(f) - f = _include_option(f) - f = _exclude_option(f) - f = _endpoint_option(f) - f = _chunk_size_bytes_option(f) f = _access_tier_option(f) - f = _access_key_option(f) return f @@ -1122,8 +1092,8 @@ def cli(ctx): @cli.command('download') -@upload_download_options @download_options +@upload_download_options @common_options @pass_cli_context def download(ctx): @@ -1157,8 +1127,8 @@ def synccopy(ctx): @cli.command('upload') -@upload_download_options @upload_options +@upload_download_options @common_options @pass_cli_context def upload(ctx): diff --git a/cli/settings.py b/cli/settings.py index d8edbc4..f0075de 100644 --- a/cli/settings.py +++ b/cli/settings.py @@ -211,6 +211,7 @@ def add_cli_options(cli_options, action): 'lmt_ge': cli_options.get('skip_on_lmt_ge'), 'md5_match': cli_options.get('skip_on_md5_match'), }, + 'strip_components': cli_options.get('strip_components'), }, } if storage_account is not None: @@ -681,6 +682,9 @@ def create_synccopy_specifications(ctx_cli_options, config): server_side_copy=_merge_setting( cli_options, conf_options, 'server_side_copy', default=True), + strip_components=_merge_setting( + cli_options, conf_options, 'strip_components', + default=0), ), skip_on_options=blobxfer.models.options.SkipOn( filesize_match=_merge_setting( diff --git a/tests/test_blobxfer_models_synccopy.py b/tests/test_blobxfer_models_synccopy.py index a32232e..b4003d9 100644 --- a/tests/test_blobxfer_models_synccopy.py +++ b/tests/test_blobxfer_models_synccopy.py @@ -28,6 +28,7 @@ def test_specification(): recursive=True, rename=False, server_side_copy=True, + strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=True, diff --git a/tests/test_blobxfer_operations_azure.py b/tests/test_blobxfer_operations_azure.py index 7efe170..ec5629b 100644 --- a/tests/test_blobxfer_operations_azure.py +++ b/tests/test_blobxfer_operations_azure.py @@ -616,6 +616,7 @@ def test_azuresourcepath_url(): recursive=None, rename=None, server_side_copy=True, + strip_components=0, ) i = 0 diff --git a/tests/test_blobxfer_operations_progress.py b/tests/test_blobxfer_operations_progress.py index 98923d2..322a32c 100644 --- a/tests/test_blobxfer_operations_progress.py +++ b/tests/test_blobxfer_operations_progress.py @@ -131,6 +131,7 @@ def test_output_parameters(): recursive=True, rename=False, server_side_copy=True, + strip_components=0, ), skip_on_options=options.SkipOn( filesize_match=True, diff --git a/tests/test_blobxfer_operations_synccopy.py b/tests/test_blobxfer_operations_synccopy.py index 07f2d11..bdeaa8c 100644 --- a/tests/test_blobxfer_operations_synccopy.py +++ b/tests/test_blobxfer_operations_synccopy.py @@ -661,6 +661,7 @@ def test_check_copy_conditions(gmfm): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False s._spec.options.server_side_copy = False + s._spec.options.strip_components = 0 src_ase = mock.MagicMock() src_ase._client.primary_endpoint = 'ep' @@ -733,6 +734,7 @@ def test_check_for_existing_remote(gbp, gfp): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False s._spec.options.server_side_copy = False + s._spec.options.strip_components = 0 sa = mock.MagicMock() sa.name = 'name' @@ -776,6 +778,8 @@ def test_get_destination_paths(): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False s._spec.options.server_side_copy = False + s._spec.options.strip_components = 0 + paths = mock.MagicMock() paths.paths = [pathlib.Path('a/b')] s._spec.destinations = [paths] @@ -790,6 +794,7 @@ def test_generate_destination_for_source(): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False s._spec.options.server_side_copy = False + s._spec.options.strip_components = 0 s._spec.options.dest_mode = azmodels.StorageModes.Block s._spec.options.rename = False s._check_for_existing_remote = mock.MagicMock() @@ -856,11 +861,21 @@ def test_generate_destination_for_source(): ase = next(s._generate_destination_for_source(src_ase)) assert pathlib.Path(ase.name) == pathlib.Path('name/remote/path') + # test strip components + s._get_destination_paths.return_value = [ + (sa, 'cont', 'name', 'dpath'), + ] + s._spec.options.strip_components = 1 + src_ase.is_arbitrary_url = True + ase = next(s._generate_destination_for_source(src_ase)) + assert pathlib.Path(ase.name) == pathlib.Path('remote/path') + def test_bind_sources_to_destination(): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False s._spec.options.server_side_copy = False + s._spec.options.strip_components = 0 s._spec.options.delete_extraneous_destination = True src_ase = mock.MagicMock() @@ -925,6 +940,7 @@ def test_run(srm, gbr, gfr): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False s._spec.options.server_side_copy = False + s._spec.options.strip_components = 0 s._general_options.concurrency.transfer_threads = 1 s._general_options.resume_file = 'resume' s._spec.options.chunk_size_bytes = 0 @@ -1004,6 +1020,7 @@ def test_start(): s._general_options.dry_run = False s._spec.options.delete_only = False s._spec.options.server_side_copy = False + s._spec.options.strip_components = 0 s._wait_for_transfer_threads = mock.MagicMock() s._resume = mock.MagicMock() s._run = mock.MagicMock()