set default dest for file transfers

NEWS.md

@@ -6,6 +6,7 @@
   - These can now accept a vector of pathnames as the source and destination arguments.
   - Alternatively, for a wildcard source, add the ability to recurse through subdirectories. Any directory structure in the source will be reproduced at the destination.
 - Related to the above: the file transfer methods can now create subdirectories that are specified in their destination argument. For ADLS and blob uploading this happens automatically; for Azure file uploading it requires a separate API call which can be slow, so is optional.
+- The default destination directory when transferring files (not connections) is now the (remote) root for uploading, and the (local) current directory for downloading.
 - Significant changes to file storage methods for greater consistency with the other storage types:
   - The default directory for `list_azure_files` is now the root, mirroring the behaviour for blobs and ADLSgen2.
   - The output of `list_azure_files` now includes the full path as part of the file/directory name.

R/adls_client_funcs.R

@@ -336,7 +336,7 @@ list_adls_files <- function(filesystem, dir="/", info=c("all", "name"),
 
 #' @rdname adls
 #' @export
-multiupload_adls_file <- function(filesystem, src, dest="/", recursive=FALSE, blocksize=2^22, lease=NULL,
+multiupload_adls_file <- function(filesystem, src, dest, recursive=FALSE, blocksize=2^22, lease=NULL,
                                    use_azcopy=FALSE,
                                    max_concurrent_transfers=10)
 {
@@ -350,7 +350,7 @@ multiupload_adls_file <- function(filesystem, src, dest="/", recursive=FALSE, bl
 
 #' @rdname adls
 #' @export
-upload_adls_file <- function(filesystem, src, dest, blocksize=2^24, lease=NULL, use_azcopy=FALSE)
+upload_adls_file <- function(filesystem, src, dest=basename(src), blocksize=2^24, lease=NULL, use_azcopy=FALSE)
 {
     if(use_azcopy)
         azcopy_upload(filesystem, src, dest, blocksize=blocksize, lease=lease)
@@ -374,7 +374,7 @@ multidownload_adls_file <- function(filesystem, src, dest, recursive=FALSE, bloc
 
 #' @rdname adls
 #' @export
-download_adls_file <- function(filesystem, src, dest, blocksize=2^24, overwrite=FALSE, use_azcopy=FALSE)
+download_adls_file <- function(filesystem, src, dest=basename(src), blocksize=2^24, overwrite=FALSE, use_azcopy=FALSE)
 {
     if(use_azcopy)
         azcopy_download(filesystem, src, dest, overwrite=overwrite)
@@ -382,7 +382,6 @@ download_adls_file <- function(filesystem, src, dest, blocksize=2^24, overwrite=
 }
 
 
-
 #' @rdname adls
 #' @export
 delete_adls_file <- function(filesystem, file, confirm=TRUE)
@@ -394,7 +393,6 @@ delete_adls_file <- function(filesystem, file, confirm=TRUE)
 }
 
 
-
 #' @rdname adls
 #' @export
 create_adls_dir <- function(filesystem, dir)

R/blob_client_funcs.R

@@ -350,7 +350,7 @@ list_blobs <- function(container, dir="/", info=c("partial", "name", "all"),
 
 #' @rdname blob
 #' @export
-upload_blob <- function(container, src, dest, type="BlockBlob", blocksize=2^24, lease=NULL,
+upload_blob <- function(container, src, dest=basename(src), type="BlockBlob", blocksize=2^24, lease=NULL,
                         use_azcopy=FALSE)
 {
     if(use_azcopy)
@@ -360,7 +360,7 @@ upload_blob <- function(container, src, dest, type="BlockBlob", blocksize=2^24,
 
 #' @rdname blob
 #' @export
-multiupload_blob <- function(container, src, dest="/", recursive=FALSE, type="BlockBlob", blocksize=2^24, lease=NULL,
+multiupload_blob <- function(container, src, dest, recursive=FALSE, type="BlockBlob", blocksize=2^24, lease=NULL,
                              use_azcopy=FALSE,
                              max_concurrent_transfers=10)
 {
@@ -373,7 +373,7 @@ multiupload_blob <- function(container, src, dest="/", recursive=FALSE, type="Bl
 
 #' @rdname blob
 #' @export
-download_blob <- function(container, src, dest, blocksize=2^24, overwrite=FALSE, lease=NULL,
+download_blob <- function(container, src, dest=basename(src), blocksize=2^24, overwrite=FALSE, lease=NULL,
                           use_azcopy=FALSE)
 {
     if(use_azcopy)

R/blob_copyurl.R

@@ -54,8 +54,11 @@ copy_url_to_blob <- function(container, src, dest, lease=NULL, async=FALSE)
 
 #' @rdname blob
 #' @export
-multicopy_url_to_blob <- function(container, src, dest="/", lease=NULL, async=FALSE, max_concurrent_transfers=10)
+multicopy_url_to_blob <- function(container, src, dest, lease=NULL, async=FALSE, max_concurrent_transfers=10)
 {
+    if(missing(dest))
+        dest <- basename(src)
+
     n_src <- length(src)
     n_dest <- length(dest)
 

R/file_client_funcs.R

@@ -306,7 +306,7 @@ list_azure_files <- function(share, dir="/", info=c("all", "name"),
 
 #' @rdname file
 #' @export
-upload_azure_file <- function(share, src, dest, create_dir=FALSE, blocksize=2^22, use_azcopy=FALSE)
+upload_azure_file <- function(share, src, dest=basename(src), create_dir=FALSE, blocksize=2^22, use_azcopy=FALSE)
 {
     if(use_azcopy)
         azcopy_upload(share, src, dest, blocksize=blocksize)
@@ -315,7 +315,7 @@ upload_azure_file <- function(share, src, dest, create_dir=FALSE, blocksize=2^22
 
 #' @rdname file
 #' @export
-multiupload_azure_file <- function(share, src, dest="/", recursive=FALSE, create_dir=recursive, blocksize=2^22,
+multiupload_azure_file <- function(share, src, dest, recursive=FALSE, create_dir=recursive, blocksize=2^22,
                                    use_azcopy=FALSE,
                                    max_concurrent_transfers=10)
 {
@@ -328,7 +328,7 @@ multiupload_azure_file <- function(share, src, dest="/", recursive=FALSE, create
 
 #' @rdname file
 #' @export
-download_azure_file <- function(share, src, dest, blocksize=2^22, overwrite=FALSE, use_azcopy=FALSE)
+download_azure_file <- function(share, src, dest=basename(src), blocksize=2^22, overwrite=FALSE, use_azcopy=FALSE)
 {
     if(use_azcopy)
         azcopy_download(share, src, dest, overwrite=overwrite)

R/transfer_utils.R

@@ -44,9 +44,14 @@ normalize_src.rawConnection <- function(src)
 multiupload_internal <- function(container, src, dest, recursive, ..., max_concurrent_transfers=10)
 {
     src <- make_upload_set(src, recursive)
+    wildcard_src <- !is.null(attr(src, "root"))
+    if(missing(dest))
+    {
+        dest <- if(wildcard_src) "/" else basename(src)
+    }
+
     n_src <- length(src)
     n_dest <- length(dest)
-    wildcard_src <- !is.null(attr(src, "root"))
 
     if(n_src == 0)
         stop("No files to transfer", call.=FALSE)
@@ -75,9 +80,14 @@ multiupload_internal <- function(container, src, dest, recursive, ..., max_concu
 multidownload_internal <- function(container, src, dest, recursive, ..., max_concurrent_transfers=10)
 {
     src <- make_download_set(container, src, recursive)
+    wildcard_src <- !is.null(attr(src, "root"))
+    if(missing(dest))
+    {
+        dest <- if(wildcard_src) "." else basename(src)
+    }
+
     n_src <- length(src)
     n_dest <- length(dest)
-    wildcard_src <- !is.null(attr(src, "root"))
 
     if(n_src == 0)
         stop("No files to transfer", call.=FALSE)

tests/testthat/test02_blob.R

@@ -263,6 +263,20 @@ test_that("chunked downloading works",
 })
 
 
+test_that("Default destination works",
+{
+    bl <- stor$get_blob_endpoint()
+    cont <- create_blob_container(bl, "defaultdest")
+
+    orig_file <- "../resources/iris.csv"
+    upload_blob(cont, orig_file)
+    download_blob(cont, basename(orig_file))
+
+    expect_true(file.exists(basename(orig_file)))
+    file.remove(basename(orig_file))
+})
+
+
 test_that("copy from url works",
 {
     bl <- stor$get_blob_endpoint()

tests/testthat/test02a_blobext.R

@@ -113,6 +113,32 @@ test_that("Blob recursive wildcard multitransfer works",
 })
 
 
+test_that("Default multitransfer destination works",
+{
+    contname <- paste0(sample(letters, 10, TRUE), collapse="")
+    cont <- create_blob_container(bl, contname)
+
+    multiupload_blob(cont, file.path(srcdir, srcs))
+    multidownload_blob(cont, srcs)
+
+    expect_true(files_identical(file.path(srcdir, srcs), srcs))
+    file.remove(srcs)
+})
+
+
+test_that("Default multitransfer destination works with wildcard src",
+{
+    contname <- paste0(sample(letters, 10, TRUE), collapse="")
+    cont <- create_blob_container(bl, contname)
+
+    multiupload_blob(cont, file.path(srcdir, "subdir/*"))
+    multidownload_blob(cont, "*")
+
+    expect_true(files_identical(file.path(srcdir, "subdir", srcs_sub), srcs_sub))
+    file.remove(srcs_sub)
+})
+
+
 test_that("Blob multicopy from URL works",
 {
     contname <- paste0(sample(letters, 10, TRUE), collapse="")
@@ -133,6 +159,7 @@ test_that("Blob multicopy from URL works",
     }, dests, origs)))
 })
 
+
 teardown(
 {
     conts <- list_blob_containers(bl)

tests/testthat/test03_file.R

@@ -248,6 +248,20 @@ test_that("chunked downloading works",
 })
 
 
+test_that("Default destination works",
+{
+    fl <- stor$get_file_endpoint()
+    cont <- create_file_share(fl, "defaultdest")
+
+    orig_file <- "../resources/iris.csv"
+    upload_azure_file(cont, orig_file)
+    download_azure_file(cont, basename(orig_file))
+
+    expect_true(file.exists(basename(orig_file)))
+    file.remove(basename(orig_file))
+})
+
+
 teardown(
 {
     fl <- stor$get_file_endpoint()

tests/testthat/test03a_fileext.R

@@ -113,6 +113,32 @@ test_that("File recursive wildcard multitransfer works",
 })
 
 
+test_that("Default multitransfer destination works",
+{
+    contname <- paste0(sample(letters, 10, TRUE), collapse="")
+    cont <- create_file_share(fl, contname)
+
+    multiupload_azure_file(cont, file.path(srcdir, srcs))
+    multidownload_azure_file(cont, srcs)
+
+    expect_true(files_identical(file.path(srcdir, srcs), srcs))
+    file.remove(srcs)
+})
+
+
+test_that("Default multitransfer destination works with wildcard src",
+{
+    contname <- paste0(sample(letters, 10, TRUE), collapse="")
+    cont <- create_file_share(fl, contname)
+
+    multiupload_azure_file(cont, file.path(srcdir, "subdir/*"))
+    multidownload_azure_file(cont, "*")
+
+    expect_true(files_identical(file.path(srcdir, "subdir", srcs_sub), srcs_sub))
+    file.remove(srcs_sub)
+})
+
+
 teardown(
 {
     conts <- list_file_shares(fl)

tests/testthat/test04_adls.R

@@ -255,6 +255,20 @@ test_that("chunked downloading works",
 })
 
 
+test_that("Default destination works",
+{
+    ad <- stor$get_adls_endpoint()
+    cont <- create_adls_filesystem(ad, "defaultdest")
+
+    orig_file <- "../resources/iris.csv"
+    upload_adls_file(cont, orig_file)
+    download_adls_file(cont, basename(orig_file))
+
+    expect_true(file.exists(basename(orig_file)))
+    file.remove(basename(orig_file))
+})
+
+
 teardown(
 {
     ad <- stor$get_adls_endpoint()

tests/testthat/test04a_adlsext.R

@@ -110,6 +110,32 @@ test_that("ADLS recursive wildcard multitransfer works",
 })
 
 
+test_that("Default multitransfer destination works",
+{
+    contname <- paste0(sample(letters, 10, TRUE), collapse="")
+    cont <- create_adls_filesystem(ad, contname)
+
+    multiupload_adls_file(cont, file.path(srcdir, srcs))
+    multidownload_adls_file(cont, srcs)
+
+    expect_true(files_identical(file.path(srcdir, srcs), srcs))
+    file.remove(srcs)
+})
+
+
+test_that("Default multitransfer destination works with wildcard src",
+{
+    contname <- paste0(sample(letters, 10, TRUE), collapse="")
+    cont <- create_adls_filesystem(ad, contname)
+
+    multiupload_adls_file(cont, file.path(srcdir, "subdir/*"))
+    multidownload_adls_file(cont, "*")
+
+    expect_true(files_identical(file.path(srcdir, "subdir", srcs_sub), srcs_sub))
+    file.remove(srcs_sub)
+})
+
+
 teardown(
 {
     conts <- list_adls_filesystems(ad)