Fix transfers with non-ASCII filenames (#63)

Fixes #62

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: AzureStor
 Title: Storage Management in 'Azure'
-Version: 3.2.2
+Version: 3.2.2.9000
 Authors@R: c(
     person("Hong", "Ooi", , "hongooi@microsoft.com", role = c("aut", "cre")),
     person("Microsoft", role="cph")

NEWS.md

@@ -1,3 +1,7 @@
+# AzureStor 3.2.2.9000
+
+- Fix file transfers with filenames containing non-ASCII characters on Windows.
+
 # AzureStor 3.2.2
 
 - Extended support for directories in blob storage. Note that since blob storage doesn't have true directories, there are some warts to be aware of; see `?blob` for more details.

R/sas.R

@@ -131,7 +131,7 @@ get_account_sas.default <- function(account, key, start=NULL, expiry=NULL, servi
         sig=sig
     )
     parts <- parts[!sapply(parts, is_empty)]
-    parts <- sapply(parts, utils::URLencode, reserved=TRUE)
+    parts <- sapply(parts, url_encode, reserved=TRUE)
     paste(names(parts), parts, sep="=", collapse="&")
 }
 
@@ -264,7 +264,7 @@ get_user_delegation_sas.default <- function(account, key, resource, start=NULL,
         sig=sig
     )
     parts <- parts[!sapply(parts, is_empty)]
-    parts <- sapply(parts, utils::URLencode, reserved=TRUE)
+    parts <- sapply(parts, url_encode, reserved=TRUE)
     paste(names(parts), parts, sep="=", collapse="&")
 }
 

R/storage_utils.R

@@ -57,7 +57,7 @@ call_storage_endpoint <- function(endpoint, path, options=list(), headers=list()
 {
     http_verb <- match.arg(http_verb)
     url <- httr::parse_url(endpoint$url)
-    url$path <- URLencode(path)
+    url$path <- url_encode(path)
     if(!is_empty(options))
         url$query <- options[order(names(options))] # must be sorted for access key signing
 
@@ -262,3 +262,8 @@ sign_sha256 <- function(string, key)
     openssl::base64_encode(openssl::sha256(charToRaw(string), openssl::base64_decode(key)))
 }
 
+
+url_encode <- function(string, reserved=FALSE)
+{
+    URLencode(enc2utf8(string), reserved=reserved)
+}

tests/testthat/test11_enc.R

@@ -0,0 +1,85 @@
+context("Filename encoding")
+
+tenant <- Sys.getenv("AZ_TEST_TENANT_ID")
+app <- Sys.getenv("AZ_TEST_APP_ID")
+password <- Sys.getenv("AZ_TEST_PASSWORD")
+subscription <- Sys.getenv("AZ_TEST_SUBSCRIPTION")
+
+if(tenant == "" || app == "" || password == "" || subscription == "")
+    skip("Authentication tests skipped: ARM credentials not set")
+
+rgname <- Sys.getenv("AZ_TEST_STORAGE_RG")
+storname <- Sys.getenv("AZ_TEST_STORAGE_HNS")
+
+if(rgname == "" || storname == "")
+    skip("Filename encoding tests skipped: resource names not set")
+
+sub <- AzureRMR::az_rm$new(tenant=tenant, app=app, password=password)$get_subscription(subscription)
+stor <- sub$get_resource_group(rgname)$get_storage_account(storname)
+options(azure_storage_progress_bar=FALSE)
+
+fname <- paste0(paste0(sample(letters, 10, replace=TRUE), collapse=""), "srcáé.txt")
+pathname <- file.path(tempdir(), fname)  # don't use basename() because that sets encoding to UTF-8
+writeLines(letters, pathname)
+
+
+test_that("Blob transfer works with non-ASCII filename",
+{
+    bl <- stor$get_blob_endpoint()
+    contname <- paste0(sample(letters, 10, TRUE), collapse="")
+    cont <- create_blob_container(bl, contname)
+
+    expect_silent(upload_blob(cont, pathname))
+    expect_silent(upload_blob(cont, pathname, fname))
+    expect_silent(upload_blob(cont, pathname, "srcáé2.txt"))
+
+    destname <- paste0(paste0(sample(letters, 10, replace=TRUE), collapse=""), "destáé.txt")
+    expect_silent(download_blob(cont, fname, file.path(tempdir(), destname)))
+})
+
+
+test_that("File transfer works with non-ASCII filename",
+{
+    fl <- stor$get_file_endpoint()
+    sharename <- paste0(sample(letters, 10, TRUE), collapse="")
+    share <- create_file_share(fl, sharename)
+
+    expect_silent(upload_azure_file(share, pathname))
+    expect_silent(upload_azure_file(share, pathname, fname))
+    expect_silent(upload_azure_file(share, pathname, "srcáé2.txt"))
+
+    destname <- paste0(paste0(sample(letters, 10, replace=TRUE), collapse=""), "destáé.txt")
+    expect_silent(download_azure_file(share, fname, file.path(tempdir(), destname)))
+
+})
+
+
+test_that("ADLS2 transfer works with non-ASCII filename",
+{
+    ad <- stor$get_adls_endpoint()
+    fsname <- paste0(sample(letters, 10, TRUE), collapse="")
+    fs <- create_adls_filesystem(ad, fsname)
+
+    expect_silent(upload_adls_file(fs, pathname))
+    expect_silent(upload_adls_file(fs, pathname, fname))
+    expect_silent(upload_adls_file(fs, pathname, "srcáé2.txt"))
+
+    destname <- paste0(paste0(sample(letters, 10, replace=TRUE), collapse=""), "destáé.txt")
+    expect_silent(download_adls_file(fs, fname, file.path(tempdir(), destname)))
+})
+
+
+teardown(
+{
+    ad <- stor$get_adls_endpoint()
+    fss <- list_adls_filesystems(ad)
+    lapply(fss, delete_adls_filesystem, confirm=FALSE)
+
+    bl <- stor$get_blob_endpoint()
+    conts <- list_blob_containers(bl)
+    lapply(conts, delete_blob_container, confirm=FALSE)
+
+    fl <- stor$get_file_endpoint()
+    shares <- list_file_shares(fl)
+    lapply(shares, delete_file_share, confirm=FALSE)
+})