Emulator support (#68)

Adds support for the Azurite and Azure SDK storage emulators, closes #66

.Rbuildignore

@@ -9,3 +9,6 @@ drat.sh
 .travis.yml
 ^LICENSE\.md$
 azure-pipelines.yml
+^__azurite
+^__blobstorage__
+^__queuestorage__

.gitignore

@@ -262,4 +262,6 @@ __pycache__/
 
 misc/
 .Rhistory
-
+__azurite*
+__blobstorage__
+__queuestorage__

NEWS.md

@@ -1,3 +1,9 @@
+# AzureStor 3.2.3.9000
+
+- Add support for the [Azurite](https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite) and [Azure SDK](https://docs.microsoft.com/en-us/azure/storage/common/storage-use-emulator) storage emulators. To connect to the endpoint, use the service-specific functions `blob_endpoint` and `queue_endpoint` (the latter from the AzureQstor package), passing the full URL including the account name: `blob_endpoint("http://127.0.0.1:10000/myaccount", key="mykey")`. The warning about an unrecognised endpoint can be ignored. See the linked pages for full details on how to authenticate to the emulator.
+
+  Note that the Azure SDK emulator is no longer being actively developed; it's recommended to use Azurite.
+
 # AzureStor 3.2.3
 
 - Fix file transfers with filenames containing non-ASCII characters on Windows.

R/client_endpoint.R

@@ -15,6 +15,11 @@
 #'
 #' If multiple authentication objects are supplied, they are used in this order of priority: first an access key, then an AAD token, then a SAS. If no authentication objects are supplied, only public (anonymous) access to the endpoint is possible.
 #'
+#' @section Storage emulators:
+#' AzureStor supports connecting to the [Azure SDK](https://docs.microsoft.com/en-us/azure/storage/common/storage-use-emulator) and [Azurite](https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite) emulators for blob and queue storage. To connect, pass the full URL of the endpoint, including the account name, to the `blob_endpoint` and `queue_endpoint` methods (the latter from the AzureQstor package). The warning about an unrecognised endpoint can be ignored. See the linked pages, and the examples below, for details on how to authenticate with the emulator.
+#'
+#' Note that the Azure SDK emulator is no longer being actively developed; it's recommended to use Azurite for development work.
+#'
 #' @return
 #' `storage_endpoint` returns an object of S3 class `"adls_endpoint"`, `"blob_endpoint"`, `"file_endpoint"`, `"queue_endpoint"` or `"table_endpoint"` depending on the type of endpoint. All of these also inherit from class `"storage_endpoint"`. `adls_endpoint`, `blob_endpoint` and `file_endpoint` return an object of the respective class.
 #'
@@ -41,6 +46,19 @@
 #'                                     "myaadtenant", "app_id", "password")
 #' adls_endpoint("https://myadlsstorage.dfs.core.windows.net/", token=token)
 #'
+#'
+#' ## Azurite storage emulator:
+#'
+#' # connecting to Azurite with the default account and key (these also work for the Azure SDK)
+#' azurite_account <- "devstoreaccount1"
+#' azurite_key <-
+#'    "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+#' blob_endpoint(paste0("http://127.0.0.1:10000/", azurite_account), key=azurite_key)
+#'
+#' # to use a custom account name and key, set the AZURITE_ACCOUNTS env var before starting Azurite
+#' Sys.setenv(AZURITE_ACCOUNTS="account1:key1")
+#' blob_endpoint("http://127.0.0.1:10000/account1", key="key1")
+#'
 #' }
 #' @aliases endpoint blob_endpoint file_endpoint queue_endpoint table_endpoint
 #' @export

R/sign_request.R

@@ -14,7 +14,9 @@ sign_request <- function(endpoint, ...)
 
 sign_request.default <- function(endpoint, verb, url, headers, api, ...)
 {
-    acct_name <- sub("\\..+$", "", url$host)
+    host_url <- httr::parse_url(endpoint$url)
+    acct_name <- if(host_url$path == "") sub("\\..+$", "", host_url$host) else host_url$path
+
     resource <- paste0("/", acct_name, "/", url$path) # don't use file.path because it strips trailing / on Windows
     # sanity check
     resource <- gsub("//", "/", resource)

R/storage_utils.R

@@ -38,9 +38,8 @@
 #' @export
 do_container_op <- function(container, operation="", options=list(), headers=list(), http_verb="GET", ...)
 {
-    # don't add trailing / if no within-container path supplied: ADLS will complain
     operation <- if(nchar(operation) > 0)
-        sub("//", "/", paste0(container$name, "/", operation))
+        paste0(container$name, "/", operation)
     else container$name
 
     call_storage_endpoint(container$endpoint, operation, options=options, headers=headers, http_verb=http_verb, ...)
@@ -57,7 +56,8 @@ call_storage_endpoint <- function(endpoint, path, options=list(), headers=list()
 {
     http_verb <- match.arg(http_verb)
     url <- httr::parse_url(endpoint$url)
-    url$path <- url_encode(path)
+    # fix doubled-up /'s which can result from file.path snafus etc
+    url$path <- gsub("/{2,}", "/", url_encode(file.path(url$path, path)))
     if(!is_empty(options))
         url$query <- options[order(names(options))] # must be sorted for access key signing
 

man/storage_endpoint.Rd

@@ -55,6 +55,13 @@ This is the starting point for the client-side storage interface in AzureRMR. \c
 
 If multiple authentication objects are supplied, they are used in this order of priority: first an access key, then an AAD token, then a SAS. If no authentication objects are supplied, only public (anonymous) access to the endpoint is possible.
 }
+\section{Storage emulators}{
+
+AzureStor supports connecting to the \href{https://docs.microsoft.com/en-us/azure/storage/common/storage-use-emulator}{Azure SDK} and \href{https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite}{Azurite} emulators for blob and queue storage. To connect, pass the full URL of the endpoint, including the account name, to the \code{blob_endpoint} and \code{queue_endpoint} methods (the latter from the AzureQstor package). The warning about an unrecognised endpoint can be ignored. See the linked pages, and the examples below, for details on how to authenticate with the emulator.
+
+Note that the Azure SDK emulator is no longer being actively developed; it's recommended to use Azurite for development work.
+}
+
 \examples{
 \dontrun{
 
@@ -73,6 +80,19 @@ token <- AzureAuth::get_azure_token("https://storage.azure.com",
                                     "myaadtenant", "app_id", "password")
 adls_endpoint("https://myadlsstorage.dfs.core.windows.net/", token=token)
 
+
+## Azurite storage emulator:
+
+# connecting to Azurite with the default account and key (these also work for the Azure SDK)
+azurite_account <- "devstoreaccount1"
+azurite_key <-
+   "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+blob_endpoint(paste0("http://127.0.0.1:10000/", azurite_account), key=azurite_key)
+
+# to use a custom account name and key, set the AZURITE_ACCOUNTS env var before starting Azurite
+Sys.setenv(AZURITE_ACCOUNTS="account1:key1")
+blob_endpoint("http://127.0.0.1:10000/account1", key="key1")
+
 }
 }
 \seealso{

tests/testthat/test12_azurite.R

@@ -0,0 +1,33 @@
+context("Azurite storage emulator")
+
+# currently hardcoded to use account name=account1, shared key=key1
+res <- try(httr::GET("http://127.0.0.1:10000/account1"), silent=TRUE)
+if(inherits(res, "try-error"))
+    skip("Storage emulator tests skipped: Azurite not running")
+
+opts <- options(azure_storage_progress_bar=FALSE)
+
+test_that("Blob storage methods work",
+{
+    expect_warning(endp <- blob_endpoint("http://127.0.0.1:10000/account1", key="key1"))
+    expect_is(endp, "blob_endpoint")
+
+    expect_is(list_blob_containers(endp), "list")
+    cont <- create_blob_container(endp, "container1")
+    expect_is(cont, "blob_container")
+
+    expect_silent(upload_blob(cont, "../resources/iris.csv"))
+    expect_is(list_blobs(cont), "data.frame")
+    expect_silent(download_blob(cont, "iris.csv", tempfile()))
+
+    expect_silent(delete_blob_container(cont, confirm=FALSE))
+    expect_true(is_empty(list_blob_containers(endp)))
+})
+
+
+teardown({
+    endp <- suppressWarnings(blob_endpoint("http://127.0.0.1:10000/account1", key="key1"))
+    conts <- list_blob_containers(endp)
+    lapply(conts, delete_blob_container, confirm=FALSE)
+    options(opts)
+})