Adds support for the Azurite and Azure SDK storage emulators, closes #66
This commit is contained in:
Hong Ooi 2020-08-24 16:31:39 +10:00 коммит произвёл GitHub
Родитель 47bad2ddb9
Коммит b0dcdff533
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
8 изменённых файлов: 89 добавлений и 5 удалений

Просмотреть файл

@ -9,3 +9,6 @@ drat.sh
.travis.yml
^LICENSE\.md$
azure-pipelines.yml
^__azurite
^__blobstorage__
^__queuestorage__

4
.gitignore поставляемый
Просмотреть файл

@ -262,4 +262,6 @@ __pycache__/
misc/
.Rhistory
__azurite*
__blobstorage__
__queuestorage__

Просмотреть файл

@ -1,3 +1,9 @@
# AzureStor 3.2.3.9000
- Add support for the [Azurite](https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite) and [Azure SDK](https://docs.microsoft.com/en-us/azure/storage/common/storage-use-emulator) storage emulators. To connect to the endpoint, use the service-specific functions `blob_endpoint` and `queue_endpoint` (the latter from the AzureQstor package), passing the full URL including the account name: `blob_endpoint("http://127.0.0.1:10000/myaccount", key="mykey")`. The warning about an unrecognised endpoint can be ignored. See the linked pages for full details on how to authenticate to the emulator.
Note that the Azure SDK emulator is no longer being actively developed; it's recommended to use Azurite.
# AzureStor 3.2.3
- Fix file transfers with filenames containing non-ASCII characters on Windows.

Просмотреть файл

@ -15,6 +15,11 @@
#'
#' If multiple authentication objects are supplied, they are used in this order of priority: first an access key, then an AAD token, then a SAS. If no authentication objects are supplied, only public (anonymous) access to the endpoint is possible.
#'
#' @section Storage emulators:
#' AzureStor supports connecting to the [Azure SDK](https://docs.microsoft.com/en-us/azure/storage/common/storage-use-emulator) and [Azurite](https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite) emulators for blob and queue storage. To connect, pass the full URL of the endpoint, including the account name, to the `blob_endpoint` and `queue_endpoint` methods (the latter from the AzureQstor package). The warning about an unrecognised endpoint can be ignored. See the linked pages, and the examples below, for details on how to authenticate with the emulator.
#'
#' Note that the Azure SDK emulator is no longer being actively developed; it's recommended to use Azurite for development work.
#'
#' @return
#' `storage_endpoint` returns an object of S3 class `"adls_endpoint"`, `"blob_endpoint"`, `"file_endpoint"`, `"queue_endpoint"` or `"table_endpoint"` depending on the type of endpoint. All of these also inherit from class `"storage_endpoint"`. `adls_endpoint`, `blob_endpoint` and `file_endpoint` return an object of the respective class.
#'
@ -41,6 +46,19 @@
#' "myaadtenant", "app_id", "password")
#' adls_endpoint("https://myadlsstorage.dfs.core.windows.net/", token=token)
#'
#'
#' ## Azurite storage emulator:
#'
#' # connecting to Azurite with the default account and key (these also work for the Azure SDK)
#' azurite_account <- "devstoreaccount1"
#' azurite_key <-
#' "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
#' blob_endpoint(paste0("http://127.0.0.1:10000/", azurite_account), key=azurite_key)
#'
#' # to use a custom account name and key, set the AZURITE_ACCOUNTS env var before starting Azurite
#' Sys.setenv(AZURITE_ACCOUNTS="account1:key1")
#' blob_endpoint("http://127.0.0.1:10000/account1", key="key1")
#'
#' }
#' @aliases endpoint blob_endpoint file_endpoint queue_endpoint table_endpoint
#' @export

Просмотреть файл

@ -14,7 +14,9 @@ sign_request <- function(endpoint, ...)
sign_request.default <- function(endpoint, verb, url, headers, api, ...)
{
acct_name <- sub("\\..+$", "", url$host)
host_url <- httr::parse_url(endpoint$url)
acct_name <- if(host_url$path == "") sub("\\..+$", "", host_url$host) else host_url$path
resource <- paste0("/", acct_name, "/", url$path) # don't use file.path because it strips trailing / on Windows
# sanity check
resource <- gsub("//", "/", resource)

Просмотреть файл

@ -38,9 +38,8 @@
#' @export
do_container_op <- function(container, operation="", options=list(), headers=list(), http_verb="GET", ...)
{
# don't add trailing / if no within-container path supplied: ADLS will complain
operation <- if(nchar(operation) > 0)
sub("//", "/", paste0(container$name, "/", operation))
paste0(container$name, "/", operation)
else container$name
call_storage_endpoint(container$endpoint, operation, options=options, headers=headers, http_verb=http_verb, ...)
@ -57,7 +56,8 @@ call_storage_endpoint <- function(endpoint, path, options=list(), headers=list()
{
http_verb <- match.arg(http_verb)
url <- httr::parse_url(endpoint$url)
url$path <- url_encode(path)
# fix doubled-up /'s which can result from file.path snafus etc
url$path <- gsub("/{2,}", "/", url_encode(file.path(url$path, path)))
if(!is_empty(options))
url$query <- options[order(names(options))] # must be sorted for access key signing

Просмотреть файл

@ -55,6 +55,13 @@ This is the starting point for the client-side storage interface in AzureRMR. \c
If multiple authentication objects are supplied, they are used in this order of priority: first an access key, then an AAD token, then a SAS. If no authentication objects are supplied, only public (anonymous) access to the endpoint is possible.
}
\section{Storage emulators}{
AzureStor supports connecting to the \href{https://docs.microsoft.com/en-us/azure/storage/common/storage-use-emulator}{Azure SDK} and \href{https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite}{Azurite} emulators for blob and queue storage. To connect, pass the full URL of the endpoint, including the account name, to the \code{blob_endpoint} and \code{queue_endpoint} methods (the latter from the AzureQstor package). The warning about an unrecognised endpoint can be ignored. See the linked pages, and the examples below, for details on how to authenticate with the emulator.
Note that the Azure SDK emulator is no longer being actively developed; it's recommended to use Azurite for development work.
}
\examples{
\dontrun{
@ -73,6 +80,19 @@ token <- AzureAuth::get_azure_token("https://storage.azure.com",
"myaadtenant", "app_id", "password")
adls_endpoint("https://myadlsstorage.dfs.core.windows.net/", token=token)
## Azurite storage emulator:
# connecting to Azurite with the default account and key (these also work for the Azure SDK)
azurite_account <- "devstoreaccount1"
azurite_key <-
"Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
blob_endpoint(paste0("http://127.0.0.1:10000/", azurite_account), key=azurite_key)
# to use a custom account name and key, set the AZURITE_ACCOUNTS env var before starting Azurite
Sys.setenv(AZURITE_ACCOUNTS="account1:key1")
blob_endpoint("http://127.0.0.1:10000/account1", key="key1")
}
}
\seealso{

Просмотреть файл

@ -0,0 +1,33 @@
context("Azurite storage emulator")
# currently hardcoded to use account name=account1, shared key=key1
res <- try(httr::GET("http://127.0.0.1:10000/account1"), silent=TRUE)
if(inherits(res, "try-error"))
skip("Storage emulator tests skipped: Azurite not running")
opts <- options(azure_storage_progress_bar=FALSE)
test_that("Blob storage methods work",
{
expect_warning(endp <- blob_endpoint("http://127.0.0.1:10000/account1", key="key1"))
expect_is(endp, "blob_endpoint")
expect_is(list_blob_containers(endp), "list")
cont <- create_blob_container(endp, "container1")
expect_is(cont, "blob_container")
expect_silent(upload_blob(cont, "../resources/iris.csv"))
expect_is(list_blobs(cont), "data.frame")
expect_silent(download_blob(cont, "iris.csv", tempfile()))
expect_silent(delete_blob_container(cont, confirm=FALSE))
expect_true(is_empty(list_blob_containers(endp)))
})
teardown({
endp <- suppressWarnings(blob_endpoint("http://127.0.0.1:10000/account1", key="key1"))
conts <- list_blob_containers(endp)
lapply(conts, delete_blob_container, confirm=FALSE)
options(opts)
})