From 948448ba9c63c0cc403641014be68491805883f4 Mon Sep 17 00:00:00 2001 From: Hong Ooi Date: Fri, 11 Jan 2019 20:14:12 +1100 Subject: [PATCH] Aad auth (#3) * enable aad auth for blob, adls * fix regressions * fixups * left api version out when using token --- NEWS.md | 1 + R/adls_client_funcs.R | 33 ++++++++++++++------ R/blob_client_funcs.R | 40 ++++++++++++++++++------ R/client.R | 68 ++++++++++++++++++++++++++++++++--------- R/file_client_funcs.R | 8 ++--- R/storage_utils.R | 38 ++++++++++++++++++++--- man/adls_filesystem.Rd | 18 +++++++---- man/blob_container.Rd | 28 +++++++++++------ man/file_transfer.Rd | 8 ++--- man/storage_endpoint.Rd | 18 ++++++----- 10 files changed, 191 insertions(+), 69 deletions(-) diff --git a/NEWS.md b/NEWS.md index b010837..b248f57 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,6 @@ # AzureStor 1.0.0.9000 +* Add role-based access control via Azure Active Directory tokens for blob and ADLSgen2 storage. * Add ADLS upload/download support to `upload_to_url` and `download_from_url`. # AzureStor 1.0.0 diff --git a/R/adls_client_funcs.R b/R/adls_client_funcs.R index 0eb1505..8722656 100644 --- a/R/adls_client_funcs.R +++ b/R/adls_client_funcs.R @@ -3,7 +3,7 @@ #' Get, list, create, or delete ADLSgen2 filesystems. Currently (as of December 2018) ADLSgen2 is in general-access public preview. #' #' @param endpoint Either an ADLSgen2 endpoint object as created by [storage_endpoint] or [adls_endpoint], or a character string giving the URL of the endpoint. -#' @param key,sas If an endpoint object is not supplied, authentication details. Currently the `sas` argument is unused. +#' @param key,token,sas If an endpoint object is not supplied, authentication credentials: either an access key, an Azure Active Directory (AAD) token, or a SAS, in that order of priority. Currently the `sas` argument is unused. #' @param api_version If an endpoint object is not supplied, the storage API version to use when interacting with the host. Currently defaults to `"2018-06-17"`. #' @param name The name of the filesystem to get, create, or delete. #' @param confirm For deleting a filesystem, whether to ask for confirmation. @@ -13,6 +13,8 @@ #' @details #' You can call these functions in a couple of ways: by passing the full URL of the share, or by passing the endpoint object and the name of the share as a string. #' +#' If authenticating via AAD, you can supply the token either as a string, or as an object of class [AzureRMR::AzureToken], created via [AzureRMR::get_azure_token]. The latter is the recommended way of doing it, as it allows for automatic refreshing of expired tokens. +#' #' Currently (as of December 2018), if the storage account has hierarchical namespaces enabled, the blob API for the account is disabled. The blob endpoint is still accessible, but blob operations on the endpoint will fail. Full interoperability between blobs and ADLS is planned for 2019. #' #' @return @@ -50,11 +52,11 @@ adls_filesystem <- function(endpoint, ...) #' @rdname adls_filesystem #' @export -adls_filesystem.character <- function(endpoint, key=NULL, sas=NULL, +adls_filesystem.character <- function(endpoint, key=NULL, token=NULL, sas=NULL, api_version=getOption("azure_storage_api_version"), ...) { - do.call(adls_filesystem, generate_endpoint_container(endpoint, key, sas, api_version)) + do.call(adls_filesystem, generate_endpoint_container(endpoint, key, token, sas, api_version)) } #' @rdname adls_filesystem @@ -72,12 +74,22 @@ print.adls_filesystem <- function(x, ...) { cat("Azure Data Lake Storage Gen2 filesystem '", x$name, "'\n", sep="") cat(sprintf("URL: %s\n", paste0(x$endpoint$url, x$name))) + if(!is_empty(x$endpoint$key)) cat("Access key: \n") else cat("Access key: \n") + + if(!is_empty(x$endpoint$token)) + { + cat("Azure Active Directory token:\n") + print(x$endpoint$token) + } + else cat("Azure Active Directory token: \n") + if(!is_empty(x$endpoint$sas)) cat("Account shared access signature: \n") else cat("Account shared access signature: \n") + cat(sprintf("Storage API version: %s\n", x$endpoint$api_version)) invisible(x) } @@ -93,11 +105,11 @@ list_adls_filesystems <- function(endpoint, ...) #' @rdname adls_filesystem #' @export -list_adls_filesystems.character <- function(endpoint, key=NULL, sas=NULL, +list_adls_filesystems.character <- function(endpoint, key=NULL, token=NULL, sas=NULL, api_version=getOption("azure_adls_api_version"), ...) { - do.call(list_adls_filesystems, generate_endpoint_container(endpoint, key, sas, api_version)) + do.call(list_adls_filesystems, generate_endpoint_container(endpoint, key, token, sas, api_version)) } #' @rdname adls_filesystem @@ -105,7 +117,8 @@ list_adls_filesystems.character <- function(endpoint, key=NULL, sas=NULL, list_adls_filesystems.adls_endpoint <- function(endpoint, ...) { lst <- do_storage_call(endpoint$url, "/", options=list(resource="account"), - key=endpoint$key, sas=endpoint$sas, api_version=endpoint$api_version) + key=endpoint$key, token=endpoint$token,, sas=endpoint$sas, + api_version=endpoint$api_version) sapply(lst$filesystems$name, function(fs) adls_filesystem(endpoint, fs), simplify=FALSE) } @@ -121,11 +134,11 @@ create_adls_filesystem <- function(endpoint, ...) #' @rdname adls_filesystem #' @export -create_adls_filesystem.character <- function(endpoint, key=NULL, sas=NULL, +create_adls_filesystem.character <- function(endpoint, key=NULL, token=NULL, sas=NULL, api_version=getOption("azure_adls_api_version"), ...) { - endp <- generate_endpoint_container(endpoint, key, sas, api_version) + endp <- generate_endpoint_container(endpoint, key, token, sas, api_version) create_adls_filesystem(endp$endpoint, endp$name, ...) } @@ -156,11 +169,11 @@ delete_adls_filesystem <- function(endpoint, ...) #' @rdname adls_filesystem #' @export -delete_adls_filesystem.character <- function(endpoint, key=NULL, sas=NULL, +delete_adls_filesystem.character <- function(endpoint, key=NULL, token=NULL, sas=NULL, api_version=getOption("azure_adls_api_version"), ...) { - endp <- generate_endpoint_container(endpoint, key, sas, api_version) + endp <- generate_endpoint_container(endpoint, key, token, sas, api_version) delete_adls_filesystem(endp$endpoint, endp$name, ...) } diff --git a/R/blob_client_funcs.R b/R/blob_client_funcs.R index e378202..55c6ac7 100644 --- a/R/blob_client_funcs.R +++ b/R/blob_client_funcs.R @@ -3,7 +3,7 @@ #' Get, list, create, or delete blob containers. #' #' @param endpoint Either a blob endpoint object as created by [storage_endpoint], or a character string giving the URL of the endpoint. -#' @param key,sas If an endpoint object is not supplied, authentication details. If a key is provided, the SAS is not used. If neither an access key nor a SAS are provided, only public (anonymous) access to the share is possible. +#' @param key,token,sas If an endpoint object is not supplied, authentication credentials: either an access key, an Azure Active Directory (AAD) token, or a SAS, in that order of priority. If no authentication credentials are provided, only public (anonymous) access to the share is possible. #' @param api_version If an endpoint object is not supplied, the storage API version to use when interacting with the host. Currently defaults to `"2018-03-28"`. #' @param name The name of the blob container to get, create, or delete. #' @param confirm For deleting a container, whether to ask for confirmation. @@ -15,6 +15,8 @@ #' @details #' You can call these functions in a couple of ways: by passing the full URL of the share, or by passing the endpoint object and the name of the container as a string. #' +#' If authenticating via AAD, you can supply the token either as a string, or as an object of class [AzureRMR::AzureToken], created via [AzureRMR::get_azure_token]. The latter is the recommended way of doing it, as it allows for automatic refreshing of expired tokens. +#' #' Currently (as of December 2018), if the storage account has hierarchical namespaces enabled, the blob API for the account is disabled. The blob endpoint is still accessible, but blob operations on the endpoint will fail. Full interoperability between blobs and ADLS is planned for 2019. #' #' @return @@ -42,6 +44,13 @@ #' create_blob_container("https://mystorage.blob.core.windows.net/newcontainer", key="access_key") #' delete_blob_container("https://mystorage.blob.core.windows.net/newcontainer", key="access_key") #' +#' # authenticating via AAD +#' token <- AzureRMR::get_azure_token(resource_host="https://storage.azure.com/", +#' tenant="myaadtenant", +#' app="myappid", +#' password="mypassword") +#' blob_container("https://mystorage.blob.core.windows.net/mycontainer", token=token) +#' #' } #' @rdname blob_container #' @export @@ -52,11 +61,11 @@ blob_container <- function(endpoint, ...) #' @rdname blob_container #' @export -blob_container.character <- function(endpoint, key=NULL, sas=NULL, +blob_container.character <- function(endpoint, key=NULL, token=NULL, sas=NULL, api_version=getOption("azure_storage_api_version"), ...) { - do.call(blob_container, generate_endpoint_container(endpoint, key, sas, api_version)) + do.call(blob_container, generate_endpoint_container(endpoint, key, token, sas, api_version)) } #' @rdname blob_container @@ -74,12 +83,22 @@ print.blob_container <- function(x, ...) { cat("Azure blob container '", x$name, "'\n", sep="") cat(sprintf("URL: %s\n", paste0(x$endpoint$url, x$name))) + if(!is_empty(x$endpoint$key)) cat("Access key: \n") else cat("Access key: \n") + + if(!is_empty(x$endpoint$token)) + { + cat("Azure Active Directory access token:\n") + print(x$endpoint$token) + } + else cat("Azure Active Directory access token: \n") + if(!is_empty(x$endpoint$sas)) cat("Account shared access signature: \n") else cat("Account shared access signature: \n") + cat(sprintf("Storage API version: %s\n", x$endpoint$api_version)) invisible(x) } @@ -94,11 +113,11 @@ list_blob_containers <- function(endpoint, ...) #' @rdname blob_container #' @export -list_blob_containers.character <- function(endpoint, key=NULL, sas=NULL, +list_blob_containers.character <- function(endpoint, key=NULL, token=NULL, sas=NULL, api_version=getOption("azure_storage_api_version"), ...) { - do.call(list_blob_containers, generate_endpoint_container(endpoint, key, sas, api_version)) + do.call(list_blob_containers, generate_endpoint_container(endpoint, key, token, sas, api_version)) } #' @rdname blob_container @@ -106,7 +125,8 @@ list_blob_containers.character <- function(endpoint, key=NULL, sas=NULL, list_blob_containers.blob_endpoint <- function(endpoint, ...) { lst <- do_storage_call(endpoint$url, "/", options=list(comp="list"), - key=endpoint$key, sas=endpoint$sas, api_version=endpoint$api_version) + key=endpoint$key, token=endpoint$token, sas=endpoint$sas, + api_version=endpoint$api_version) lst <- lapply(lst$Containers, function(cont) blob_container(endpoint, cont$Name[[1]])) named_list(lst) @@ -123,11 +143,11 @@ create_blob_container <- function(endpoint, ...) #' @rdname blob_container #' @export -create_blob_container.character <- function(endpoint, key=NULL, sas=NULL, +create_blob_container.character <- function(endpoint, key=NULL, token=NULL, sas=NULL, api_version=getOption("azure_storage_api_version"), ...) { - endp <- generate_endpoint_container(endpoint, key, sas, api_version) + endp <- generate_endpoint_container(endpoint, key, token, sas, api_version) create_blob_container(endp$endpoint, endp$name, ...) } @@ -163,11 +183,11 @@ delete_blob_container <- function(endpoint, ...) #' @rdname blob_container #' @export -delete_blob_container.character <- function(endpoint, key=NULL, sas=NULL, +delete_blob_container.character <- function(endpoint, key=NULL, token=NULL, sas=NULL, api_version=getOption("azure_storage_api_version"), ...) { - endp <- generate_endpoint_container(endpoint, key, sas, api_version) + endp <- generate_endpoint_container(endpoint, key, token, sas, api_version) delete_blob_container(endp$endpoint, endp$name, ...) } diff --git a/R/client.R b/R/client.R index 13a4617..7384ae3 100644 --- a/R/client.R +++ b/R/client.R @@ -4,7 +4,8 @@ #' #' @param endpoint The URL (hostname) for the endpoint. This must be of the form `http[s]://{account-name}.{type}.{core-host-name}`, where `type` is one of `"dfs"` (corresponding to ADLSgen2), `"blob"`, `"file"`, `"queue"` or `"table"`. On the public Azure cloud, endpoints will be of the form `https://{account-name}.{type}.core.windows.net`. #' @param key The access key for the storage account. -#' @param sas A shared access signature (SAS) for the account. If `key` is also provided, the SAS is not used. If neither `key` nor `sas` are provided, only public (anonymous) access to the endpoint is possible. Note that authentication with a SAS is not supported by ADLSgen2. +#' @param token An Azure Active Directory (AAD) authentication token. This can be either a string, or an object of class [AzureRMR::AzureToken] created by [AzureRMR::get_azure_token]. The latter is the recommended way of doing it, as it allows for automatic refreshing of expired tokens. +#' @param sas A shared access signature (SAS) for the account. #' @param api_version The storage API version to use when interacting with the host. Defaults to `"2018-06-17"` for the ADLSgen2 endpoint, and `"2018-03-28"` for the others. #' @param x For the print method, a storage endpoint object. #' @param ... For the print method, further arguments passed to lower-level functions. @@ -12,6 +13,8 @@ #' @details #' This is the starting point for the client-side storage interface in AzureRMR. `storage_endpoint` is a generic function to create an endpoint for any type of Azure storage while `adls_endpoint`, `blob_endpoint` and `file_endpoint` create endpoints for those types. #' +#' If multiple authentication objects are supplied, they are used in this order of priority: first an access key, then an AAD token, then a SAS. If no authentication objects are supplied, only public (anonymous) access to the endpoint is possible. Note that authentication with a SAS is not currently supported by ADLSgen2, and authentication with an AAD token is not supported by file storage. +#' #' @return #' `storage_endpoint` returns an object of S3 class `"adls_endpoint"`, `"blob_endpoint"`, `"file_endpoint"`, `"queue_endpoint"` or `"table_endpoint"` depending on the type of endpoint. All of these also inherit from class `"storage_endpoint"`. `adls_endpoint`, `blob_endpoint` and `file_endpoint` return an object of the respective class. #' @@ -34,7 +37,7 @@ #' } #' @aliases endpoint blob_endpoint file_endpoint queue_endpoint table_endpoint #' @export -storage_endpoint <- function(endpoint, key=NULL, sas=NULL, api_version=getOption("azure_storage_api_version")) +storage_endpoint <- function(endpoint, key=NULL, token=NULL, sas=NULL, api_version) { type <- sapply(c("blob", "file", "queue", "table", "adls"), function(x) is_endpoint_url(endpoint, x)) @@ -42,41 +45,58 @@ storage_endpoint <- function(endpoint, key=NULL, sas=NULL, api_version=getOption stop("Unknown endpoint type", call.=FALSE) type <- names(type)[type] + # handle api version wart + if(missing(api_version)) + { + api_version <- if(type == "adls") + getOption("azure_adls_api_version") + else getOption("azure_storage_api_version") + } + if(type == "adls" && !is_empty(sas)) warning("ADLSgen2 does not support authentication with a shared access signature") - obj <- list(url=endpoint, key=key, sas=sas, api_version=api_version) + if(type == "file" && !is_empty(token)) + warning("File storage does not support authentication with an AAD token") + + obj <- list(url=endpoint, key=key, token=token, sas=sas, api_version=api_version) class(obj) <- c(paste0(type, "_endpoint"), "storage_endpoint") obj } #' @rdname storage_endpoint #' @export -blob_endpoint <- function(endpoint, key=NULL, sas=NULL, api_version=getOption("azure_storage_api_version")) +blob_endpoint <- function(endpoint, key=NULL, token=NULL, sas=NULL, + api_version=getOption("azure_storage_api_version")) { if(!is_endpoint_url(endpoint, "blob")) stop("Not a blob endpoint", call.=FALSE) - obj <- list(url=endpoint, key=key, sas=sas, api_version=api_version) + obj <- list(url=endpoint, key=key, token=token, sas=sas, api_version=api_version) class(obj) <- c("blob_endpoint", "storage_endpoint") obj } #' @rdname storage_endpoint #' @export -file_endpoint <- function(endpoint, key=NULL, sas=NULL, api_version=getOption("azure_storage_api_version")) +file_endpoint <- function(endpoint, key=NULL, token=NULL, sas=NULL, + api_version=getOption("azure_storage_api_version")) { if(!is_endpoint_url(endpoint, "file")) stop("Not a file endpoint", call.=FALSE) - obj <- list(url=endpoint, key=key, sas=sas, api_version=api_version) + if(!is_empty(token)) + warning("File storage does not support authentication with an AAD token") + + obj <- list(url=endpoint, key=key, token=token, sas=sas, api_version=api_version) class(obj) <- c("file_endpoint", "storage_endpoint") obj } #' @rdname storage_endpoint #' @export -adls_endpoint <- function(endpoint, key=NULL, sas=NULL, api_version=getOption("azure_adls_api_version")) +adls_endpoint <- function(endpoint, key=NULL, token=NULL, sas=NULL, + api_version=getOption("azure_adls_api_version")) { if(!is_endpoint_url(endpoint, "adls")) stop("Not an ADLS Gen2 endpoint", call.=FALSE) @@ -84,7 +104,7 @@ adls_endpoint <- function(endpoint, key=NULL, sas=NULL, api_version=getOption("a if(!is_empty(sas)) warning("ADLSgen2 does not support authentication with a shared access signature") - obj <- list(url=endpoint, key=key, sas=sas, api_version=api_version) + obj <- list(url=endpoint, key=key, token=token, sas=sas, api_version=api_version) class(obj) <- c("adls_endpoint", "storage_endpoint") obj } @@ -97,12 +117,22 @@ print.storage_endpoint <- function(x, ...) type <- sub("_endpoint$", "", class(x)[1]) cat(sprintf("Azure %s storage endpoint\n", type)) cat(sprintf("URL: %s\n", x$url)) + if(!is_empty(x$key)) cat("Access key: \n") else cat("Access key: \n") + + if(!is_empty(x$token)) + { + cat("Azure Active Directory token:\n") + print(x$token) + } + else cat("Azure Active Directory token: \n") + if(!is_empty(x$sas)) cat("Account shared access signature: \n") else cat("Account shared access signature: \n") + cat(sprintf("Storage API version: %s\n", x$api_version)) invisible(x) } @@ -114,12 +144,22 @@ print.adls_endpoint <- function(x, ...) { cat("Azure Data Lake Storage Gen2 endpoint\n") cat(sprintf("URL: %s\n", x$url)) + if(!is_empty(x$key)) cat("Access key: \n") else cat("Access key: \n") + + if(!is_empty(x$token)) + { + cat("Azure Active Directory token:\n") + print(x$token) + } + else cat("Azure Active Directory token: \n") + if(!is_empty(x$sas)) cat("Account shared access signature: \n") else cat("Account shared access signature: \n") + cat(sprintf("Storage API version: %s\n", x$api_version)) invisible(x) } @@ -129,7 +169,7 @@ print.adls_endpoint <- function(x, ...) #' Generic upload and download #' #' @param src,dest The source and destination files/URLs. Paths are allowed. -#' @param key,sas Authentication arguments: an access key or a shared access signature (SAS). If a key is is provided, the SAS is not used. If neither an access key nor a SAS are provided, only public (anonymous) access to the share is possible. +#' @param key,token,sas Authentication arguments: an access key or a shared access signature (SAS). If a key is is provided, the SAS is not used. If neither an access key nor a SAS are provided, only public (anonymous) access to the share is possible. #' @param ... Further arguments to pass to lower-level functions. #' @param overwrite For downloading, whether to overwrite any destination files that exist. #' @@ -152,12 +192,12 @@ print.adls_endpoint <- function(x, ...) #' } #' @rdname file_transfer #' @export -download_from_url <- function(src, dest, key=NULL, sas=NULL, ..., overwrite=FALSE) +download_from_url <- function(src, dest, key=NULL, token=NULL, sas=NULL, ..., overwrite=FALSE) { az_path <- parse_storage_url(src) if(is.null(sas)) sas <- find_sas(src) - endpoint <- storage_endpoint(az_path[1], key=key, sas=sas, ...) + endpoint <- storage_endpoint(az_path[1], key=key, token=token, sas=sas, ...) if(inherits(endpoint, "blob_endpoint")) { @@ -180,12 +220,12 @@ download_from_url <- function(src, dest, key=NULL, sas=NULL, ..., overwrite=FALS #' @rdname file_transfer #' @export -upload_to_url <- function(src, dest, key=NULL, sas=NULL, ...) +upload_to_url <- function(src, dest, key=NULL, token=token, sas=NULL, ...) { az_path <- parse_storage_url(dest) if(is.null(sas)) sas <- find_sas(dest) - endpoint <- storage_endpoint(az_path[1], key=key, sas=sas, ...) + endpoint <- storage_endpoint(az_path[1], key=key, token=token, sas=sas, ...) if(inherits(endpoint, "blob_endpoint")) { diff --git a/R/file_client_funcs.R b/R/file_client_funcs.R index f9581a0..e5d3b4f 100644 --- a/R/file_client_funcs.R +++ b/R/file_client_funcs.R @@ -52,7 +52,7 @@ file_share.character <- function(endpoint, key=NULL, sas=NULL, api_version=getOption("azure_storage_api_version"), ...) { - do.call(file_share, generate_endpoint_container(endpoint, key, sas, api_version)) + do.call(file_share, generate_endpoint_container(endpoint, key, token=NULL, sas, api_version)) } #' @rdname file_share @@ -95,7 +95,7 @@ list_file_shares.character <- function(endpoint, key=NULL, sas=NULL, api_version=getOption("azure_storage_api_version"), ...) { - do.call(list_file_shares, generate_endpoint_container(endpoint, key, sas, api_version)) + do.call(list_file_shares, generate_endpoint_container(endpoint, key, token=NULL, sas, api_version)) } #' @rdname file_share @@ -124,7 +124,7 @@ create_file_share.character <- function(endpoint, key=NULL, sas=NULL, api_version=getOption("azure_storage_api_version"), ...) { - endp <- generate_endpoint_container(endpoint, key, sas, api_version) + endp <- generate_endpoint_container(endpoint, key, token=NULL, sas, api_version) create_file_share(endp$endpoint, endp$name, ...) } @@ -159,7 +159,7 @@ delete_file_share.character <- function(endpoint, key=NULL, sas=NULL, api_version=getOption("azure_storage_api_version"), ...) { - endp <- generate_endpoint_container(endpoint, key, sas, api_version) + endp <- generate_endpoint_container(endpoint, key, token=NULL, sas, api_version) delete_file_share(endp$endpoint, endp$name, ...) } diff --git a/R/storage_utils.R b/R/storage_utils.R index 746de48..37145c4 100644 --- a/R/storage_utils.R +++ b/R/storage_utils.R @@ -8,13 +8,13 @@ do_container_op <- function(container, path="", options=list(), headers=list(), else container$name invisible(do_storage_call(endp$url, path, options=options, headers=headers, - key=endp$key, sas=endp$sas, api_version=endp$api_version, + key=endp$key, token=endp$token, sas=endp$sas, api_version=endp$api_version, http_verb=http_verb, ...)) } do_storage_call <- function(endpoint_url, path, options=list(), headers=list(), body=NULL, ..., - key=NULL, sas=NULL, + key=NULL, token=NULL, sas=NULL, api_version=getOption("azure_storage_api_version"), http_verb=c("GET", "DELETE", "PUT", "POST", "HEAD", "PATCH"), http_status_handler=c("stop", "warn", "message", "pass")) @@ -25,9 +25,11 @@ do_storage_call <- function(endpoint_url, path, options=list(), headers=list(), if(!is_empty(options)) url$query <- options[order(names(options))] # must be sorted for access key signing - # use key if provided, otherwise sas if provided, otherwise anonymous access + # use key if provided, otherwise AAD token if provided, otherwise sas if provided, otherwise anonymous access if(!is.null(key)) headers <- sign_request(key, verb, url, headers, api_version) + else if(!is.null(token)) + headers <- add_token(token, headers, api_version) else if(!is.null(sas)) url <- add_sas(sas, url) @@ -59,6 +61,32 @@ do_storage_call <- function(endpoint_url, path, options=list(), headers=list(), } +add_token <- function(token, headers, api) +{ + if(is.null(headers$`x-ms-version`)) + headers$`x-ms-version` <- api + + if(inherits(token, "R6") && inherits(token, "AzureToken")) + { + # if token has expired, renew it + if(!token$validate()) + { + message("Access token has expired or is no longer valid; refreshing") + token$refresh() + } + type <- token$credentials$token_type + token <- token$credentials$access_token + } + else + { + if(!is.character(token) || length(token) != 1) + stop("Token must be a string, or an object of class AzureRMR::AzureToken", call.=FALSE) + type <- "Bearer" + } + c(headers, Authorization=paste(type, token)) +} + + add_sas <- function(sas, url) { full_url <- httr::build_url(url) @@ -169,10 +197,10 @@ is_endpoint_url <- function(url, type) } -generate_endpoint_container <- function(url, key, sas, api_version) +generate_endpoint_container <- function(url, key, token, sas, api_version) { stor_path <- parse_storage_url(url) - endpoint <- storage_endpoint(stor_path[1], key, sas, api_version) + endpoint <- storage_endpoint(stor_path[1], key, token, sas, api_version) name <- stor_path[2] list(endpoint=endpoint, name=name) } diff --git a/man/adls_filesystem.Rd b/man/adls_filesystem.Rd index 4cb6265..3c04beb 100644 --- a/man/adls_filesystem.Rd +++ b/man/adls_filesystem.Rd @@ -20,8 +20,9 @@ \usage{ adls_filesystem(endpoint, ...) -\method{adls_filesystem}{character}(endpoint, key = NULL, sas = NULL, - api_version = getOption("azure_storage_api_version"), ...) +\method{adls_filesystem}{character}(endpoint, key = NULL, token = NULL, + sas = NULL, api_version = getOption("azure_storage_api_version"), + ...) \method{adls_filesystem}{adls_endpoint}(endpoint, name, ...) @@ -30,14 +31,16 @@ adls_filesystem(endpoint, ...) list_adls_filesystems(endpoint, ...) \method{list_adls_filesystems}{character}(endpoint, key = NULL, - sas = NULL, api_version = getOption("azure_adls_api_version"), ...) + token = NULL, sas = NULL, + api_version = getOption("azure_adls_api_version"), ...) \method{list_adls_filesystems}{adls_endpoint}(endpoint, ...) create_adls_filesystem(endpoint, ...) \method{create_adls_filesystem}{character}(endpoint, key = NULL, - sas = NULL, api_version = getOption("azure_adls_api_version"), ...) + token = NULL, sas = NULL, + api_version = getOption("azure_adls_api_version"), ...) \method{create_adls_filesystem}{adls_filesystem}(endpoint, ...) @@ -46,7 +49,8 @@ create_adls_filesystem(endpoint, ...) delete_adls_filesystem(endpoint, ...) \method{delete_adls_filesystem}{character}(endpoint, key = NULL, - sas = NULL, api_version = getOption("azure_adls_api_version"), ...) + token = NULL, sas = NULL, + api_version = getOption("azure_adls_api_version"), ...) \method{delete_adls_filesystem}{adls_filesystem}(endpoint, ...) @@ -58,7 +62,7 @@ delete_adls_filesystem(endpoint, ...) \item{...}{Further arguments passed to lower-level functions.} -\item{key, sas}{If an endpoint object is not supplied, authentication details. Currently the \code{sas} argument is unused.} +\item{key, token, sas}{If an endpoint object is not supplied, authentication credentials: either an access key, an Azure Active Directory (AAD) token, or a SAS, in that order of priority. Currently the \code{sas} argument is unused.} \item{api_version}{If an endpoint object is not supplied, the storage API version to use when interacting with the host. Currently defaults to \code{"2018-06-17"}.} @@ -79,6 +83,8 @@ Get, list, create, or delete ADLSgen2 filesystems. Currently (as of December 201 \details{ You can call these functions in a couple of ways: by passing the full URL of the share, or by passing the endpoint object and the name of the share as a string. +If authenticating via AAD, you can supply the token either as a string, or as an object of class \link[AzureRMR:AzureToken]{AzureRMR::AzureToken}, created via \link[AzureRMR:get_azure_token]{AzureRMR::get_azure_token}. The latter is the recommended way of doing it, as it allows for automatic refreshing of expired tokens. + Currently (as of December 2018), if the storage account has hierarchical namespaces enabled, the blob API for the account is disabled. The blob endpoint is still accessible, but blob operations on the endpoint will fail. Full interoperability between blobs and ADLS is planned for 2019. } \examples{ diff --git a/man/blob_container.Rd b/man/blob_container.Rd index 54fad18..47282a4 100644 --- a/man/blob_container.Rd +++ b/man/blob_container.Rd @@ -20,8 +20,9 @@ \usage{ blob_container(endpoint, ...) -\method{blob_container}{character}(endpoint, key = NULL, sas = NULL, - api_version = getOption("azure_storage_api_version"), ...) +\method{blob_container}{character}(endpoint, key = NULL, token = NULL, + sas = NULL, api_version = getOption("azure_storage_api_version"), + ...) \method{blob_container}{blob_endpoint}(endpoint, name, ...) @@ -30,16 +31,16 @@ blob_container(endpoint, ...) list_blob_containers(endpoint, ...) \method{list_blob_containers}{character}(endpoint, key = NULL, - sas = NULL, api_version = getOption("azure_storage_api_version"), - ...) + token = NULL, sas = NULL, + api_version = getOption("azure_storage_api_version"), ...) \method{list_blob_containers}{blob_endpoint}(endpoint, ...) create_blob_container(endpoint, ...) \method{create_blob_container}{character}(endpoint, key = NULL, - sas = NULL, api_version = getOption("azure_storage_api_version"), - ...) + token = NULL, sas = NULL, + api_version = getOption("azure_storage_api_version"), ...) \method{create_blob_container}{blob_container}(endpoint, ...) @@ -49,8 +50,8 @@ create_blob_container(endpoint, ...) delete_blob_container(endpoint, ...) \method{delete_blob_container}{character}(endpoint, key = NULL, - sas = NULL, api_version = getOption("azure_storage_api_version"), - ...) + token = NULL, sas = NULL, + api_version = getOption("azure_storage_api_version"), ...) \method{delete_blob_container}{blob_container}(endpoint, ...) @@ -62,7 +63,7 @@ delete_blob_container(endpoint, ...) \item{...}{Further arguments passed to lower-level functions.} -\item{key, sas}{If an endpoint object is not supplied, authentication details. If a key is provided, the SAS is not used. If neither an access key nor a SAS are provided, only public (anonymous) access to the share is possible.} +\item{key, token, sas}{If an endpoint object is not supplied, authentication credentials: either an access key, an Azure Active Directory (AAD) token, or a SAS, in that order of priority. If no authentication credentials are provided, only public (anonymous) access to the share is possible.} \item{api_version}{If an endpoint object is not supplied, the storage API version to use when interacting with the host. Currently defaults to \code{"2018-03-28"}.} @@ -87,6 +88,8 @@ Get, list, create, or delete blob containers. \details{ You can call these functions in a couple of ways: by passing the full URL of the share, or by passing the endpoint object and the name of the container as a string. +If authenticating via AAD, you can supply the token either as a string, or as an object of class \link[AzureRMR:AzureToken]{AzureRMR::AzureToken}, created via \link[AzureRMR:get_azure_token]{AzureRMR::get_azure_token}. The latter is the recommended way of doing it, as it allows for automatic refreshing of expired tokens. + Currently (as of December 2018), if the storage account has hierarchical namespaces enabled, the blob API for the account is disabled. The blob endpoint is still accessible, but blob operations on the endpoint will fail. Full interoperability between blobs and ADLS is planned for 2019. } \examples{ @@ -107,6 +110,13 @@ blob_container("https://mystorage.blob.core.windows.net/mycontainer", key="acces create_blob_container("https://mystorage.blob.core.windows.net/newcontainer", key="access_key") delete_blob_container("https://mystorage.blob.core.windows.net/newcontainer", key="access_key") +# authenticating via AAD +token <- AzureRMR::get_azure_token(resource_host="https://storage.azure.com/", + tenant="myaadtenant", + app="myappid", + password="mypassword") +blob_container("https://mystorage.blob.core.windows.net/mycontainer", token=token) + } } \seealso{ diff --git a/man/file_transfer.Rd b/man/file_transfer.Rd index 64a0362..3367290 100644 --- a/man/file_transfer.Rd +++ b/man/file_transfer.Rd @@ -5,15 +5,15 @@ \alias{upload_to_url} \title{Generic upload and download} \usage{ -download_from_url(src, dest, key = NULL, sas = NULL, ..., - overwrite = FALSE) +download_from_url(src, dest, key = NULL, token = NULL, sas = NULL, + ..., overwrite = FALSE) -upload_to_url(src, dest, key = NULL, sas = NULL, ...) +upload_to_url(src, dest, key = NULL, token = token, sas = NULL, ...) } \arguments{ \item{src, dest}{The source and destination files/URLs. Paths are allowed.} -\item{key, sas}{Authentication arguments: an access key or a shared access signature (SAS). If a key is is provided, the SAS is not used. If neither an access key nor a SAS are provided, only public (anonymous) access to the share is possible.} +\item{key, token, sas}{Authentication arguments: an access key or a shared access signature (SAS). If a key is is provided, the SAS is not used. If neither an access key nor a SAS are provided, only public (anonymous) access to the share is possible.} \item{...}{Further arguments to pass to lower-level functions.} diff --git a/man/storage_endpoint.Rd b/man/storage_endpoint.Rd index c7d590b..4eadc90 100644 --- a/man/storage_endpoint.Rd +++ b/man/storage_endpoint.Rd @@ -12,16 +12,16 @@ \alias{print.adls_endpoint} \title{Create a storage endpoint object} \usage{ -storage_endpoint(endpoint, key = NULL, sas = NULL, +storage_endpoint(endpoint, key = NULL, token = NULL, sas = NULL, + api_version) + +blob_endpoint(endpoint, key = NULL, token = NULL, sas = NULL, api_version = getOption("azure_storage_api_version")) -blob_endpoint(endpoint, key = NULL, sas = NULL, +file_endpoint(endpoint, key = NULL, token = NULL, sas = NULL, api_version = getOption("azure_storage_api_version")) -file_endpoint(endpoint, key = NULL, sas = NULL, - api_version = getOption("azure_storage_api_version")) - -adls_endpoint(endpoint, key = NULL, sas = NULL, +adls_endpoint(endpoint, key = NULL, token = NULL, sas = NULL, api_version = getOption("azure_adls_api_version")) \method{print}{storage_endpoint}(x, ...) @@ -33,7 +33,9 @@ adls_endpoint(endpoint, key = NULL, sas = NULL, \item{key}{The access key for the storage account.} -\item{sas}{A shared access signature (SAS) for the account. If \code{key} is also provided, the SAS is not used. If neither \code{key} nor \code{sas} are provided, only public (anonymous) access to the endpoint is possible. Note that authentication with a SAS is not supported by ADLSgen2.} +\item{token}{An Azure Active Directory (AAD) authentication token. This can be either a string, or an object of class \link[AzureRMR:AzureToken]{AzureRMR::AzureToken} created by \link[AzureRMR:get_azure_token]{AzureRMR::get_azure_token}. The latter is the recommended way of doing it, as it allows for automatic refreshing of expired tokens.} + +\item{sas}{A shared access signature (SAS) for the account.} \item{api_version}{The storage API version to use when interacting with the host. Defaults to \code{"2018-06-17"} for the ADLSgen2 endpoint, and \code{"2018-03-28"} for the others.} @@ -53,6 +55,8 @@ Create a storage endpoint object, for interacting with blob, file, table, queue } \details{ This is the starting point for the client-side storage interface in AzureRMR. \code{storage_endpoint} is a generic function to create an endpoint for any type of Azure storage while \code{adls_endpoint}, \code{blob_endpoint} and \code{file_endpoint} create endpoints for those types. + +If multiple authentication objects are supplied, they are used in this order of priority: first an access key, then an AAD token, then a SAS. If no authentication objects are supplied, only public (anonymous) access to the endpoint is possible. Note that authentication with a SAS is not currently supported by ADLSgen2, and authentication with an AAD token is not supported by file storage. } \examples{ \dontrun{