зеркало из https://github.com/Azure/AzureStor.git
Merge branch 'master' of https://github.com/cloudyr/AzureStor
This commit is contained in:
hong-revo
Коммит
a54bd8d116
1
NEWS.md
1
NEWS.md
|
|
@ -1,5 +1,6 @@
|
|||
# AzureStor 1.0.0.9000
|
||||
|
||||
* Add role-based access control via Azure Active Directory tokens for blob and ADLSgen2 storage.
|
||||
* Add ADLS upload/download support to `upload_to_url` and `download_from_url`.
|
||||
|
||||
# AzureStor 1.0.0
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
#' Get, list, create, or delete ADLSgen2 filesystems. Currently (as of December 2018) ADLSgen2 is in general-access public preview.
|
||||
#'
|
||||
#' @param endpoint Either an ADLSgen2 endpoint object as created by [storage_endpoint] or [adls_endpoint], or a character string giving the URL of the endpoint.
|
||||
#' @param key,sas If an endpoint object is not supplied, authentication details. Currently the `sas` argument is unused.
|
||||
#' @param key,token,sas If an endpoint object is not supplied, authentication credentials: either an access key, an Azure Active Directory (AAD) token, or a SAS, in that order of priority. Currently the `sas` argument is unused.
|
||||
#' @param api_version If an endpoint object is not supplied, the storage API version to use when interacting with the host. Currently defaults to `"2018-06-17"`.
|
||||
#' @param name The name of the filesystem to get, create, or delete.
|
||||
#' @param confirm For deleting a filesystem, whether to ask for confirmation.
|
||||
|
|
@ -13,6 +13,8 @@
|
|||
#' @details
|
||||
#' You can call these functions in a couple of ways: by passing the full URL of the share, or by passing the endpoint object and the name of the share as a string.
|
||||
#'
|
||||
#' If authenticating via AAD, you can supply the token either as a string, or as an object of class [AzureRMR::AzureToken], created via [AzureRMR::get_azure_token]. The latter is the recommended way of doing it, as it allows for automatic refreshing of expired tokens.
|
||||
#'
|
||||
#' Currently (as of December 2018), if the storage account has hierarchical namespaces enabled, the blob API for the account is disabled. The blob endpoint is still accessible, but blob operations on the endpoint will fail. Full interoperability between blobs and ADLS is planned for 2019.
|
||||
#'
|
||||
#' @return
|
||||
|
|
@ -50,11 +52,11 @@ adls_filesystem <- function(endpoint, ...)
|
|||
|
||||
#' @rdname adls_filesystem
|
||||
#' @export
|
||||
adls_filesystem.character <- function(endpoint, key=NULL, sas=NULL,
|
||||
adls_filesystem.character <- function(endpoint, key=NULL, token=NULL, sas=NULL,
|
||||
api_version=getOption("azure_storage_api_version"),
|
||||
...)
|
||||
{
|
||||
do.call(adls_filesystem, generate_endpoint_container(endpoint, key, sas, api_version))
|
||||
do.call(adls_filesystem, generate_endpoint_container(endpoint, key, token, sas, api_version))
|
||||
}
|
||||
|
||||
#' @rdname adls_filesystem
|
||||
|
|
@ -72,12 +74,22 @@ print.adls_filesystem <- function(x, ...)
|
|||
{
|
||||
cat("Azure Data Lake Storage Gen2 filesystem '", x$name, "'\n", sep="")
|
||||
cat(sprintf("URL: %s\n", paste0(x$endpoint$url, x$name)))
|
||||
|
||||
if(!is_empty(x$endpoint$key))
|
||||
cat("Access key: <hidden>\n")
|
||||
else cat("Access key: <none supplied>\n")
|
||||
|
||||
if(!is_empty(x$endpoint$token))
|
||||
{
|
||||
cat("Azure Active Directory token:\n")
|
||||
print(x$endpoint$token)
|
||||
}
|
||||
else cat("Azure Active Directory token: <none supplied>\n")
|
||||
|
||||
if(!is_empty(x$endpoint$sas))
|
||||
cat("Account shared access signature: <hidden>\n")
|
||||
else cat("Account shared access signature: <none supplied>\n")
|
||||
|
||||
cat(sprintf("Storage API version: %s\n", x$endpoint$api_version))
|
||||
invisible(x)
|
||||
}
|
||||
|
|
@ -93,11 +105,11 @@ list_adls_filesystems <- function(endpoint, ...)
|
|||
|
||||
#' @rdname adls_filesystem
|
||||
#' @export
|
||||
list_adls_filesystems.character <- function(endpoint, key=NULL, sas=NULL,
|
||||
list_adls_filesystems.character <- function(endpoint, key=NULL, token=NULL, sas=NULL,
|
||||
api_version=getOption("azure_adls_api_version"),
|
||||
...)
|
||||
{
|
||||
do.call(list_adls_filesystems, generate_endpoint_container(endpoint, key, sas, api_version))
|
||||
do.call(list_adls_filesystems, generate_endpoint_container(endpoint, key, token, sas, api_version))
|
||||
}
|
||||
|
||||
#' @rdname adls_filesystem
|
||||
|
|
@ -105,7 +117,8 @@ list_adls_filesystems.character <- function(endpoint, key=NULL, sas=NULL,
|
|||
list_adls_filesystems.adls_endpoint <- function(endpoint, ...)
|
||||
{
|
||||
lst <- do_storage_call(endpoint$url, "/", options=list(resource="account"),
|
||||
key=endpoint$key, sas=endpoint$sas, api_version=endpoint$api_version)
|
||||
key=endpoint$key, token=endpoint$token,, sas=endpoint$sas,
|
||||
api_version=endpoint$api_version)
|
||||
|
||||
sapply(lst$filesystems$name, function(fs) adls_filesystem(endpoint, fs), simplify=FALSE)
|
||||
}
|
||||
|
|
@ -121,11 +134,11 @@ create_adls_filesystem <- function(endpoint, ...)
|
|||
|
||||
#' @rdname adls_filesystem
|
||||
#' @export
|
||||
create_adls_filesystem.character <- function(endpoint, key=NULL, sas=NULL,
|
||||
create_adls_filesystem.character <- function(endpoint, key=NULL, token=NULL, sas=NULL,
|
||||
api_version=getOption("azure_adls_api_version"),
|
||||
...)
|
||||
{
|
||||
endp <- generate_endpoint_container(endpoint, key, sas, api_version)
|
||||
endp <- generate_endpoint_container(endpoint, key, token, sas, api_version)
|
||||
create_adls_filesystem(endp$endpoint, endp$name, ...)
|
||||
}
|
||||
|
||||
|
|
@ -156,11 +169,11 @@ delete_adls_filesystem <- function(endpoint, ...)
|
|||
|
||||
#' @rdname adls_filesystem
|
||||
#' @export
|
||||
delete_adls_filesystem.character <- function(endpoint, key=NULL, sas=NULL,
|
||||
delete_adls_filesystem.character <- function(endpoint, key=NULL, token=NULL, sas=NULL,
|
||||
api_version=getOption("azure_adls_api_version"),
|
||||
...)
|
||||
{
|
||||
endp <- generate_endpoint_container(endpoint, key, sas, api_version)
|
||||
endp <- generate_endpoint_container(endpoint, key, token, sas, api_version)
|
||||
delete_adls_filesystem(endp$endpoint, endp$name, ...)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
#' Get, list, create, or delete blob containers.
|
||||
#'
|
||||
#' @param endpoint Either a blob endpoint object as created by [storage_endpoint], or a character string giving the URL of the endpoint.
|
||||
#' @param key,sas If an endpoint object is not supplied, authentication details. If a key is provided, the SAS is not used. If neither an access key nor a SAS are provided, only public (anonymous) access to the share is possible.
|
||||
#' @param key,token,sas If an endpoint object is not supplied, authentication credentials: either an access key, an Azure Active Directory (AAD) token, or a SAS, in that order of priority. If no authentication credentials are provided, only public (anonymous) access to the share is possible.
|
||||
#' @param api_version If an endpoint object is not supplied, the storage API version to use when interacting with the host. Currently defaults to `"2018-03-28"`.
|
||||
#' @param name The name of the blob container to get, create, or delete.
|
||||
#' @param confirm For deleting a container, whether to ask for confirmation.
|
||||
|
|
@ -15,6 +15,8 @@
|
|||
#' @details
|
||||
#' You can call these functions in a couple of ways: by passing the full URL of the share, or by passing the endpoint object and the name of the container as a string.
|
||||
#'
|
||||
#' If authenticating via AAD, you can supply the token either as a string, or as an object of class [AzureRMR::AzureToken], created via [AzureRMR::get_azure_token]. The latter is the recommended way of doing it, as it allows for automatic refreshing of expired tokens.
|
||||
#'
|
||||
#' Currently (as of December 2018), if the storage account has hierarchical namespaces enabled, the blob API for the account is disabled. The blob endpoint is still accessible, but blob operations on the endpoint will fail. Full interoperability between blobs and ADLS is planned for 2019.
|
||||
#'
|
||||
#' @return
|
||||
|
|
@ -42,6 +44,13 @@
|
|||
#' create_blob_container("https://mystorage.blob.core.windows.net/newcontainer", key="access_key")
|
||||
#' delete_blob_container("https://mystorage.blob.core.windows.net/newcontainer", key="access_key")
|
||||
#'
|
||||
#' # authenticating via AAD
|
||||
#' token <- AzureRMR::get_azure_token(resource_host="https://storage.azure.com/",
|
||||
#' tenant="myaadtenant",
|
||||
#' app="myappid",
|
||||
#' password="mypassword")
|
||||
#' blob_container("https://mystorage.blob.core.windows.net/mycontainer", token=token)
|
||||
#'
|
||||
#' }
|
||||
#' @rdname blob_container
|
||||
#' @export
|
||||
|
|
@ -52,11 +61,11 @@ blob_container <- function(endpoint, ...)
|
|||
|
||||
#' @rdname blob_container
|
||||
#' @export
|
||||
blob_container.character <- function(endpoint, key=NULL, sas=NULL,
|
||||
blob_container.character <- function(endpoint, key=NULL, token=NULL, sas=NULL,
|
||||
api_version=getOption("azure_storage_api_version"),
|
||||
...)
|
||||
{
|
||||
do.call(blob_container, generate_endpoint_container(endpoint, key, sas, api_version))
|
||||
do.call(blob_container, generate_endpoint_container(endpoint, key, token, sas, api_version))
|
||||
}
|
||||
|
||||
#' @rdname blob_container
|
||||
|
|
@ -74,12 +83,22 @@ print.blob_container <- function(x, ...)
|
|||
{
|
||||
cat("Azure blob container '", x$name, "'\n", sep="")
|
||||
cat(sprintf("URL: %s\n", paste0(x$endpoint$url, x$name)))
|
||||
|
||||
if(!is_empty(x$endpoint$key))
|
||||
cat("Access key: <hidden>\n")
|
||||
else cat("Access key: <none supplied>\n")
|
||||
|
||||
if(!is_empty(x$endpoint$token))
|
||||
{
|
||||
cat("Azure Active Directory access token:\n")
|
||||
print(x$endpoint$token)
|
||||
}
|
||||
else cat("Azure Active Directory access token: <none supplied>\n")
|
||||
|
||||
if(!is_empty(x$endpoint$sas))
|
||||
cat("Account shared access signature: <hidden>\n")
|
||||
else cat("Account shared access signature: <none supplied>\n")
|
||||
|
||||
cat(sprintf("Storage API version: %s\n", x$endpoint$api_version))
|
||||
invisible(x)
|
||||
}
|
||||
|
|
@ -94,11 +113,11 @@ list_blob_containers <- function(endpoint, ...)
|
|||
|
||||
#' @rdname blob_container
|
||||
#' @export
|
||||
list_blob_containers.character <- function(endpoint, key=NULL, sas=NULL,
|
||||
list_blob_containers.character <- function(endpoint, key=NULL, token=NULL, sas=NULL,
|
||||
api_version=getOption("azure_storage_api_version"),
|
||||
...)
|
||||
{
|
||||
do.call(list_blob_containers, generate_endpoint_container(endpoint, key, sas, api_version))
|
||||
do.call(list_blob_containers, generate_endpoint_container(endpoint, key, token, sas, api_version))
|
||||
}
|
||||
|
||||
#' @rdname blob_container
|
||||
|
|
@ -106,7 +125,8 @@ list_blob_containers.character <- function(endpoint, key=NULL, sas=NULL,
|
|||
list_blob_containers.blob_endpoint <- function(endpoint, ...)
|
||||
{
|
||||
lst <- do_storage_call(endpoint$url, "/", options=list(comp="list"),
|
||||
key=endpoint$key, sas=endpoint$sas, api_version=endpoint$api_version)
|
||||
key=endpoint$key, token=endpoint$token, sas=endpoint$sas,
|
||||
api_version=endpoint$api_version)
|
||||
|
||||
lst <- lapply(lst$Containers, function(cont) blob_container(endpoint, cont$Name[[1]]))
|
||||
named_list(lst)
|
||||
|
|
@ -123,11 +143,11 @@ create_blob_container <- function(endpoint, ...)
|
|||
|
||||
#' @rdname blob_container
|
||||
#' @export
|
||||
create_blob_container.character <- function(endpoint, key=NULL, sas=NULL,
|
||||
create_blob_container.character <- function(endpoint, key=NULL, token=NULL, sas=NULL,
|
||||
api_version=getOption("azure_storage_api_version"),
|
||||
...)
|
||||
{
|
||||
endp <- generate_endpoint_container(endpoint, key, sas, api_version)
|
||||
endp <- generate_endpoint_container(endpoint, key, token, sas, api_version)
|
||||
create_blob_container(endp$endpoint, endp$name, ...)
|
||||
}
|
||||
|
||||
|
|
@ -163,11 +183,11 @@ delete_blob_container <- function(endpoint, ...)
|
|||
|
||||
#' @rdname blob_container
|
||||
#' @export
|
||||
delete_blob_container.character <- function(endpoint, key=NULL, sas=NULL,
|
||||
delete_blob_container.character <- function(endpoint, key=NULL, token=NULL, sas=NULL,
|
||||
api_version=getOption("azure_storage_api_version"),
|
||||
...)
|
||||
{
|
||||
endp <- generate_endpoint_container(endpoint, key, sas, api_version)
|
||||
endp <- generate_endpoint_container(endpoint, key, token, sas, api_version)
|
||||
delete_blob_container(endp$endpoint, endp$name, ...)
|
||||
}
|
||||
|
||||
|
|
|
|||
68
R/client.R
68
R/client.R
|
|
@ -4,7 +4,8 @@
|
|||
#'
|
||||
#' @param endpoint The URL (hostname) for the endpoint. This must be of the form `http[s]://{account-name}.{type}.{core-host-name}`, where `type` is one of `"dfs"` (corresponding to ADLSgen2), `"blob"`, `"file"`, `"queue"` or `"table"`. On the public Azure cloud, endpoints will be of the form `https://{account-name}.{type}.core.windows.net`.
|
||||
#' @param key The access key for the storage account.
|
||||
#' @param sas A shared access signature (SAS) for the account. If `key` is also provided, the SAS is not used. If neither `key` nor `sas` are provided, only public (anonymous) access to the endpoint is possible. Note that authentication with a SAS is not supported by ADLSgen2.
|
||||
#' @param token An Azure Active Directory (AAD) authentication token. This can be either a string, or an object of class [AzureRMR::AzureToken] created by [AzureRMR::get_azure_token]. The latter is the recommended way of doing it, as it allows for automatic refreshing of expired tokens.
|
||||
#' @param sas A shared access signature (SAS) for the account.
|
||||
#' @param api_version The storage API version to use when interacting with the host. Defaults to `"2018-06-17"` for the ADLSgen2 endpoint, and `"2018-03-28"` for the others.
|
||||
#' @param x For the print method, a storage endpoint object.
|
||||
#' @param ... For the print method, further arguments passed to lower-level functions.
|
||||
|
|
@ -12,6 +13,8 @@
|
|||
#' @details
|
||||
#' This is the starting point for the client-side storage interface in AzureRMR. `storage_endpoint` is a generic function to create an endpoint for any type of Azure storage while `adls_endpoint`, `blob_endpoint` and `file_endpoint` create endpoints for those types.
|
||||
#'
|
||||
#' If multiple authentication objects are supplied, they are used in this order of priority: first an access key, then an AAD token, then a SAS. If no authentication objects are supplied, only public (anonymous) access to the endpoint is possible. Note that authentication with a SAS is not currently supported by ADLSgen2, and authentication with an AAD token is not supported by file storage.
|
||||
#'
|
||||
#' @return
|
||||
#' `storage_endpoint` returns an object of S3 class `"adls_endpoint"`, `"blob_endpoint"`, `"file_endpoint"`, `"queue_endpoint"` or `"table_endpoint"` depending on the type of endpoint. All of these also inherit from class `"storage_endpoint"`. `adls_endpoint`, `blob_endpoint` and `file_endpoint` return an object of the respective class.
|
||||
#'
|
||||
|
|
@ -34,7 +37,7 @@
|
|||
#' }
|
||||
#' @aliases endpoint blob_endpoint file_endpoint queue_endpoint table_endpoint
|
||||
#' @export
|
||||
storage_endpoint <- function(endpoint, key=NULL, sas=NULL, api_version=getOption("azure_storage_api_version"))
|
||||
storage_endpoint <- function(endpoint, key=NULL, token=NULL, sas=NULL, api_version)
|
||||
{
|
||||
type <- sapply(c("blob", "file", "queue", "table", "adls"),
|
||||
function(x) is_endpoint_url(endpoint, x))
|
||||
|
|
@ -42,41 +45,58 @@ storage_endpoint <- function(endpoint, key=NULL, sas=NULL, api_version=getOption
|
|||
stop("Unknown endpoint type", call.=FALSE)
|
||||
type <- names(type)[type]
|
||||
|
||||
# handle api version wart
|
||||
if(missing(api_version))
|
||||
{
|
||||
api_version <- if(type == "adls")
|
||||
getOption("azure_adls_api_version")
|
||||
else getOption("azure_storage_api_version")
|
||||
}
|
||||
|
||||
if(type == "adls" && !is_empty(sas))
|
||||
warning("ADLSgen2 does not support authentication with a shared access signature")
|
||||
|
||||
obj <- list(url=endpoint, key=key, sas=sas, api_version=api_version)
|
||||
if(type == "file" && !is_empty(token))
|
||||
warning("File storage does not support authentication with an AAD token")
|
||||
|
||||
obj <- list(url=endpoint, key=key, token=token, sas=sas, api_version=api_version)
|
||||
class(obj) <- c(paste0(type, "_endpoint"), "storage_endpoint")
|
||||
obj
|
||||
}
|
||||
|
||||
#' @rdname storage_endpoint
|
||||
#' @export
|
||||
blob_endpoint <- function(endpoint, key=NULL, sas=NULL, api_version=getOption("azure_storage_api_version"))
|
||||
blob_endpoint <- function(endpoint, key=NULL, token=NULL, sas=NULL,
|
||||
api_version=getOption("azure_storage_api_version"))
|
||||
{
|
||||
if(!is_endpoint_url(endpoint, "blob"))
|
||||
stop("Not a blob endpoint", call.=FALSE)
|
||||
|
||||
obj <- list(url=endpoint, key=key, sas=sas, api_version=api_version)
|
||||
obj <- list(url=endpoint, key=key, token=token, sas=sas, api_version=api_version)
|
||||
class(obj) <- c("blob_endpoint", "storage_endpoint")
|
||||
obj
|
||||
}
|
||||
|
||||
#' @rdname storage_endpoint
|
||||
#' @export
|
||||
file_endpoint <- function(endpoint, key=NULL, sas=NULL, api_version=getOption("azure_storage_api_version"))
|
||||
file_endpoint <- function(endpoint, key=NULL, token=NULL, sas=NULL,
|
||||
api_version=getOption("azure_storage_api_version"))
|
||||
{
|
||||
if(!is_endpoint_url(endpoint, "file"))
|
||||
stop("Not a file endpoint", call.=FALSE)
|
||||
|
||||
obj <- list(url=endpoint, key=key, sas=sas, api_version=api_version)
|
||||
if(!is_empty(token))
|
||||
warning("File storage does not support authentication with an AAD token")
|
||||
|
||||
obj <- list(url=endpoint, key=key, token=token, sas=sas, api_version=api_version)
|
||||
class(obj) <- c("file_endpoint", "storage_endpoint")
|
||||
obj
|
||||
}
|
||||
|
||||
#' @rdname storage_endpoint
|
||||
#' @export
|
||||
adls_endpoint <- function(endpoint, key=NULL, sas=NULL, api_version=getOption("azure_adls_api_version"))
|
||||
adls_endpoint <- function(endpoint, key=NULL, token=NULL, sas=NULL,
|
||||
api_version=getOption("azure_adls_api_version"))
|
||||
{
|
||||
if(!is_endpoint_url(endpoint, "adls"))
|
||||
stop("Not an ADLS Gen2 endpoint", call.=FALSE)
|
||||
|
|
@ -84,7 +104,7 @@ adls_endpoint <- function(endpoint, key=NULL, sas=NULL, api_version=getOption("a
|
|||
if(!is_empty(sas))
|
||||
warning("ADLSgen2 does not support authentication with a shared access signature")
|
||||
|
||||
obj <- list(url=endpoint, key=key, sas=sas, api_version=api_version)
|
||||
obj <- list(url=endpoint, key=key, token=token, sas=sas, api_version=api_version)
|
||||
class(obj) <- c("adls_endpoint", "storage_endpoint")
|
||||
obj
|
||||
}
|
||||
|
|
@ -97,12 +117,22 @@ print.storage_endpoint <- function(x, ...)
|
|||
type <- sub("_endpoint$", "", class(x)[1])
|
||||
cat(sprintf("Azure %s storage endpoint\n", type))
|
||||
cat(sprintf("URL: %s\n", x$url))
|
||||
|
||||
if(!is_empty(x$key))
|
||||
cat("Access key: <hidden>\n")
|
||||
else cat("Access key: <none supplied>\n")
|
||||
|
||||
if(!is_empty(x$token))
|
||||
{
|
||||
cat("Azure Active Directory token:\n")
|
||||
print(x$token)
|
||||
}
|
||||
else cat("Azure Active Directory token: <none supplied>\n")
|
||||
|
||||
if(!is_empty(x$sas))
|
||||
cat("Account shared access signature: <hidden>\n")
|
||||
else cat("Account shared access signature: <none supplied>\n")
|
||||
|
||||
cat(sprintf("Storage API version: %s\n", x$api_version))
|
||||
invisible(x)
|
||||
}
|
||||
|
|
@ -114,12 +144,22 @@ print.adls_endpoint <- function(x, ...)
|
|||
{
|
||||
cat("Azure Data Lake Storage Gen2 endpoint\n")
|
||||
cat(sprintf("URL: %s\n", x$url))
|
||||
|
||||
if(!is_empty(x$key))
|
||||
cat("Access key: <hidden>\n")
|
||||
else cat("Access key: <none supplied>\n")
|
||||
|
||||
if(!is_empty(x$token))
|
||||
{
|
||||
cat("Azure Active Directory token:\n")
|
||||
print(x$token)
|
||||
}
|
||||
else cat("Azure Active Directory token: <none supplied>\n")
|
||||
|
||||
if(!is_empty(x$sas))
|
||||
cat("Account shared access signature: <hidden>\n")
|
||||
else cat("Account shared access signature: <none supplied>\n")
|
||||
|
||||
cat(sprintf("Storage API version: %s\n", x$api_version))
|
||||
invisible(x)
|
||||
}
|
||||
|
|
@ -129,7 +169,7 @@ print.adls_endpoint <- function(x, ...)
|
|||
#' Generic upload and download
|
||||
#'
|
||||
#' @param src,dest The source and destination files/URLs. Paths are allowed.
|
||||
#' @param key,sas Authentication arguments: an access key or a shared access signature (SAS). If a key is is provided, the SAS is not used. If neither an access key nor a SAS are provided, only public (anonymous) access to the share is possible.
|
||||
#' @param key,token,sas Authentication arguments: an access key or a shared access signature (SAS). If a key is is provided, the SAS is not used. If neither an access key nor a SAS are provided, only public (anonymous) access to the share is possible.
|
||||
#' @param ... Further arguments to pass to lower-level functions.
|
||||
#' @param overwrite For downloading, whether to overwrite any destination files that exist.
|
||||
#'
|
||||
|
|
@ -152,12 +192,12 @@ print.adls_endpoint <- function(x, ...)
|
|||
#' }
|
||||
#' @rdname file_transfer
|
||||
#' @export
|
||||
download_from_url <- function(src, dest, key=NULL, sas=NULL, ..., overwrite=FALSE)
|
||||
download_from_url <- function(src, dest, key=NULL, token=NULL, sas=NULL, ..., overwrite=FALSE)
|
||||
{
|
||||
az_path <- parse_storage_url(src)
|
||||
if(is.null(sas))
|
||||
sas <- find_sas(src)
|
||||
endpoint <- storage_endpoint(az_path[1], key=key, sas=sas, ...)
|
||||
endpoint <- storage_endpoint(az_path[1], key=key, token=token, sas=sas, ...)
|
||||
|
||||
if(inherits(endpoint, "blob_endpoint"))
|
||||
{
|
||||
|
|
@ -180,12 +220,12 @@ download_from_url <- function(src, dest, key=NULL, sas=NULL, ..., overwrite=FALS
|
|||
|
||||
#' @rdname file_transfer
|
||||
#' @export
|
||||
upload_to_url <- function(src, dest, key=NULL, sas=NULL, ...)
|
||||
upload_to_url <- function(src, dest, key=NULL, token=token, sas=NULL, ...)
|
||||
{
|
||||
az_path <- parse_storage_url(dest)
|
||||
if(is.null(sas))
|
||||
sas <- find_sas(dest)
|
||||
endpoint <- storage_endpoint(az_path[1], key=key, sas=sas, ...)
|
||||
endpoint <- storage_endpoint(az_path[1], key=key, token=token, sas=sas, ...)
|
||||
|
||||
if(inherits(endpoint, "blob_endpoint"))
|
||||
{
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ file_share.character <- function(endpoint, key=NULL, sas=NULL,
|
|||
api_version=getOption("azure_storage_api_version"),
|
||||
...)
|
||||
{
|
||||
do.call(file_share, generate_endpoint_container(endpoint, key, sas, api_version))
|
||||
do.call(file_share, generate_endpoint_container(endpoint, key, token=NULL, sas, api_version))
|
||||
}
|
||||
|
||||
#' @rdname file_share
|
||||
|
|
@ -95,7 +95,7 @@ list_file_shares.character <- function(endpoint, key=NULL, sas=NULL,
|
|||
api_version=getOption("azure_storage_api_version"),
|
||||
...)
|
||||
{
|
||||
do.call(list_file_shares, generate_endpoint_container(endpoint, key, sas, api_version))
|
||||
do.call(list_file_shares, generate_endpoint_container(endpoint, key, token=NULL, sas, api_version))
|
||||
}
|
||||
|
||||
#' @rdname file_share
|
||||
|
|
@ -124,7 +124,7 @@ create_file_share.character <- function(endpoint, key=NULL, sas=NULL,
|
|||
api_version=getOption("azure_storage_api_version"),
|
||||
...)
|
||||
{
|
||||
endp <- generate_endpoint_container(endpoint, key, sas, api_version)
|
||||
endp <- generate_endpoint_container(endpoint, key, token=NULL, sas, api_version)
|
||||
create_file_share(endp$endpoint, endp$name, ...)
|
||||
}
|
||||
|
||||
|
|
@ -159,7 +159,7 @@ delete_file_share.character <- function(endpoint, key=NULL, sas=NULL,
|
|||
api_version=getOption("azure_storage_api_version"),
|
||||
...)
|
||||
{
|
||||
endp <- generate_endpoint_container(endpoint, key, sas, api_version)
|
||||
endp <- generate_endpoint_container(endpoint, key, token=NULL, sas, api_version)
|
||||
delete_file_share(endp$endpoint, endp$name, ...)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -8,13 +8,13 @@ do_container_op <- function(container, path="", options=list(), headers=list(),
|
|||
else container$name
|
||||
|
||||
invisible(do_storage_call(endp$url, path, options=options, headers=headers,
|
||||
key=endp$key, sas=endp$sas, api_version=endp$api_version,
|
||||
key=endp$key, token=endp$token, sas=endp$sas, api_version=endp$api_version,
|
||||
http_verb=http_verb, ...))
|
||||
}
|
||||
|
||||
|
||||
do_storage_call <- function(endpoint_url, path, options=list(), headers=list(), body=NULL, ...,
|
||||
key=NULL, sas=NULL,
|
||||
key=NULL, token=NULL, sas=NULL,
|
||||
api_version=getOption("azure_storage_api_version"),
|
||||
http_verb=c("GET", "DELETE", "PUT", "POST", "HEAD", "PATCH"),
|
||||
http_status_handler=c("stop", "warn", "message", "pass"))
|
||||
|
|
@ -25,9 +25,11 @@ do_storage_call <- function(endpoint_url, path, options=list(), headers=list(),
|
|||
if(!is_empty(options))
|
||||
url$query <- options[order(names(options))] # must be sorted for access key signing
|
||||
|
||||
# use key if provided, otherwise sas if provided, otherwise anonymous access
|
||||
# use key if provided, otherwise AAD token if provided, otherwise sas if provided, otherwise anonymous access
|
||||
if(!is.null(key))
|
||||
headers <- sign_request(key, verb, url, headers, api_version)
|
||||
else if(!is.null(token))
|
||||
headers <- add_token(token, headers, api_version)
|
||||
else if(!is.null(sas))
|
||||
url <- add_sas(sas, url)
|
||||
|
||||
|
|
@ -59,6 +61,32 @@ do_storage_call <- function(endpoint_url, path, options=list(), headers=list(),
|
|||
}
|
||||
|
||||
|
||||
add_token <- function(token, headers, api)
|
||||
{
|
||||
if(is.null(headers$`x-ms-version`))
|
||||
headers$`x-ms-version` <- api
|
||||
|
||||
if(inherits(token, "R6") && inherits(token, "AzureToken"))
|
||||
{
|
||||
# if token has expired, renew it
|
||||
if(!token$validate())
|
||||
{
|
||||
message("Access token has expired or is no longer valid; refreshing")
|
||||
token$refresh()
|
||||
}
|
||||
type <- token$credentials$token_type
|
||||
token <- token$credentials$access_token
|
||||
}
|
||||
else
|
||||
{
|
||||
if(!is.character(token) || length(token) != 1)
|
||||
stop("Token must be a string, or an object of class AzureRMR::AzureToken", call.=FALSE)
|
||||
type <- "Bearer"
|
||||
}
|
||||
c(headers, Authorization=paste(type, token))
|
||||
}
|
||||
|
||||
|
||||
add_sas <- function(sas, url)
|
||||
{
|
||||
full_url <- httr::build_url(url)
|
||||
|
|
@ -169,10 +197,10 @@ is_endpoint_url <- function(url, type)
|
|||
}
|
||||
|
||||
|
||||
generate_endpoint_container <- function(url, key, sas, api_version)
|
||||
generate_endpoint_container <- function(url, key, token, sas, api_version)
|
||||
{
|
||||
stor_path <- parse_storage_url(url)
|
||||
endpoint <- storage_endpoint(stor_path[1], key, sas, api_version)
|
||||
endpoint <- storage_endpoint(stor_path[1], key, token, sas, api_version)
|
||||
name <- stor_path[2]
|
||||
list(endpoint=endpoint, name=name)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,8 +20,9 @@
|
|||
\usage{
|
||||
adls_filesystem(endpoint, ...)
|
||||
|
||||
\method{adls_filesystem}{character}(endpoint, key = NULL, sas = NULL,
|
||||
api_version = getOption("azure_storage_api_version"), ...)
|
||||
\method{adls_filesystem}{character}(endpoint, key = NULL, token = NULL,
|
||||
sas = NULL, api_version = getOption("azure_storage_api_version"),
|
||||
...)
|
||||
|
||||
\method{adls_filesystem}{adls_endpoint}(endpoint, name, ...)
|
||||
|
||||
|
|
@ -30,14 +31,16 @@ adls_filesystem(endpoint, ...)
|
|||
list_adls_filesystems(endpoint, ...)
|
||||
|
||||
\method{list_adls_filesystems}{character}(endpoint, key = NULL,
|
||||
sas = NULL, api_version = getOption("azure_adls_api_version"), ...)
|
||||
token = NULL, sas = NULL,
|
||||
api_version = getOption("azure_adls_api_version"), ...)
|
||||
|
||||
\method{list_adls_filesystems}{adls_endpoint}(endpoint, ...)
|
||||
|
||||
create_adls_filesystem(endpoint, ...)
|
||||
|
||||
\method{create_adls_filesystem}{character}(endpoint, key = NULL,
|
||||
sas = NULL, api_version = getOption("azure_adls_api_version"), ...)
|
||||
token = NULL, sas = NULL,
|
||||
api_version = getOption("azure_adls_api_version"), ...)
|
||||
|
||||
\method{create_adls_filesystem}{adls_filesystem}(endpoint, ...)
|
||||
|
||||
|
|
@ -46,7 +49,8 @@ create_adls_filesystem(endpoint, ...)
|
|||
delete_adls_filesystem(endpoint, ...)
|
||||
|
||||
\method{delete_adls_filesystem}{character}(endpoint, key = NULL,
|
||||
sas = NULL, api_version = getOption("azure_adls_api_version"), ...)
|
||||
token = NULL, sas = NULL,
|
||||
api_version = getOption("azure_adls_api_version"), ...)
|
||||
|
||||
\method{delete_adls_filesystem}{adls_filesystem}(endpoint, ...)
|
||||
|
||||
|
|
@ -58,7 +62,7 @@ delete_adls_filesystem(endpoint, ...)
|
|||
|
||||
\item{...}{Further arguments passed to lower-level functions.}
|
||||
|
||||
\item{key, sas}{If an endpoint object is not supplied, authentication details. Currently the \code{sas} argument is unused.}
|
||||
\item{key, token, sas}{If an endpoint object is not supplied, authentication credentials: either an access key, an Azure Active Directory (AAD) token, or a SAS, in that order of priority. Currently the \code{sas} argument is unused.}
|
||||
|
||||
\item{api_version}{If an endpoint object is not supplied, the storage API version to use when interacting with the host. Currently defaults to \code{"2018-06-17"}.}
|
||||
|
||||
|
|
@ -79,6 +83,8 @@ Get, list, create, or delete ADLSgen2 filesystems. Currently (as of December 201
|
|||
\details{
|
||||
You can call these functions in a couple of ways: by passing the full URL of the share, or by passing the endpoint object and the name of the share as a string.
|
||||
|
||||
If authenticating via AAD, you can supply the token either as a string, or as an object of class \link[AzureRMR:AzureToken]{AzureRMR::AzureToken}, created via \link[AzureRMR:get_azure_token]{AzureRMR::get_azure_token}. The latter is the recommended way of doing it, as it allows for automatic refreshing of expired tokens.
|
||||
|
||||
Currently (as of December 2018), if the storage account has hierarchical namespaces enabled, the blob API for the account is disabled. The blob endpoint is still accessible, but blob operations on the endpoint will fail. Full interoperability between blobs and ADLS is planned for 2019.
|
||||
}
|
||||
\examples{
|
||||
|
|
|
|||
|
|
@ -20,8 +20,9 @@
|
|||
\usage{
|
||||
blob_container(endpoint, ...)
|
||||
|
||||
\method{blob_container}{character}(endpoint, key = NULL, sas = NULL,
|
||||
api_version = getOption("azure_storage_api_version"), ...)
|
||||
\method{blob_container}{character}(endpoint, key = NULL, token = NULL,
|
||||
sas = NULL, api_version = getOption("azure_storage_api_version"),
|
||||
...)
|
||||
|
||||
\method{blob_container}{blob_endpoint}(endpoint, name, ...)
|
||||
|
||||
|
|
@ -30,16 +31,16 @@ blob_container(endpoint, ...)
|
|||
list_blob_containers(endpoint, ...)
|
||||
|
||||
\method{list_blob_containers}{character}(endpoint, key = NULL,
|
||||
sas = NULL, api_version = getOption("azure_storage_api_version"),
|
||||
...)
|
||||
token = NULL, sas = NULL,
|
||||
api_version = getOption("azure_storage_api_version"), ...)
|
||||
|
||||
\method{list_blob_containers}{blob_endpoint}(endpoint, ...)
|
||||
|
||||
create_blob_container(endpoint, ...)
|
||||
|
||||
\method{create_blob_container}{character}(endpoint, key = NULL,
|
||||
sas = NULL, api_version = getOption("azure_storage_api_version"),
|
||||
...)
|
||||
token = NULL, sas = NULL,
|
||||
api_version = getOption("azure_storage_api_version"), ...)
|
||||
|
||||
\method{create_blob_container}{blob_container}(endpoint, ...)
|
||||
|
||||
|
|
@ -49,8 +50,8 @@ create_blob_container(endpoint, ...)
|
|||
delete_blob_container(endpoint, ...)
|
||||
|
||||
\method{delete_blob_container}{character}(endpoint, key = NULL,
|
||||
sas = NULL, api_version = getOption("azure_storage_api_version"),
|
||||
...)
|
||||
token = NULL, sas = NULL,
|
||||
api_version = getOption("azure_storage_api_version"), ...)
|
||||
|
||||
\method{delete_blob_container}{blob_container}(endpoint, ...)
|
||||
|
||||
|
|
@ -62,7 +63,7 @@ delete_blob_container(endpoint, ...)
|
|||
|
||||
\item{...}{Further arguments passed to lower-level functions.}
|
||||
|
||||
\item{key, sas}{If an endpoint object is not supplied, authentication details. If a key is provided, the SAS is not used. If neither an access key nor a SAS are provided, only public (anonymous) access to the share is possible.}
|
||||
\item{key, token, sas}{If an endpoint object is not supplied, authentication credentials: either an access key, an Azure Active Directory (AAD) token, or a SAS, in that order of priority. If no authentication credentials are provided, only public (anonymous) access to the share is possible.}
|
||||
|
||||
\item{api_version}{If an endpoint object is not supplied, the storage API version to use when interacting with the host. Currently defaults to \code{"2018-03-28"}.}
|
||||
|
||||
|
|
@ -87,6 +88,8 @@ Get, list, create, or delete blob containers.
|
|||
\details{
|
||||
You can call these functions in a couple of ways: by passing the full URL of the share, or by passing the endpoint object and the name of the container as a string.
|
||||
|
||||
If authenticating via AAD, you can supply the token either as a string, or as an object of class \link[AzureRMR:AzureToken]{AzureRMR::AzureToken}, created via \link[AzureRMR:get_azure_token]{AzureRMR::get_azure_token}. The latter is the recommended way of doing it, as it allows for automatic refreshing of expired tokens.
|
||||
|
||||
Currently (as of December 2018), if the storage account has hierarchical namespaces enabled, the blob API for the account is disabled. The blob endpoint is still accessible, but blob operations on the endpoint will fail. Full interoperability between blobs and ADLS is planned for 2019.
|
||||
}
|
||||
\examples{
|
||||
|
|
@ -107,6 +110,13 @@ blob_container("https://mystorage.blob.core.windows.net/mycontainer", key="acces
|
|||
create_blob_container("https://mystorage.blob.core.windows.net/newcontainer", key="access_key")
|
||||
delete_blob_container("https://mystorage.blob.core.windows.net/newcontainer", key="access_key")
|
||||
|
||||
# authenticating via AAD
|
||||
token <- AzureRMR::get_azure_token(resource_host="https://storage.azure.com/",
|
||||
tenant="myaadtenant",
|
||||
app="myappid",
|
||||
password="mypassword")
|
||||
blob_container("https://mystorage.blob.core.windows.net/mycontainer", token=token)
|
||||
|
||||
}
|
||||
}
|
||||
\seealso{
|
||||
|
|
|
|||
|
|
@ -5,15 +5,15 @@
|
|||
\alias{upload_to_url}
|
||||
\title{Generic upload and download}
|
||||
\usage{
|
||||
download_from_url(src, dest, key = NULL, sas = NULL, ...,
|
||||
overwrite = FALSE)
|
||||
download_from_url(src, dest, key = NULL, token = NULL, sas = NULL,
|
||||
..., overwrite = FALSE)
|
||||
|
||||
upload_to_url(src, dest, key = NULL, sas = NULL, ...)
|
||||
upload_to_url(src, dest, key = NULL, token = token, sas = NULL, ...)
|
||||
}
|
||||
\arguments{
|
||||
\item{src, dest}{The source and destination files/URLs. Paths are allowed.}
|
||||
|
||||
\item{key, sas}{Authentication arguments: an access key or a shared access signature (SAS). If a key is is provided, the SAS is not used. If neither an access key nor a SAS are provided, only public (anonymous) access to the share is possible.}
|
||||
\item{key, token, sas}{Authentication arguments: an access key or a shared access signature (SAS). If a key is is provided, the SAS is not used. If neither an access key nor a SAS are provided, only public (anonymous) access to the share is possible.}
|
||||
|
||||
\item{...}{Further arguments to pass to lower-level functions.}
|
||||
|
||||
|
|
|
|||
|
|
@ -12,16 +12,16 @@
|
|||
\alias{print.adls_endpoint}
|
||||
\title{Create a storage endpoint object}
|
||||
\usage{
|
||||
storage_endpoint(endpoint, key = NULL, sas = NULL,
|
||||
storage_endpoint(endpoint, key = NULL, token = NULL, sas = NULL,
|
||||
api_version)
|
||||
|
||||
blob_endpoint(endpoint, key = NULL, token = NULL, sas = NULL,
|
||||
api_version = getOption("azure_storage_api_version"))
|
||||
|
||||
blob_endpoint(endpoint, key = NULL, sas = NULL,
|
||||
file_endpoint(endpoint, key = NULL, token = NULL, sas = NULL,
|
||||
api_version = getOption("azure_storage_api_version"))
|
||||
|
||||
file_endpoint(endpoint, key = NULL, sas = NULL,
|
||||
api_version = getOption("azure_storage_api_version"))
|
||||
|
||||
adls_endpoint(endpoint, key = NULL, sas = NULL,
|
||||
adls_endpoint(endpoint, key = NULL, token = NULL, sas = NULL,
|
||||
api_version = getOption("azure_adls_api_version"))
|
||||
|
||||
\method{print}{storage_endpoint}(x, ...)
|
||||
|
|
@ -33,7 +33,9 @@ adls_endpoint(endpoint, key = NULL, sas = NULL,
|
|||
|
||||
\item{key}{The access key for the storage account.}
|
||||
|
||||
\item{sas}{A shared access signature (SAS) for the account. If \code{key} is also provided, the SAS is not used. If neither \code{key} nor \code{sas} are provided, only public (anonymous) access to the endpoint is possible. Note that authentication with a SAS is not supported by ADLSgen2.}
|
||||
\item{token}{An Azure Active Directory (AAD) authentication token. This can be either a string, or an object of class \link[AzureRMR:AzureToken]{AzureRMR::AzureToken} created by \link[AzureRMR:get_azure_token]{AzureRMR::get_azure_token}. The latter is the recommended way of doing it, as it allows for automatic refreshing of expired tokens.}
|
||||
|
||||
\item{sas}{A shared access signature (SAS) for the account.}
|
||||
|
||||
\item{api_version}{The storage API version to use when interacting with the host. Defaults to \code{"2018-06-17"} for the ADLSgen2 endpoint, and \code{"2018-03-28"} for the others.}
|
||||
|
||||
|
|
@ -53,6 +55,8 @@ Create a storage endpoint object, for interacting with blob, file, table, queue
|
|||
}
|
||||
\details{
|
||||
This is the starting point for the client-side storage interface in AzureRMR. \code{storage_endpoint} is a generic function to create an endpoint for any type of Azure storage while \code{adls_endpoint}, \code{blob_endpoint} and \code{file_endpoint} create endpoints for those types.
|
||||
|
||||
If multiple authentication objects are supplied, they are used in this order of priority: first an access key, then an AAD token, then a SAS. If no authentication objects are supplied, only public (anonymous) access to the endpoint is possible. Note that authentication with a SAS is not currently supported by ADLSgen2, and authentication with an AAD token is not supported by file storage.
|
||||
}
|
||||
\examples{
|
||||
\dontrun{
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче