adls bug fixes (#42)

DESCRIPTION

@@ -26,4 +26,4 @@ Suggests:
     testthat,
     processx
 Roxygen: list(markdown=TRUE, r6=FALSE, old_usage=TRUE)
-RoxygenNote: 7.0.2
+RoxygenNote: 7.0.2.9000

NEWS.md

@@ -1,6 +1,8 @@
 # AzureStor 3.1.0.9000
 
 - Expose `sign_request` as a generic, dispatching on the endpoint class. This is to allow for the fact that table storage uses a different signing scheme to the other storage services.
+- Fix a bug in `list_adls_files` that could result in an error with large numbers of files.
+- Add a `timeout` argument to `call_storage_endpoint` that sets the number of seconds to wait for an API call.
 
 # AzureStor 3.1.0
 

R/adls_client_funcs.R

@@ -295,7 +295,38 @@ list_adls_files <- function(filesystem, dir="/", info=c("all", "name"),
     {
         res <- do_container_op(filesystem, "", options=opts, http_status_handler="pass")
         httr::stop_for_status(res, storage_error_message(res))
-        out <- rbind(out, httr::content(res, simplifyVector=TRUE)$paths)
+        dfchunk <- httr::content(res, simplifyVector=TRUE)$paths
+
+        # normalise output
+        if(is.null(dfchunk$isDirectory))
+            dfchunk$isDirectory <- FALSE
+        else dfchunk$isDirectory <- !is.na(dfchunk$isDirectory)
+        if(is.null(dfchunk$contentLength))
+            dfchunk$contentLength <- NA_real_
+        else dfchunk$contentLength[is.na(dfchunk$contentLength)] <- NA_real_
+        if(is.null(dfchunk$etag))
+            dfchunk$etag <- ""
+        else dfchunk$etag[is.na(dfchunk$etag)] <- ""
+        if(is.null(dfchunk$permissions))
+            dfchunk$permissions <- ""
+        else dfchunk$permissions[is.na(dfchunk$permissions)] <- ""
+
+        dfchunk <- dfchunk[c("name", "contentLength", "isDirectory", "lastModified", "permissions", "etag")]
+        if(!is.null(dfchunk$contentLength))
+            dfchunk$contentLength <- as.numeric(dfchunk$contentLength)
+        if(!is.null(dfchunk$lastModified))
+            dfchunk$lastModified <- as_datetime(dfchunk$lastModified)
+        names(dfchunk)[c(2, 3)] <- c("size", "isdir")
+
+        if(all(dfchunk$permissions == ""))
+            dfchunk$permissions <- NULL
+        if(all(dfchunk$etag == ""))
+            dfchunk$etag <- NULL
+
+        # needed when dir was created in a non-HNS enabled account
+        dfchunk$size[dfchunk$isdir] <- NA
+
+        out <- rbind(out, dfchunk)
         headers <- httr::headers(res)
         if(is_empty(headers$`x-ms-continuation`))
             break
@@ -308,38 +339,11 @@ list_adls_files <- function(filesystem, dir="/", info=c("all", "name"),
         if(is_empty(out))
             return(data.frame(
                 name=character(0),
-                contentLength=numeric(0),
-                isDirectory=logical(0),
-                lastModified=numeric(0)))
-
-        # normalise output
-        if(is.null(out$isDirectory))
-            out$isDirectory <- FALSE
-        else out$isDirectory <- !is.na(out$isDirectory)
-        if(is.null(out$contentLength))
-            out$contentLength <- NA_real_
-        else out$contentLength[is.na(out$contentLength)] <- NA_real_
-        if(is.null(out$etag))
-            out$etag <- ""
-        else out$etag[is.na(out$etag)] <- ""
-        if(is.null(out$permissions))
-            out$permissions <- ""
-        else out$permissions[is.na(out$permissions)] <- ""
-
-        out <- out[c("name", "contentLength", "isDirectory", "lastModified", "permissions", "etag")]
-        if(!is.null(out$contentLength))
-            out$contentLength <- as.numeric(out$contentLength)
-        if(!is.null(out$lastModified))
-            out$lastModified <- as_datetime(out$lastModified)
-        names(out)[c(2, 3)] <- c("size", "isdir")
-
-        if(all(out$permissions == ""))
-            out$permissions <- NULL
-        if(all(out$etag == ""))
-            out$etag <- NULL
-
-        # needed when dir was created in a non-HNS enabled account
-        out$size[out$isdir] <- NA
+                size=numeric(0),
+                isdir=logical(0),
+                lastModified=structure(numeric(0), class=c("POSIXct", "POSIXt"), tzone="GMT"),
+                permissions=character(0),
+                etag=character(0)))
 
         out
     }

R/storage_utils.R

@@ -9,6 +9,7 @@
 #' @param ... Any additional arguments to pass to `httr::VERB`.
 #' @param http_verb The HTTP verb as a string, one of `GET`, `DELETE`, `PUT`, `POST`, `HEAD` or `PATCH`.
 #' @param http_status_handler The R handler for the HTTP status code of the response. `"stop"`, `"warn"` or `"message"` will call the corresponding handlers in httr, while `"pass"` ignores the status code. The latter is primarily useful for debugging purposes.
+#' @param timeout Optionally, the number of seconds to wait for a result. If the timeout interval elapses before the storage service has finished processing the operation, it returns an error. The default timeout is taken from the system option `azure_storage_timeout`; if this is `NULL` it means to use the service default.
 #' @param progress Used by the file transfer functions, to display a progress bar.
 #' @param return_headers Whether to return the (parsed) response headers, rather than the body. Ignored if `http_status_handler="pass"`.
 #' @details
@@ -51,6 +52,7 @@ do_container_op <- function(container, operation="", options=list(), headers=lis
 call_storage_endpoint <- function(endpoint, path, options=list(), headers=list(), body=NULL, ...,
                                   http_verb=c("GET", "DELETE", "PUT", "POST", "HEAD", "PATCH"),
                                   http_status_handler=c("stop", "warn", "message", "pass"),
+                                  timeout=getOption("azure_storage_timeout"),
                                   progress=NULL, return_headers=(http_verb == "HEAD"))
 {
     http_verb <- match.arg(http_verb)
@@ -59,6 +61,8 @@ call_storage_endpoint <- function(endpoint, path, options=list(), headers=list()
     if(!is_empty(options))
         url$query <- options[order(names(options))] # must be sorted for access key signing
 
+    options$timeout <- timeout
+
     # use key if provided, otherwise AAD token if provided, otherwise sas if provided, otherwise anonymous access
     if(!is.null(endpoint$key))
         headers <- sign_request(endpoint, http_verb, url, headers, endpoint$api_version)

man/storage_call.Rd

@@ -11,7 +11,8 @@ do_container_op(container, operation = "", options = list(),
 call_storage_endpoint(endpoint, path, options = list(), headers = list(),
   body = NULL, ..., http_verb = c("GET", "DELETE", "PUT", "POST", "HEAD",
   "PATCH"), http_status_handler = c("stop", "warn", "message", "pass"),
-  progress = NULL, return_headers = (http_verb == "HEAD"))
+  timeout = getOption("azure_storage_timeout"), progress = NULL,
+  return_headers = (http_verb == "HEAD"))
 }
 \arguments{
 \item{container, endpoint}{For \code{do_container_op}, a storage container object (inheriting from \code{storage_container}). For \code{call_storage_endpoint}, a  storage endpoint object (inheriting from \code{storage_endpoint}).}
@@ -32,6 +33,8 @@ call_storage_endpoint(endpoint, path, options = list(), headers = list(),
 
 \item{http_status_handler}{The R handler for the HTTP status code of the response. \code{"stop"}, \code{"warn"} or \code{"message"} will call the corresponding handlers in httr, while \code{"pass"} ignores the status code. The latter is primarily useful for debugging purposes.}
 
+\item{timeout}{Optionally, the number of seconds to wait for a result. If the timeout interval elapses before the storage service has finished processing the operation, it returns an error. The default timeout is taken from the system option \code{azure_storage_timeout}; if this is \code{NULL} it means to use the service default.}
+
 \item{progress}{Used by the file transfer functions, to display a progress bar.}
 
 \item{return_headers}{Whether to return the (parsed) response headers, rather than the body. Ignored if \code{http_status_handler="pass"}.}