diff --git a/DESCRIPTION b/DESCRIPTION index 9174c02..036bafc 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,12 +1,12 @@ Package: AzureSMR -Title: Manage and Interact with Azure Resources. +Title: Manage and Interact with Azure Resources Description: Helps users to manage Azure Services and objects from within an R Session. This includes Azure Storage (e.g. containers and blobs), Virtual Machines and HDInsight (Spark, Hive). To use the package, you must configure an Azure Active Directory application and service principal in the Azure portal. Type: Package -Version: 0.2.3 -Date: 2017-05-21 +Version: 0.2.4 +Date: 2017-05-30 Authors@R: c( person(family="Microsoft Corporation", role="cph"), person("Alan", "Weaver", role=c("aut", "cre"), email="alanwe@microsoft.com"), @@ -22,7 +22,6 @@ Imports: httr, jsonlite, XML, - plyr, base64enc, digest Depends: diff --git a/NAMESPACE b/NAMESPACE index 1d86a2f..b5c4d85 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -2,6 +2,7 @@ S3method(print,azureActiveContext) S3method(str,azureActiveContext) +S3method(summary,azureScriptActionHistory) export(AzureListRG) export(as.azureActiveContext) export(azureAuthenticate) @@ -77,7 +78,6 @@ importFrom(httr,headers) importFrom(httr,http_status) importFrom(httr,status_code) importFrom(jsonlite,fromJSON) -importFrom(plyr,rbind.fill) importFrom(utils,URLencode) importFrom(utils,browseURL) importFrom(utils,ls.str) diff --git a/R/AzureContainer.R b/R/AzureContainer.R index f4e4975..c6c4a7c 100644 --- a/R/AzureContainer.R +++ b/R/AzureContainer.R @@ -28,7 +28,6 @@ azureListStorageContainers <- function(azureActiveContext, storageAccount, stora verbosity <- set_verbosity(verbose) URL <- paste0("http://", storageAccount, ".blob.core.windows.net/?comp=list") - #browser() xdate <- x_ms_date() SIG <- getSig(azureActiveContext, url = URL, verb = "GET", key = storageKey, storageAccount = storageAccount, CMD = "\ncomp:list", date = xdate) diff --git a/R/AzureHDI.R b/R/AzureHDI.R index 39b6880..135a610 100644 --- a/R/AzureHDI.R +++ b/R/AzureHDI.R @@ -5,8 +5,9 @@ #' @inheritParams azureListAllResources #' #' @family HDInsight functions +#' @references https://docs.microsoft.com/en-us/rest/api/hdinsight/hdinsight-cluster#list-by-subscription #' -#' @return Returns Dataframe of HDInsight Clusters +#' @return data frame with summary information of HDI clusters #' @export azureListHDI <- function(azureActiveContext, resourceGroup, clustername = "*", subscriptionID, name, type, location, verbose = FALSE) { @@ -17,7 +18,7 @@ azureListHDI <- function(azureActiveContext, resourceGroup, clustername = "*", if (missing(subscriptionID)) subscriptionID <- azureActiveContext$subscriptionID if (missing(resourceGroup)) resourceGroup <- azureActiveContext$resourceGroup verbosity <- set_verbosity(verbose) - + assert_that(is_subscription_id(subscriptionID)) if (clustername != "*") { assert_that(is_clustername(clustername)) @@ -26,33 +27,36 @@ azureListHDI <- function(azureActiveContext, resourceGroup, clustername = "*", rg <- if (clustername == "*") "" else paste0("/resourceGroups/", resourceGroup) cn <- if (clustername == "*") "" else clustername - + URL <- paste0("https://management.azure.com/subscriptions/", subscriptionID, rg, - "/providers/Microsoft.HDInsight/clusters/", cn, + "/providers/Microsoft.HDInsight/clusters/", cn, "?api-version=2015-03-01-preview") r <- GET(URL, azureApiHeaders(azToken), verbosity) stopWithAzureError(r) - - rc <- content(r)$value - z <- do.call(rbind, lapply(rc, function(x) { - as.data.frame( + rc <- content(r) + extract_one <- function(x) { + as.data.frame( c( x[c("name", "id", "location", "type")], x$properties[c("tier", "osType", "provisioningState", "clusterState", "createdDate")], x$properties$clusterDefinition[c("kind")] - ) - ) - })) + )) + } + + z <- if (is.null(rc$value)) { + extract_one(rc) + } else { + do.call(rbind, lapply(rc$value, extract_one)) + } azureActiveContext$resourceGroup <- resourceGroup - return(z) } -#' Get Configuration Information for a specified cluster name. +#' Get configuration information for a specified cluster name. #' #' @inheritParams setAzureContext #' @inheritParams azureAuthenticate @@ -64,69 +68,57 @@ azureListHDI <- function(azureActiveContext, resourceGroup, clustername = "*", #' @export azureHDIConf <- function(azureActiveContext, clustername, resourceGroup, subscriptionID, name, type, location, verbose = FALSE) { + assert_that(is.azureActiveContext(azureActiveContext)) azureCheckToken(azureActiveContext) azToken <- azureActiveContext$Token - if (missing(subscriptionID)) { - subscriptionID <- azureActiveContext$subscriptionID - } else (subscriptionID <- subscriptionID) - if (missing(resourceGroup)) { - resourceGroup <- azureActiveContext$resourceGroup - } else (resourceGroup <- resourceGroup) - if (missing(clustername)) { - clustername <- azureActiveContext$clustername - } else (clustername <- clustername) + if (missing(subscriptionID)) subscriptionID <- azureActiveContext$subscriptionID + if (missing(resourceGroup)) resourceGroup <- azureActiveContext$resourceGroup + if (missing(clustername)) clustername <- azureActiveContext$clustername + verbosity <- set_verbosity(verbose) + assert_that(is_subscription_id(subscriptionID)) + assert_that(is_resource_group(resourceGroup)) + assert_that(is_clustername(clustername)) - if (!length(azToken)) { - stop("Error: No Token / Not currently Authenticated.") - } - if (!length(subscriptionID)) { - stop("Error: No subscriptionID provided: Use SUBID argument or set in AzureContext") - } - if (!length(clustername)) { - stop("Error: No clustername Provided.") + URL <- paste0("https://management.azure.com/subscriptions/", subscriptionID, + "/resourceGroups/", resourceGroup, + "/providers/Microsoft.HDInsight/clusters/", clustername, + "?api-version=2015-03-01-preview") + + r <- GET(URL, azureApiHeaders(azToken), verbosity) + browser() + rc <- content(r) + + if (length(rc) == 0) { + warning("No HDInsight clusters found", immediate. = TRUE) } - URL <- paste("https://management.azure.com/subscriptions/", subscriptionID, - "/resourceGroups/", resourceGroup, "/providers/Microsoft.HDInsight/clusters/", - clustername, "?api-version=2015-03-01-preview", sep = "") + info <- paste(vapply(rc$properties$computeProfile$roles, function(x) { + sprintf("%s: %s * %s", + x$name, + x$targetInstanceCount, + x$hardwareProfile$vmSize)}, FUN.VALUE = character(1) + ), collapse = ", " + ) - r <- GET(URL, add_headers(.headers = c(Host = "management.azure.com", - Authorization = azToken, `Content-type` = "application/json")), verbosity) - rl <- content(r, "text") + dfn <- with(rc, data.frame( + name = name, + id = id, + location = location, + type = type, + tier = rc$properties$tier, + kind = rc$properties$clusterDefinition$kind, + osType = rc$properties$osType, + provisioningState = rc$properties$provisioningState, + status = rc$properties$clusterState, + created = rc$properties$createdDate, + numCores = rc$properties$quotaInfo$coresUsed, + information = info, + stringsAsFactors = FALSE + )) - df <- fromJSON(rl) - dfn <- as.data.frame(df$name) - clust <- nrow(dfn) - if (clust < 1) { - warning("No HDInsight Clusters found") - return(NULL) - } - - dfn[1, 1] <- df$name - dfn[1, 2] <- df$id - dfn[1, 3] <- df$location - dfn[1, 4] <- df$type - dfn[1, 5] <- df$properties$tier - dfn[1, 6] <- df$properties$clusterDefinition$kind - dfn[1, 7] <- df$properties$osType - dfn[1, 8] <- df$properties$provisioningState - dfn[1, 9] <- df$properties$clusterState - dfn[1, 10] <- df$properties$createdDate - dfn[1, 11] <- df$properties$quotaInfo$coresUsed - roles1 <- df$properties$computeProfile$roles - rt <- "" - for (i in 1:nrow(roles1)) { - row <- roles1[i, ] - rt <- paste(rt, row$name, "(", row$targetInstanceCount, "*", row[, - 3], ")") - } - dfn[1, 12] <- rt - colnames(dfn) <- c("name", "ID", "location", "type", "tier", "kind", - "OS", "provState", "status", "created", "numCores", "information") - - return(t(dfn)) + return(dfn) } #' Create HDInsight cluster. @@ -150,10 +142,12 @@ azureHDIConf <- function(azureActiveContext, clustername, resourceGroup, #' @param componentVersion Spark componentVersion. Default : 1.6.2 #' @param vmSize Size of nodes: "Large", "Small", "Standard_D14_V2", etc. #' @param mode Provisioning mode, "Sync" or "Async". Use "Async" to immediately return to R session after submission of request +#' @param debug Used for debugging purposes. If TRUE, returns json without attempting to connect to Azure #' #' @return Success message #' @family HDInsight functions #' @note See \url{https://docs.microsoft.com/en-us/azure/hdinsight/hdinsight-component-versioning} to learn about HDInsight Versions +#' @references https://docs.microsoft.com/en-us/rest/api/hdinsight/hdinsight-cluster#create #' @export azureCreateHDI <- function(azureActiveContext, resourceGroup, location, clustername, kind = c("rserver", "spark", "hadoop"), @@ -162,7 +156,7 @@ azureCreateHDI <- function(azureActiveContext, resourceGroup, location, workers = 2, adminUser, adminPassword, sshUser, sshPassword, hiveServer, hiveDB, hiveUser, hivePassword, - vmSize = "Small", + vmSize = "Large", subscriptionID, mode = c("Sync", "Async"), verbose = FALSE, debug = FALSE) { assert_that(is.azureActiveContext(azureActiveContext)) @@ -245,7 +239,6 @@ azureCreateHDI <- function(azureActiveContext, resourceGroup, location, "?api-version=2015-03-01-preview") r <- PUT(URL, azureApiHeaders(azToken), body = bodyI, encode = "json", verbosity) - #browser() stopWithAzureError(r) azureActiveContext$resourceGroup <- resourceGroup @@ -363,8 +356,9 @@ azureResizeHDI <- function(azureActiveContext, clustername, #' @inheritParams azureAuthenticate #' @inheritParams azureListHDI #' -#' @return Returns Dataframe of HDInsight Clusters information +#' @return Data frame with HDInsight clusters information #' @family HDInsight functions +#' @references https://docs.microsoft.com/en-us/rest/api/hdinsight/hdinsight-cluster#delete #' @export azureDeleteHDI <- function(azureActiveContext, clustername, subscriptionID, resourceGroup, verbose = FALSE) { @@ -382,8 +376,9 @@ azureDeleteHDI <- function(azureActiveContext, clustername, subscriptionID, assert_that(is_clustername(clustername)) URL <- paste0("https://management.azure.com/subscriptions/", subscriptionID, - "/resourceGroups/", resourceGroup, "/providers/Microsoft.HDInsight/clusters/", - clustername, "?api-version=2015-03-01-preview") + "/resourceGroups/", resourceGroup, + "/providers/Microsoft.HDInsight/clusters/", clustername, + "?api-version=2015-03-01-preview") r <- DELETE(URL, azureApiHeaders(azToken), verbosity) stopWithAzureError(r) @@ -405,14 +400,17 @@ azureDeleteHDI <- function(azureActiveContext, clustername, subscriptionID, #' @param workerNode install on worker nodes #' @param edgeNode install on worker nodes #' @param parameters parameters +#' @param wait If TRUE, runs script action synchronously, i.e. waits for successfull completion. If FALSE, submits the action asynchronously #' #' @return Returns Success Message #' @family HDInsight functions +#' @references https://docs.microsoft.com/en-us/rest/api/hdinsight/hdinsight-cluster#run-script-actions-on-a-running-cluster-linux-cluster-only #' @export azureRunScriptAction <- function(azureActiveContext, scriptname, scriptURL, headNode = TRUE, workerNode = FALSE, edgeNode = FALSE, clustername, resourceGroup, - parameters = "", subscriptionID, verbose = FALSE) { + parameters = "", subscriptionID, + wait = TRUE, verbose = FALSE) { assert_that(is.azureActiveContext(azureActiveContext)) azureCheckToken(azureActiveContext) azToken <- azureActiveContext$Token @@ -426,13 +424,8 @@ azureRunScriptAction <- function(azureActiveContext, scriptname, scriptURL, assert_that(is_subscription_id(subscriptionID)) assert_that(is_clustername(clustername)) - if (!length(scriptname)) { - stop("Error: No Valid scriptname provided") - } - if (!length(scriptURL)) { - stop("Error: No Valid scriptURL provided") - } - + if (!length(scriptname)) stop("Error: No Valid scriptname provided") + if (!length(scriptURL)) stop("Error: No Valid scriptURL provided") if (!any(headNode, workerNode, edgeNode)) { stop("Error: No role(headNode,workerNode,edgeNode) flag set to TRUE") } @@ -465,6 +458,7 @@ azureRunScriptAction <- function(azureActiveContext, scriptname, scriptURL, azureActiveContext$clustername <- clustername message("Accepted") + if (wait) pollStatusScriptAction(azureActiveContext, scriptname = scriptname) return(TRUE) } @@ -477,6 +471,7 @@ azureRunScriptAction <- function(azureActiveContext, scriptname, scriptURL, #' #' @return Dataframe of HDInsight Clusters #' @family HDInsight functions +#' @references https://docs.microsoft.com/en-us/rest/api/hdinsight/hdinsight-cluster#list-all-persisted-script-actions-for-a-cluster-linux-cluster-only #' @export azureScriptActionHistory <- function(azureActiveContext, resourceGroup, clustername = "*", subscriptionID, @@ -503,20 +498,26 @@ azureScriptActionHistory <- function(azureActiveContext, resourceGroup, stopWithAzureError(r) rc <- content(r)$value - if (length(rc) == 0) { message("No script action history found") } + class(rc) <- "azureScriptActionHistory" - dfn <- do.call(rbind, lapply(rc, function(x) { - data.frame( - x[c("name", "scriptExecutionId", "startTime")], - if (is.null(x$endTime)) list(endTime = NA) else x["endTime"], - x[c("status", "uri", "parameters")] - ) - })) azureActiveContext$clustername <- clustername - return(dfn) + return(rc) } +#' @export +#' @param object azureScriptActionHistory object, created by [azureScriptActionHistory()] +#' @param ... not used +#' @rdname azureScriptActionHistory +summary.azureScriptActionHistory <- function(object, ...) { + do.call(rbind, lapply(object, function(x) { + data.frame( + x[c("name", "scriptExecutionId", "startTime")], + if (is.null(x$endTime)) list(endTime = NA) else x["endTime"], + x[c("status", "uri", "parameters")] + ) + })) +} diff --git a/R/AzureResources.R b/R/AzureResources.R index c08359d..e1cc3f0 100644 --- a/R/AzureResources.R +++ b/R/AzureResources.R @@ -4,7 +4,7 @@ #' @inheritParams setAzureContext #' @inheritParams azureAuthenticate #' -#' @return Dataframe of subscriptionID; Sets AzureContext subscriptionID +#' @return data frame with subscriptionID; Sets AzureContext subscriptionID #' @family Resource group functions #' @export azureListSubscriptions <- function(azureActiveContext, verbose = FALSE) { @@ -18,8 +18,8 @@ azureListSubscriptions <- function(azureActiveContext, verbose = FALSE) { stopWithAzureError(r) dfs <- lapply(content(r), data.frame, stringsAsFactors = FALSE) - df1 <- do.call(rbind, dfs)) - if (nrow(df1) == 1) azureActiveContext$subscriptionID <- df1$subscriptionID[1] + df1 <- do.call(rbind, dfs) + if (nrow(df1) == 1) azureActiveContext$subscriptionID <- df1$subscriptionId[1] return(df1) } diff --git a/R/AzureSMR-package.R b/R/AzureSMR-package.R index 7667821..81111a0 100644 --- a/R/AzureSMR-package.R +++ b/R/AzureSMR-package.R @@ -16,13 +16,18 @@ #' - [azureListStorageContainers()] #' - [azureListStorageBlobs()] #' * Virtual Machines +#' - List VMs: [azureListVM()] #' - To create a virtual machine, use [azureDeployTemplate()] with a suitable template #' - Start a VM: [azureStartVM()] #' - Stop a VM: [azureStopVM()] #' - Get status: [azureVMStatus()] -#' - List VMs: [azureListVM()] -#' * HDInsight (Nodes, Hive, Spark) -#' * Azure Resource Manager +#' * HDInsight clusters: +#' - [azureListHDI()] +#' - [azureCreateHDI()] +#' - [azureResizeHDI()] +#' - [azureDeleteHDI()] +#' - [azureRunScriptAction()] +#' - [azureScriptActionHistory()] #' #' #' @name AzureSMR @@ -33,7 +38,6 @@ #' @importFrom utils browseURL URLencode ls.str str #' @importFrom digest hmac #' @importFrom base64enc base64encode base64decode -#' @importFrom plyr rbind.fill #' @importFrom jsonlite fromJSON #' @importFrom httr add_headers headers content status_code http_status authenticate #' @importFrom httr GET PUT DELETE POST diff --git a/R/AzureStorageAccount.R b/R/AzureStorageAccount.R index 47455a9..cd2e1cd 100644 --- a/R/AzureStorageAccount.R +++ b/R/AzureStorageAccount.R @@ -79,6 +79,7 @@ azureSAGetKey <- function(azureActiveContext, storageAccount, #' @inheritParams azureAuthenticate #' @inheritParams azureSAGetKey #' @param location A string for the location to create storage account +#' @param asynchronous If TRUE, submits asynchronous request to Azure. Otherwise waits until storage account is created. #' @family Storage account functions #' @export azureCreateStorageAccount <- function(azureActiveContext, storageAccount, diff --git a/R/azureScalesets.R b/R/azureScalesets.R index 828cc02..61bbebf 100644 --- a/R/azureScalesets.R +++ b/R/azureScalesets.R @@ -83,7 +83,6 @@ azureListScaleSetNetwork <- function(azureActiveContext, resourceGroup, location lbs <- df$value$name dfn <- data.frame(lbname = "", publicIpAdress = "", inport = "", outport = "") clust <- length(lbs) - #browser() if (clust > 0) { for (i in seq_along(lbs)) { lb <- lbs[i] @@ -142,7 +141,6 @@ azureListScaleSetNetwork <- function(azureActiveContext, resourceGroup, location clust <- nrow(dfn) clust2 <- length(df2$properties$ipAddress) if (clust2 > 0) { - #if (length(df2$properties$dnsSettings$fqdn) != length(df2$properties$ipAddress)) browser() data.frame( fqdn = if (is.null(df2$properties$dnsSettings$fqdn)) "" else df2$properties$dnsSettings$fqdn, ipAddress = df2$properties$ipAddress, diff --git a/R/internal.R b/R/internal.R index 82b11a9..737f679 100644 --- a/R/internal.R +++ b/R/internal.R @@ -133,9 +133,7 @@ getSig <- function(azureActiveContext, url, verb, key, storageAccount, stopWithAzureError <- function(r) { - #if (status_code(r) %in% c(200, 201, 202, 204)) return() - #browser() - if(status_code(r) < 300) return() + if (status_code(r) < 300) return() msg <- paste0(as.character(sys.call(1))[1], "()") # Name of calling fucntion addToMsg <- function(x) { if (!is.null(x)) x <- strwrap(x) diff --git a/R/pollStatus.R b/R/pollStatus.R index 5eb8994..b83390c 100644 --- a/R/pollStatus.R +++ b/R/pollStatus.R @@ -88,7 +88,7 @@ pollStatusVM <- function(azureActiveContext) { pollStatusHDI <- function(azureActiveContext, clustername) { message("HDI request submitted: ", Sys.time()) - message("Key: (.) - in progress, S - succeeded, E - error") + message("Key: (.) - in progress, S - succeeded, E - error, F - failed") iteration <- 0 waiting <- TRUE while (iteration < 500 && waiting) { @@ -100,26 +100,68 @@ pollStatusHDI <- function(azureActiveContext, clustername) { rc <- switch(tolower(summary), succeeded = "S", error = "E", + failed = "F", inprogress = ".", "?" ) message(rc, appendLF = FALSE) - if (rc %in% c("S", "E")) { + if (rc %in% c("S", "E", "F")) { waiting = FALSE } - if (rc == "E") { - warning(paste("Error deploying: ", Sys.time())) - warning(fromJSON(status$properties$error$details$message)$error$message) + if (rc %in% c("E", "F")) { + message("") + warning(paste("Error deploying: ", Sys.time()), call. = FALSE, immediate. = TRUE) return(FALSE) } - if (rc == "?") message(status) + if (rc == "?") message(summary) iteration <- iteration + 1 - if (!rc %in% c("S", "E")) Sys.sleep(10) + if (!rc %in% c("S", "E", "F")) Sys.sleep(10) } message("") message("HDI request completed: ", Sys.time()) return(TRUE) } + + +pollStatusScriptAction <- function(azureActiveContext, scriptname) { + + message("Script action request submitted: ", Sys.time()) + message("Key: A - accepted, (.) - in progress, S - succeeded, E - error, F - failed") + iteration <- 0 + waiting <- TRUE + while (iteration < 500 && waiting) { + + status <- azureScriptActionHistory(azureActiveContext) + idx <- which(sapply(status, "[[", "name") == scriptname)[1] + summary <- status[[idx]]$status + rc <- switch(tolower(summary), + accepted = "A", + succeeded = "S", + error = "E", + failed = "F", + inprogress = ".", + "?" + ) + + message(rc, appendLF = FALSE) + if (rc %in% c("S", "E", "F")) { + waiting = FALSE + } + if (rc %in% c("E", "F")) { + message("") + warning(paste("Error deploying: ", Sys.time()), call. = FALSE, immediate. = TRUE) + return(FALSE) + } + + if (rc == "?") message(summary) + + iteration <- iteration + 1 + if (!rc %in% c("S", "E", "F")) Sys.sleep(5) + } + message("") + message("Script action completed: ", Sys.time()) + return(TRUE) +} diff --git a/man/AzureSMR.Rd b/man/AzureSMR.Rd index 21e8a28..8e4a5f9 100644 --- a/man/AzureSMR.Rd +++ b/man/AzureSMR.Rd @@ -31,14 +31,21 @@ This enables you to use and change many Azure resources. The following is an inc } \item Virtual Machines \itemize{ +\item List VMs: \code{\link[=azureListVM]{azureListVM()}} \item To create a virtual machine, use \code{\link[=azureDeployTemplate]{azureDeployTemplate()}} with a suitable template \item Start a VM: \code{\link[=azureStartVM]{azureStartVM()}} \item Stop a VM: \code{\link[=azureStopVM]{azureStopVM()}} \item Get status: \code{\link[=azureVMStatus]{azureVMStatus()}} -\item List VMs: \code{\link[=azureListVM]{azureListVM()}} } -\item HDInsight (Nodes, Hive, Spark) -\item Azure Resource Manager +\item HDInsight clusters: +\itemize{ +\item \code{\link[=azureListHDI]{azureListHDI()}} +\item \code{\link[=azureCreateHDI]{azureCreateHDI()}} +\item \code{\link[=azureResizeHDI]{azureResizeHDI()}} +\item \code{\link[=azureDeleteHDI]{azureDeleteHDI()}} +\item \code{\link[=azureRunScriptAction]{azureRunScriptAction()}} +\item \code{\link[=azureScriptActionHistory]{azureScriptActionHistory()}} +} } } \keyword{package} diff --git a/man/azureBlobCD.Rd b/man/azureBlobCD.Rd index 93b42ed..a049310 100644 --- a/man/azureBlobCD.Rd +++ b/man/azureBlobCD.Rd @@ -18,7 +18,7 @@ azureBlobCD(azureActiveContext, directory, container, file, storageAccount, \item Local filename to store in Azure blob }} -\item{storageAccount}{Name of the azure storage account} +\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.} \item{storageKey}{Storage key associated with storage account} diff --git a/man/azureBlobFind.Rd b/man/azureBlobFind.Rd index f7814a7..58a6766 100644 --- a/man/azureBlobFind.Rd +++ b/man/azureBlobFind.Rd @@ -14,7 +14,7 @@ azureBlobFind(azureActiveContext, file, storageAccount, storageKey, container, \item Local filename to store in Azure blob }} -\item{storageAccount}{Name of the azure storage account} +\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.} \item{storageKey}{Storage key associated with storage account} diff --git a/man/azureBlobLS.Rd b/man/azureBlobLS.Rd index 86525c2..4043bd1 100644 --- a/man/azureBlobLS.Rd +++ b/man/azureBlobLS.Rd @@ -14,7 +14,7 @@ azureBlobLS(azureActiveContext, directory, recursive = FALSE, storageAccount, \item{recursive}{If TRUE, list blob store directories recursively} -\item{storageAccount}{Name of the azure storage account} +\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.} \item{storageKey}{Storage key associated with storage account} diff --git a/man/azureCreateHDI.Rd b/man/azureCreateHDI.Rd index e1494f8..dde1fd1 100644 --- a/man/azureCreateHDI.Rd +++ b/man/azureCreateHDI.Rd @@ -2,24 +2,27 @@ % Please edit documentation in R/AzureHDI.R \name{azureCreateHDI} \alias{azureCreateHDI} -\title{Create Specifed HDInsight Cluster.} +\title{Create HDInsight cluster.} \usage{ -azureCreateHDI(azureActiveContext, clustername, location, kind = "spark", - storageAccount, storageKey, version = "3.4", componentVersion = "1.6.2", - workers = 2, adminUser, adminPassword, sshUser, sshPassword, hiveServer, - hiveDB, hiveUser, hivePassword, resourceGroup, vmSize = "Large", - subscriptionID, mode = "Sync", verbose = FALSE) +azureCreateHDI(azureActiveContext, resourceGroup, location, clustername, + kind = c("rserver", "spark", "hadoop"), storageAccount, storageKey, + version = "3.5", componentVersion = "1.6.2", workers = 2, adminUser, + adminPassword, sshUser, sshPassword, hiveServer, hiveDB, hiveUser, + hivePassword, vmSize = "Large", subscriptionID, mode = c("Sync", "Async"), + verbose = FALSE, debug = FALSE) } \arguments{ \item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} -\item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} +\item{resourceGroup}{Name of the resource group} \item{location}{Azure region, e.g. 'westeurope' or 'southcentralus'} +\item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} + \item{kind}{HDinsight kind: "hadoop","spark" or "rserver"} -\item{storageAccount}{Name of the azure storage account} +\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.} \item{storageKey}{Storage key associated with storage account} @@ -45,8 +48,6 @@ azureCreateHDI(azureActiveContext, clustername, location, kind = "spark", \item{hivePassword}{Hive user password} -\item{resourceGroup}{Name of the resource group} - \item{vmSize}{Size of nodes: "Large", "Small", "Standard_D14_V2", etc.} \item{subscriptionID}{Subscription ID. This is obtained automatically by \code{\link[=azureAuthenticate]{azureAuthenticate()}} when only a single subscriptionID is available via Active Directory} @@ -54,16 +55,21 @@ azureCreateHDI(azureActiveContext, clustername, location, kind = "spark", \item{mode}{Provisioning mode, "Sync" or "Async". Use "Async" to immediately return to R session after submission of request} \item{verbose}{Print Tracing information (Default False)} + +\item{debug}{Used for debugging purposes. If TRUE, returns json without attempting to connect to Azure} } \value{ Success message } \description{ -Create Specifed HDInsight Cluster. +Create HDInsight cluster. } \note{ See \url{https://docs.microsoft.com/en-us/azure/hdinsight/hdinsight-component-versioning} to learn about HDInsight Versions } +\references{ +https://docs.microsoft.com/en-us/rest/api/hdinsight/hdinsight-cluster#create +} \seealso{ Other HDInsight functions: \code{\link{azureDeleteHDI}}, \code{\link{azureHDIConf}}, \code{\link{azureListHDI}}, diff --git a/man/azureCreateStorageAccount.Rd b/man/azureCreateStorageAccount.Rd index 63dafbe..7a59191 100644 --- a/man/azureCreateStorageAccount.Rd +++ b/man/azureCreateStorageAccount.Rd @@ -5,12 +5,13 @@ \title{Create an Azure Storage Account.} \usage{ azureCreateStorageAccount(azureActiveContext, storageAccount, - location = "northeurope", resourceGroup, subscriptionID, verbose = FALSE) + location = "northeurope", resourceGroup, subscriptionID, + asynchronous = FALSE, verbose = FALSE) } \arguments{ \item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} -\item{storageAccount}{Name of the azure storage account} +\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.} \item{location}{A string for the location to create storage account} @@ -18,6 +19,8 @@ azureCreateStorageAccount(azureActiveContext, storageAccount, \item{subscriptionID}{Subscription ID. This is obtained automatically by \code{\link[=azureAuthenticate]{azureAuthenticate()}} when only a single subscriptionID is available via Active Directory} +\item{asynchronous}{If TRUE, submits asynchronous request to Azure. Otherwise waits until storage account is created.} + \item{verbose}{Print Tracing information (Default False)} } \description{ diff --git a/man/azureCreateStorageContainer.Rd b/man/azureCreateStorageContainer.Rd index c2b6da8..c92cfe7 100644 --- a/man/azureCreateStorageContainer.Rd +++ b/man/azureCreateStorageContainer.Rd @@ -12,7 +12,7 @@ azureCreateStorageContainer(azureActiveContext, container, storageAccount, \item{container}{Storage container name. See \code{\link[=azureListStorageContainers]{azureListStorageContainers()}}} -\item{storageAccount}{Name of the azure storage account} +\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.} \item{storageKey}{Storage key associated with storage account} diff --git a/man/azureDeleteBlob.Rd b/man/azureDeleteBlob.Rd index 8043750..dde2dde 100644 --- a/man/azureDeleteBlob.Rd +++ b/man/azureDeleteBlob.Rd @@ -14,7 +14,7 @@ azureDeleteBlob(azureActiveContext, blob, directory, storageAccount, storageKey, \item{directory}{Blob store directory to list for content} -\item{storageAccount}{Name of the azure storage account} +\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.} \item{storageKey}{Storage key associated with storage account} diff --git a/man/azureDeleteHDI.Rd b/man/azureDeleteHDI.Rd index 7723fcd..0130994 100644 --- a/man/azureDeleteHDI.Rd +++ b/man/azureDeleteHDI.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/AzureHDI.R \name{azureDeleteHDI} \alias{azureDeleteHDI} -\title{Delete Specifed HDInsight Cluster.} +\title{Delete HDInsight cluster.} \usage{ azureDeleteHDI(azureActiveContext, clustername, subscriptionID, resourceGroup, verbose = FALSE) @@ -19,10 +19,13 @@ azureDeleteHDI(azureActiveContext, clustername, subscriptionID, resourceGroup, \item{verbose}{Print Tracing information (Default False)} } \value{ -Returns Dataframe of HDInsight Clusters information +Data frame with HDInsight clusters information } \description{ -Delete Specifed HDInsight Cluster. +Delete HDInsight cluster. +} +\references{ +https://docs.microsoft.com/en-us/rest/api/hdinsight/hdinsight-cluster#delete } \seealso{ Other HDInsight functions: \code{\link{azureCreateHDI}}, diff --git a/man/azureDeleteStorageContainer.Rd b/man/azureDeleteStorageContainer.Rd index 0cc4288..b546b70 100644 --- a/man/azureDeleteStorageContainer.Rd +++ b/man/azureDeleteStorageContainer.Rd @@ -12,7 +12,7 @@ azureDeleteStorageContainer(azureActiveContext, container, storageAccount, \item{container}{Storage container name. See \code{\link[=azureListStorageContainers]{azureListStorageContainers()}}} -\item{storageAccount}{Name of the azure storage account} +\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.} \item{storageKey}{Storage key associated with storage account} diff --git a/man/azureDeletestorageAccount.Rd b/man/azureDeletestorageAccount.Rd index cc62428..6fd43ee 100644 --- a/man/azureDeletestorageAccount.Rd +++ b/man/azureDeletestorageAccount.Rd @@ -10,7 +10,7 @@ azureDeletestorageAccount(azureActiveContext, storageAccount, resourceGroup, \arguments{ \item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} -\item{storageAccount}{Name of the azure storage account} +\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.} \item{resourceGroup}{Name of the resource group} diff --git a/man/azureDeployTemplate.Rd b/man/azureDeployTemplate.Rd index 8aa52a9..9b4a5ee 100644 --- a/man/azureDeployTemplate.Rd +++ b/man/azureDeployTemplate.Rd @@ -5,8 +5,8 @@ \title{Deploy a resource from an Azure Resource Manager (ARM) template.} \usage{ azureDeployTemplate(azureActiveContext, deplname, templateURL, paramURL, - templateJSON, paramJSON, mode = "Sync", resourceGroup, subscriptionID, - verbose = FALSE) + templateJSON, paramJSON, mode = c("Sync", "Async"), resourceGroup, + subscriptionID, verbose = FALSE) } \arguments{ \item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} diff --git a/man/azureGetBlob.Rd b/man/azureGetBlob.Rd index e745398..e1bf690 100644 --- a/man/azureGetBlob.Rd +++ b/man/azureGetBlob.Rd @@ -16,7 +16,7 @@ azureGetBlob(azureActiveContext, blob, directory, type = "text", \item{type}{String, either "text" or "raw". Passed to \code{\link[httr:content]{httr::content()}}} -\item{storageAccount}{Name of the azure storage account} +\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.} \item{storageKey}{Storage key associated with storage account} diff --git a/man/azureHDIConf.Rd b/man/azureHDIConf.Rd index cf721ef..e20a2d2 100644 --- a/man/azureHDIConf.Rd +++ b/man/azureHDIConf.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/AzureHDI.R \name{azureHDIConf} \alias{azureHDIConf} -\title{Get Configuration Information for a specified cluster name.} +\title{Get configuration information for a specified cluster name.} \usage{ azureHDIConf(azureActiveContext, clustername, resourceGroup, subscriptionID, name, type, location, verbose = FALSE) @@ -28,7 +28,7 @@ azureHDIConf(azureActiveContext, clustername, resourceGroup, subscriptionID, Returns Dataframe of HDInsight Clusters information } \description{ -Get Configuration Information for a specified cluster name. +Get configuration information for a specified cluster name. } \seealso{ Other HDInsight functions: \code{\link{azureCreateHDI}}, diff --git a/man/azureHiveSQL.Rd b/man/azureHiveSQL.Rd index e2df73c..37760cd 100644 --- a/man/azureHiveSQL.Rd +++ b/man/azureHiveSQL.Rd @@ -14,9 +14,9 @@ azureHiveSQL(azureActiveContext, CMD, clustername, hdiAdmin, hdiPassword, \item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} -\item{hdiAdmin}{HDInsight admin username} +\item{hdiAdmin}{HDInsight admin username. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} -\item{hdiPassword}{HDInsight admin password} +\item{hdiPassword}{HDInsight admin password. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} \item{path}{path} diff --git a/man/azureHiveStatus.Rd b/man/azureHiveStatus.Rd index abae439..3ef46b6 100644 --- a/man/azureHiveStatus.Rd +++ b/man/azureHiveStatus.Rd @@ -12,9 +12,9 @@ azureHiveStatus(azureActiveContext, clustername, hdiAdmin, hdiPassword, \item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} -\item{hdiAdmin}{HDInsight admin username} +\item{hdiAdmin}{HDInsight admin username. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} -\item{hdiPassword}{HDInsight admin password} +\item{hdiPassword}{HDInsight admin password. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} \item{verbose}{Print Tracing information (Default False)} } diff --git a/man/azureListHDI.Rd b/man/azureListHDI.Rd index 60c9bbb..5ed9b4b 100644 --- a/man/azureListHDI.Rd +++ b/man/azureListHDI.Rd @@ -25,11 +25,14 @@ azureListHDI(azureActiveContext, resourceGroup, clustername = "*", \item{verbose}{Print Tracing information (Default False)} } \value{ -Returns Dataframe of HDInsight Clusters +data frame with summary information of HDI clusters } \description{ Get all HDInsight Clusters in default Subscription or details for a specified cluster name. } +\references{ +https://docs.microsoft.com/en-us/rest/api/hdinsight/hdinsight-cluster#list-by-subscription +} \seealso{ Other HDInsight functions: \code{\link{azureCreateHDI}}, \code{\link{azureDeleteHDI}}, \code{\link{azureHDIConf}}, diff --git a/man/azureListStorageBlobs.Rd b/man/azureListStorageBlobs.Rd index 132be19..620ab68 100644 --- a/man/azureListStorageBlobs.Rd +++ b/man/azureListStorageBlobs.Rd @@ -10,7 +10,7 @@ azureListStorageBlobs(azureActiveContext, storageAccount, storageKey, container, \arguments{ \item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} -\item{storageAccount}{Name of the azure storage account} +\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.} \item{storageKey}{Storage key associated with storage account} diff --git a/man/azureListStorageContainers.Rd b/man/azureListStorageContainers.Rd index 82c20d3..6d7bfc6 100644 --- a/man/azureListStorageContainers.Rd +++ b/man/azureListStorageContainers.Rd @@ -10,7 +10,7 @@ azureListStorageContainers(azureActiveContext, storageAccount, storageKey, \arguments{ \item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} -\item{storageAccount}{Name of the azure storage account} +\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.} \item{storageKey}{Storage key associated with storage account} diff --git a/man/azureListSubscriptions.Rd b/man/azureListSubscriptions.Rd index 22de00a..9a1c587 100644 --- a/man/azureListSubscriptions.Rd +++ b/man/azureListSubscriptions.Rd @@ -12,7 +12,7 @@ azureListSubscriptions(azureActiveContext, verbose = FALSE) \item{verbose}{Print Tracing information (Default False)} } \value{ -Dataframe of subscriptionID; Sets AzureContext subscriptionID +data frame with subscriptionID; Sets AzureContext subscriptionID } \description{ Get available subscriptions. diff --git a/man/azurePutBlob.Rd b/man/azurePutBlob.Rd index 4a6bce1..91d61d3 100644 --- a/man/azurePutBlob.Rd +++ b/man/azurePutBlob.Rd @@ -22,7 +22,7 @@ azurePutBlob(azureActiveContext, blob, contents = "", file = "", directory, \item{directory}{Blob store directory to list for content} -\item{storageAccount}{Name of the azure storage account} +\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.} \item{storageKey}{Storage key associated with storage account} diff --git a/man/azureResizeHDI.Rd b/man/azureResizeHDI.Rd index 67f7237..bbff30d 100644 --- a/man/azureResizeHDI.Rd +++ b/man/azureResizeHDI.Rd @@ -2,19 +2,20 @@ % Please edit documentation in R/AzureHDI.R \name{azureResizeHDI} \alias{azureResizeHDI} -\title{Resize a HDInsight Cluster role.} +\title{Resize a HDInsight cluster role.} \usage{ -azureResizeHDI(azureActiveContext, clustername, role = "worker", size = 2, - mode = "Sync", subscriptionID, resourceGroup, verbose = FALSE) +azureResizeHDI(azureActiveContext, clustername, role = c("worker", "head", + "edge"), size = 2, mode = c("Sync", "Async"), subscriptionID, + resourceGroup, verbose = FALSE) } \arguments{ \item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} \item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} -\item{role}{role type: 'worker', 'head' or 'Edge'} +\item{role}{role type: 'worker', 'head' or 'edge'} -\item{size}{Desired size of role type} +\item{size}{Numeric: the number of nodes for this type of role} \item{mode}{Provisioning mode, "Sync" or "Async". Use "Async" to immediately return to R session after submission of request} @@ -25,7 +26,7 @@ azureResizeHDI(azureActiveContext, clustername, role = "worker", size = 2, \item{verbose}{Print Tracing information (Default False)} } \description{ -Resize a HDInsight Cluster role. +Resize a HDInsight cluster role. } \seealso{ Other HDInsight functions: \code{\link{azureCreateHDI}}, diff --git a/man/azureRunScriptAction.Rd b/man/azureRunScriptAction.Rd index 4c3a588..8d62722 100644 --- a/man/azureRunScriptAction.Rd +++ b/man/azureRunScriptAction.Rd @@ -2,24 +2,25 @@ % Please edit documentation in R/AzureHDI.R \name{azureRunScriptAction} \alias{azureRunScriptAction} -\title{Run Script Action on HDI Cluster.} +\title{Run script action on HDI cluster.} \usage{ -azureRunScriptAction(azureActiveContext, scriptname = "script1", scriptURL, +azureRunScriptAction(azureActiveContext, scriptname, scriptURL, headNode = TRUE, workerNode = FALSE, edgeNode = FALSE, clustername, - resourceGroup, parameters = "", subscriptionID, verbose = FALSE) + resourceGroup, parameters = "", subscriptionID, wait = TRUE, + verbose = FALSE) } \arguments{ \item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} -\item{scriptname}{Identifier for Custom action scrript operation} +\item{scriptname}{Identifier for Custom action script operation} -\item{scriptURL}{URL to custom action script (Sring)} +\item{scriptURL}{URL to custom action script} -\item{headNode}{install on head nodes (default FALSE)} +\item{headNode}{install on head nodes} -\item{workerNode}{install on worker nodes (default FALSE)} +\item{workerNode}{install on worker nodes} -\item{edgeNode}{install on worker nodes (default FALSE)} +\item{edgeNode}{install on worker nodes} \item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} @@ -29,13 +30,18 @@ azureRunScriptAction(azureActiveContext, scriptname = "script1", scriptURL, \item{subscriptionID}{Subscription ID. This is obtained automatically by \code{\link[=azureAuthenticate]{azureAuthenticate()}} when only a single subscriptionID is available via Active Directory} +\item{wait}{If TRUE, runs script action synchronously, i.e. waits for successfull completion. If FALSE, submits the action asynchronously} + \item{verbose}{Print Tracing information (Default False)} } \value{ Returns Success Message } \description{ -Run Script Action on HDI Cluster. +Run script action on HDI cluster. +} +\references{ +https://docs.microsoft.com/en-us/rest/api/hdinsight/hdinsight-cluster#run-script-actions-on-a-running-cluster-linux-cluster-only } \seealso{ Other HDInsight functions: \code{\link{azureCreateHDI}}, diff --git a/man/azureSAGetKey.Rd b/man/azureSAGetKey.Rd index d3dbf74..fca79e0 100644 --- a/man/azureSAGetKey.Rd +++ b/man/azureSAGetKey.Rd @@ -10,7 +10,7 @@ azureSAGetKey(azureActiveContext, storageAccount, resourceGroup, subscriptionID, \arguments{ \item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} -\item{storageAccount}{storageAccount} +\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.} \item{resourceGroup}{Name of the resource group} diff --git a/man/azureScriptActionHistory.Rd b/man/azureScriptActionHistory.Rd index 8917365..50eb15e 100644 --- a/man/azureScriptActionHistory.Rd +++ b/man/azureScriptActionHistory.Rd @@ -2,10 +2,13 @@ % Please edit documentation in R/AzureHDI.R \name{azureScriptActionHistory} \alias{azureScriptActionHistory} -\title{Get all HDInsight Script Action History for a specified cluster name.} +\alias{summary.azureScriptActionHistory} +\title{Get all HDInsight script action history for a specified cluster name.} \usage{ azureScriptActionHistory(azureActiveContext, resourceGroup, clustername = "*", subscriptionID, name, type, verbose = FALSE) + +\method{summary}{azureScriptActionHistory}(object, ...) } \arguments{ \item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} @@ -21,12 +24,19 @@ azureScriptActionHistory(azureActiveContext, resourceGroup, clustername = "*", \item{type}{filter by resource type} \item{verbose}{Print Tracing information (Default False)} + +\item{object}{azureScriptActionHistory object, created by \code{\link[=azureScriptActionHistory]{azureScriptActionHistory()}}} + +\item{...}{not used} } \value{ Dataframe of HDInsight Clusters } \description{ -Get all HDInsight Script Action History for a specified cluster name. +Get all HDInsight script action history for a specified cluster name. +} +\references{ +https://docs.microsoft.com/en-us/rest/api/hdinsight/hdinsight-cluster#list-all-persisted-script-actions-for-a-cluster-linux-cluster-only } \seealso{ Other HDInsight functions: \code{\link{azureCreateHDI}}, diff --git a/man/azureSparkCMD.Rd b/man/azureSparkCMD.Rd index f6748ad..e22f754 100644 --- a/man/azureSparkCMD.Rd +++ b/man/azureSparkCMD.Rd @@ -14,9 +14,9 @@ azureSparkCMD(azureActiveContext, CMD, clustername, hdiAdmin, hdiPassword, \item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} -\item{hdiAdmin}{HDInsight admin username} +\item{hdiAdmin}{HDInsight admin username. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} -\item{hdiPassword}{HDInsight admin password} +\item{hdiPassword}{HDInsight admin password. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} \item{sessionID}{Spark sessionID. See \code{\link[=azureSparkCMD]{azureSparkCMD()}}} diff --git a/man/azureSparkJob.Rd b/man/azureSparkJob.Rd index 17233a9..1da35c2 100644 --- a/man/azureSparkJob.Rd +++ b/man/azureSparkJob.Rd @@ -14,9 +14,9 @@ azureSparkJob(azureActiveContext, FILE, clustername, hdiAdmin, hdiPassword, \item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} -\item{hdiAdmin}{HDInsight admin username} +\item{hdiAdmin}{HDInsight admin username. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} -\item{hdiPassword}{HDInsight admin password} +\item{hdiPassword}{HDInsight admin password. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} \item{log}{log} diff --git a/man/azureSparkListJobs.Rd b/man/azureSparkListJobs.Rd index d07dc45..7af149a 100644 --- a/man/azureSparkListJobs.Rd +++ b/man/azureSparkListJobs.Rd @@ -12,9 +12,9 @@ azureSparkListJobs(azureActiveContext, clustername, hdiAdmin, hdiPassword, \item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} -\item{hdiAdmin}{HDInsight admin username} +\item{hdiAdmin}{HDInsight admin username. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} -\item{hdiPassword}{HDInsight admin password} +\item{hdiPassword}{HDInsight admin password. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} \item{verbose}{Print Tracing information (Default False)} } diff --git a/man/azureSparkListSessions.Rd b/man/azureSparkListSessions.Rd index 11b4ecd..3089519 100644 --- a/man/azureSparkListSessions.Rd +++ b/man/azureSparkListSessions.Rd @@ -12,9 +12,9 @@ azureSparkListSessions(azureActiveContext, clustername, hdiAdmin, hdiPassword, \item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} -\item{hdiAdmin}{HDInsight admin username} +\item{hdiAdmin}{HDInsight admin username. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} -\item{hdiPassword}{HDInsight admin password} +\item{hdiPassword}{HDInsight admin password. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} \item{verbose}{Print Tracing information (Default False)} } diff --git a/man/azureSparkNewSession.Rd b/man/azureSparkNewSession.Rd index 77b7214..4ff5096 100644 --- a/man/azureSparkNewSession.Rd +++ b/man/azureSparkNewSession.Rd @@ -12,11 +12,11 @@ azureSparkNewSession(azureActiveContext, clustername, hdiAdmin, hdiPassword, \item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} -\item{hdiAdmin}{HDInsight admin username} +\item{hdiAdmin}{HDInsight admin username. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} -\item{hdiPassword}{HDInsight admin password} +\item{hdiPassword}{HDInsight admin password. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} -\item{kind}{HDinsight kind: "hadoop","spark" or "pyspark"} +\item{kind}{HDinsight kind: "hadoop","spark" or "rserver". See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} \item{verbose}{Print Tracing information (Default False)} } diff --git a/man/azureSparkStopSession.Rd b/man/azureSparkStopSession.Rd index ed2890e..2496ed6 100644 --- a/man/azureSparkStopSession.Rd +++ b/man/azureSparkStopSession.Rd @@ -12,9 +12,9 @@ azureSparkStopSession(azureActiveContext, clustername, hdiAdmin, hdiPassword, \item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} -\item{hdiAdmin}{HDInsight admin username} +\item{hdiAdmin}{HDInsight admin username. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} -\item{hdiPassword}{HDInsight admin password} +\item{hdiPassword}{HDInsight admin password. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} \item{sessionID}{Spark sessionID. See \code{\link[=azureSparkCMD]{azureSparkCMD()}}} diff --git a/man/setAzureContext.Rd b/man/setAzureContext.Rd index 26f645d..dbf933f 100644 --- a/man/setAzureContext.Rd +++ b/man/setAzureContext.Rd @@ -4,9 +4,9 @@ \alias{setAzureContext} \title{Updates azureActiveContext object.} \usage{ -setAzureContext(azureActiveContext, tenantID, clientID, authKey, azToken, - subscriptionID, resourceGroup, storageKey, storageAccount, container, blob, - vmName, hdiAdmin, hdiPassword, clustername, kind, sessionID) +setAzureContext(azureActiveContext, tenantID, clientID, authKey, subscriptionID, + resourceGroup, storageKey, storageAccount, container, blob, vmName, hdiAdmin, + hdiPassword, clustername, kind, sessionID) } \arguments{ \item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}} @@ -17,15 +17,13 @@ setAzureContext(azureActiveContext, tenantID, clientID, authKey, azToken, \item{authKey}{The Authentication Key provided during creation of the Active Directory application / service principal} -\item{azToken}{Azure authentication token, obtained by \code{\link[=azureAuthenticate]{azureAuthenticate()}}} - \item{subscriptionID}{Subscription ID. This is obtained automatically by \code{\link[=azureAuthenticate]{azureAuthenticate()}} when only a single subscriptionID is available via Active Directory} \item{resourceGroup}{Name of the resource group} \item{storageKey}{Storage key associated with storage account} -\item{storageAccount}{Name of the azure storage account} +\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.} \item{container}{Storage container name. See \code{\link[=azureListStorageContainers]{azureListStorageContainers()}}} @@ -33,13 +31,13 @@ setAzureContext(azureActiveContext, tenantID, clientID, authKey, azToken, \item{vmName}{Name of the virtual machine} -\item{hdiAdmin}{HDInsight admin username} +\item{hdiAdmin}{HDInsight admin username. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} -\item{hdiPassword}{HDInsight admin password} +\item{hdiPassword}{HDInsight admin password. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} \item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} -\item{kind}{HDinsight kind: "hadoop","spark" or "pyspark"} +\item{kind}{HDinsight kind: "hadoop","spark" or "rserver". See \code{\link[=azureCreateHDI]{azureCreateHDI()}}} \item{sessionID}{Spark sessionID. See \code{\link[=azureSparkCMD]{azureSparkCMD()}}} } diff --git a/tests/testthat/test-5-HDI.R b/tests/testthat/test-5-HDI.R index 25ef456..9bd9271 100644 --- a/tests/testthat/test-5-HDI.R +++ b/tests/testthat/test-5-HDI.R @@ -17,6 +17,8 @@ azureAuthenticate(asc, verbose = FALSE) timestamp <- format(Sys.time(), format = "%y%m%d%H%M") resourceGroup_name <- paste0("_AzureSMtest_", timestamp) +cluster_name <- paste0("azuresmrhdi", timestamp) +storage_name <- paste0("azuresmrhdist", timestamp) test_that("Can create resource group", { skip_if_missing_config(settingsfile) @@ -46,58 +48,63 @@ test_that("Can create HDI cluster", { "storageAccount" ) expect_message( - azureCreateHDI(asc, resourceGroup = resourceGroup_name, clustername = "azuresmr_hdi_test", - storageAccount = "azuresmrhditest", - adminUser = "Azuresmr_test1", adminPassword = "Password_1", - sshUser = "sssUser_test1", sshPassword = "Password_1", - debug = TRUE + azureCreateHDI(asc, resourceGroup = resourceGroup_name, + clustername = cluster_name, + storageAccount = storage_name, + adminUser = "Azuresmr_test1", adminPassword = "Password_1", + sshUser = "sssUser_test1", sshPassword = "Password_1", + debug = TRUE ), "creating storage account" ) expect_error( - azureCreateHDI(asc, resourceGroup = resourceGroup_name, clustername = "azuresmr_hdi_test", - storageAccount = "azuresmrhditest", - adminUser = "Azuresmr_test1", adminPassword = "Azuresmr_test1", - sshUser = "sssUser_test1", sshPassword = "sssUser_test1", - debug = FALSE + azureCreateHDI(asc, resourceGroup = resourceGroup_name, + clustername = cluster_name, + storageAccount = storage_name, + adminUser = "Azuresmr_test1", adminPassword = "Azuresmr_test1", + sshUser = "sssUser_test1", sshPassword = "sssUser_test1", + debug = FALSE ), "should not contain 3 consecutive letters from the username" ) # debug - default expect_is( - azureCreateHDI(asc, resourceGroup = resourceGroup_name, clustername = "azuresmr_hdi_test", - storageAccount = "azuresmrhditest", + azureCreateHDI(asc, resourceGroup = resourceGroup_name, + clustername = cluster_name, + storageAccount = storage_name, adminUser = "x", adminPassword = "Azuresmr_test1", sshUser = "sssUser_test1", sshPassword = "sshUser_test1", debug = TRUE - ), - "list" + ), + "list" ) # debug - rserver expect_is( - azureCreateHDI(asc, resourceGroup = resourceGroup_name, clustername = "azuresmr_hdi_test", - storageAccount = "azuresmrhditest", + azureCreateHDI(asc, resourceGroup = resourceGroup_name, + clustername = cluster_name, + storageAccount = storage_name, adminUser = "x", adminPassword = "Azuresmr_test1", sshUser = "sssUser_test1", sshPassword = "sshUser_test1", kind = "rserver", debug = TRUE - ), - "list" + ), + "list" ) # debug - hadoop expect_is( - azureCreateHDI(asc, resourceGroup = resourceGroup_name, clustername = "azuresmr_hdi_test", - storageAccount = "azuresmrhditest", + azureCreateHDI(asc, resourceGroup = resourceGroup_name, + clustername = cluster_name, + storageAccount = storage_name, adminUser = "x", adminPassword = "Azuresmr_test1", sshUser = "sssUser_test1", sshPassword = "sshUser_test1", kind = "hadoop", debug = TRUE - ), - "list" + ), + "list" ) }) @@ -109,14 +116,15 @@ test_that("can create HDI cluster", { # create the actual instance - rserver expect_true( - azureCreateHDI(asc, resourceGroup = resourceGroup_name, clustername = "azuresmrhditest", - storageAccount = paste0("azuresmrhdi", timestamp), - adminUser = "x", adminPassword = "Azuresmr_test1", - sshUser = "sssUser_test1", sshPassword = "sshUser_test1", - kind = "rserver", - debug = FALSE - ) + azureCreateHDI(asc, resourceGroup = resourceGroup_name, + clustername = cluster_name, + storageAccount = storage_name, + adminUser = "x", adminPassword = "Azuresmr_test1", + sshUser = "sssUser_test1", sshPassword = "sshUser_test1", + kind = "rserver", + debug = FALSE ) + ) }) @@ -128,15 +136,15 @@ test_that("can run action scripts", { # run an action script expect_true( azureRunScriptAction(asc, - scriptname = "installPackages", - scriptURL = "http://mrsactionscripts.blob.core.windows.net/rpackages-v01/InstallRPackages.sh", - workerNode = TRUE, edgeNode = TRUE, - parameters = "useCRAN stringr") + scriptname = "installPackages", + scriptURL = "http://mrsactionscripts.blob.core.windows.net/rpackages-v01/InstallRPackages.sh", + workerNode = TRUE, edgeNode = TRUE, + parameters = "useCRAN stringr") ) # retrieve action script history z <- azureScriptActionHistory(asc) - expect_is(z, "data.frame") + expect_is(z, "azureScriptActionHistory") }) # -------- @@ -148,9 +156,14 @@ test_that("can delete HDI cluster", { z <- azureListHDI(asc) expect_is(z, "data.frame") + z <- azureListHDI(asc, + clustername = cluster_name, + resourceGroup = resourceGroup_name) + expect_is(z, "data.frame") + # delete cluster expect_true( - azureDeleteHDI(asc, clustername = "azuresmrhditest") + azureDeleteHDI(asc, clustername = cluster_name) ) azureDeleteResourceGroup(asc, resourceGroup = resourceGroup_name)