Merge branch 'dev'
* dev: Bump to version 0.2.4 and update date Rebuild docs Add pollStatus function for script actions #62 Fix unit tests Run script actions asynchronously #62 Remove instances of browser() Rebuild docs Refactor and fix issues with HDI #62
This commit is contained in:
Andrie de Vries
Коммит
48b1d87459
|
|
@ -1,12 +1,12 @@
|
|||
Package: AzureSMR
|
||||
Title: Manage and Interact with Azure Resources.
|
||||
Title: Manage and Interact with Azure Resources
|
||||
Description: Helps users to manage Azure Services and objects from within an
|
||||
R Session. This includes Azure Storage (e.g. containers and blobs), Virtual
|
||||
Machines and HDInsight (Spark, Hive). To use the package, you must configure
|
||||
an Azure Active Directory application and service principal in the Azure portal.
|
||||
Type: Package
|
||||
Version: 0.2.3
|
||||
Date: 2017-05-21
|
||||
Version: 0.2.4
|
||||
Date: 2017-05-30
|
||||
Authors@R: c(
|
||||
person(family="Microsoft Corporation", role="cph"),
|
||||
person("Alan", "Weaver", role=c("aut", "cre"), email="alanwe@microsoft.com"),
|
||||
|
|
@ -22,7 +22,6 @@ Imports:
|
|||
httr,
|
||||
jsonlite,
|
||||
XML,
|
||||
plyr,
|
||||
base64enc,
|
||||
digest
|
||||
Depends:
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
S3method(print,azureActiveContext)
|
||||
S3method(str,azureActiveContext)
|
||||
S3method(summary,azureScriptActionHistory)
|
||||
export(AzureListRG)
|
||||
export(as.azureActiveContext)
|
||||
export(azureAuthenticate)
|
||||
|
|
@ -77,7 +78,6 @@ importFrom(httr,headers)
|
|||
importFrom(httr,http_status)
|
||||
importFrom(httr,status_code)
|
||||
importFrom(jsonlite,fromJSON)
|
||||
importFrom(plyr,rbind.fill)
|
||||
importFrom(utils,URLencode)
|
||||
importFrom(utils,browseURL)
|
||||
importFrom(utils,ls.str)
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ azureListStorageContainers <- function(azureActiveContext, storageAccount, stora
|
|||
verbosity <- set_verbosity(verbose)
|
||||
|
||||
URL <- paste0("http://", storageAccount, ".blob.core.windows.net/?comp=list")
|
||||
#browser()
|
||||
xdate <- x_ms_date()
|
||||
SIG <- getSig(azureActiveContext, url = URL, verb = "GET", key = storageKey, storageAccount = storageAccount,
|
||||
CMD = "\ncomp:list", date = xdate)
|
||||
|
|
|
|||
179
R/AzureHDI.R
179
R/AzureHDI.R
|
|
@ -5,8 +5,9 @@
|
|||
#' @inheritParams azureListAllResources
|
||||
#'
|
||||
#' @family HDInsight functions
|
||||
#' @references https://docs.microsoft.com/en-us/rest/api/hdinsight/hdinsight-cluster#list-by-subscription
|
||||
#'
|
||||
#' @return Returns Dataframe of HDInsight Clusters
|
||||
#' @return data frame with summary information of HDI clusters
|
||||
#' @export
|
||||
azureListHDI <- function(azureActiveContext, resourceGroup, clustername = "*",
|
||||
subscriptionID, name, type, location, verbose = FALSE) {
|
||||
|
|
@ -17,7 +18,7 @@ azureListHDI <- function(azureActiveContext, resourceGroup, clustername = "*",
|
|||
if (missing(subscriptionID)) subscriptionID <- azureActiveContext$subscriptionID
|
||||
if (missing(resourceGroup)) resourceGroup <- azureActiveContext$resourceGroup
|
||||
verbosity <- set_verbosity(verbose)
|
||||
|
||||
|
||||
assert_that(is_subscription_id(subscriptionID))
|
||||
if (clustername != "*") {
|
||||
assert_that(is_clustername(clustername))
|
||||
|
|
@ -26,33 +27,36 @@ azureListHDI <- function(azureActiveContext, resourceGroup, clustername = "*",
|
|||
|
||||
rg <- if (clustername == "*") "" else paste0("/resourceGroups/", resourceGroup)
|
||||
cn <- if (clustername == "*") "" else clustername
|
||||
|
||||
|
||||
URL <- paste0("https://management.azure.com/subscriptions/", subscriptionID, rg,
|
||||
"/providers/Microsoft.HDInsight/clusters/", cn,
|
||||
"/providers/Microsoft.HDInsight/clusters/", cn,
|
||||
"?api-version=2015-03-01-preview")
|
||||
|
||||
r <- GET(URL, azureApiHeaders(azToken), verbosity)
|
||||
stopWithAzureError(r)
|
||||
|
||||
rc <- content(r)$value
|
||||
z <- do.call(rbind, lapply(rc, function(x) {
|
||||
as.data.frame(
|
||||
rc <- content(r)
|
||||
extract_one <- function(x) {
|
||||
as.data.frame(
|
||||
c(
|
||||
x[c("name", "id", "location", "type")],
|
||||
x$properties[c("tier", "osType", "provisioningState", "clusterState", "createdDate")],
|
||||
x$properties$clusterDefinition[c("kind")]
|
||||
)
|
||||
)
|
||||
}))
|
||||
))
|
||||
}
|
||||
|
||||
z <- if (is.null(rc$value)) {
|
||||
extract_one(rc)
|
||||
} else {
|
||||
do.call(rbind, lapply(rc$value, extract_one))
|
||||
}
|
||||
|
||||
azureActiveContext$resourceGroup <- resourceGroup
|
||||
|
||||
return(z)
|
||||
}
|
||||
|
||||
|
||||
|
||||
#' Get Configuration Information for a specified cluster name.
|
||||
#' Get configuration information for a specified cluster name.
|
||||
#'
|
||||
#' @inheritParams setAzureContext
|
||||
#' @inheritParams azureAuthenticate
|
||||
|
|
@ -64,69 +68,57 @@ azureListHDI <- function(azureActiveContext, resourceGroup, clustername = "*",
|
|||
#' @export
|
||||
azureHDIConf <- function(azureActiveContext, clustername, resourceGroup,
|
||||
subscriptionID, name, type, location, verbose = FALSE) {
|
||||
assert_that(is.azureActiveContext(azureActiveContext))
|
||||
azureCheckToken(azureActiveContext)
|
||||
azToken <- azureActiveContext$Token
|
||||
if (missing(subscriptionID)) {
|
||||
subscriptionID <- azureActiveContext$subscriptionID
|
||||
} else (subscriptionID <- subscriptionID)
|
||||
if (missing(resourceGroup)) {
|
||||
resourceGroup <- azureActiveContext$resourceGroup
|
||||
} else (resourceGroup <- resourceGroup)
|
||||
if (missing(clustername)) {
|
||||
clustername <- azureActiveContext$clustername
|
||||
} else (clustername <- clustername)
|
||||
if (missing(subscriptionID)) subscriptionID <- azureActiveContext$subscriptionID
|
||||
if (missing(resourceGroup)) resourceGroup <- azureActiveContext$resourceGroup
|
||||
if (missing(clustername)) clustername <- azureActiveContext$clustername
|
||||
|
||||
verbosity <- set_verbosity(verbose)
|
||||
|
||||
assert_that(is_subscription_id(subscriptionID))
|
||||
assert_that(is_resource_group(resourceGroup))
|
||||
assert_that(is_clustername(clustername))
|
||||
|
||||
if (!length(azToken)) {
|
||||
stop("Error: No Token / Not currently Authenticated.")
|
||||
}
|
||||
if (!length(subscriptionID)) {
|
||||
stop("Error: No subscriptionID provided: Use SUBID argument or set in AzureContext")
|
||||
}
|
||||
if (!length(clustername)) {
|
||||
stop("Error: No clustername Provided.")
|
||||
URL <- paste0("https://management.azure.com/subscriptions/", subscriptionID,
|
||||
"/resourceGroups/", resourceGroup,
|
||||
"/providers/Microsoft.HDInsight/clusters/", clustername,
|
||||
"?api-version=2015-03-01-preview")
|
||||
|
||||
r <- GET(URL, azureApiHeaders(azToken), verbosity)
|
||||
browser()
|
||||
rc <- content(r)
|
||||
|
||||
if (length(rc) == 0) {
|
||||
warning("No HDInsight clusters found", immediate. = TRUE)
|
||||
}
|
||||
|
||||
URL <- paste("https://management.azure.com/subscriptions/", subscriptionID,
|
||||
"/resourceGroups/", resourceGroup, "/providers/Microsoft.HDInsight/clusters/",
|
||||
clustername, "?api-version=2015-03-01-preview", sep = "")
|
||||
info <- paste(vapply(rc$properties$computeProfile$roles, function(x) {
|
||||
sprintf("%s: %s * %s",
|
||||
x$name,
|
||||
x$targetInstanceCount,
|
||||
x$hardwareProfile$vmSize)}, FUN.VALUE = character(1)
|
||||
), collapse = ", "
|
||||
)
|
||||
|
||||
r <- GET(URL, add_headers(.headers = c(Host = "management.azure.com",
|
||||
Authorization = azToken, `Content-type` = "application/json")), verbosity)
|
||||
rl <- content(r, "text")
|
||||
dfn <- with(rc, data.frame(
|
||||
name = name,
|
||||
id = id,
|
||||
location = location,
|
||||
type = type,
|
||||
tier = rc$properties$tier,
|
||||
kind = rc$properties$clusterDefinition$kind,
|
||||
osType = rc$properties$osType,
|
||||
provisioningState = rc$properties$provisioningState,
|
||||
status = rc$properties$clusterState,
|
||||
created = rc$properties$createdDate,
|
||||
numCores = rc$properties$quotaInfo$coresUsed,
|
||||
information = info,
|
||||
stringsAsFactors = FALSE
|
||||
))
|
||||
|
||||
df <- fromJSON(rl)
|
||||
dfn <- as.data.frame(df$name)
|
||||
clust <- nrow(dfn)
|
||||
if (clust < 1) {
|
||||
warning("No HDInsight Clusters found")
|
||||
return(NULL)
|
||||
}
|
||||
|
||||
dfn[1, 1] <- df$name
|
||||
dfn[1, 2] <- df$id
|
||||
dfn[1, 3] <- df$location
|
||||
dfn[1, 4] <- df$type
|
||||
dfn[1, 5] <- df$properties$tier
|
||||
dfn[1, 6] <- df$properties$clusterDefinition$kind
|
||||
dfn[1, 7] <- df$properties$osType
|
||||
dfn[1, 8] <- df$properties$provisioningState
|
||||
dfn[1, 9] <- df$properties$clusterState
|
||||
dfn[1, 10] <- df$properties$createdDate
|
||||
dfn[1, 11] <- df$properties$quotaInfo$coresUsed
|
||||
roles1 <- df$properties$computeProfile$roles
|
||||
rt <- ""
|
||||
for (i in 1:nrow(roles1)) {
|
||||
row <- roles1[i, ]
|
||||
rt <- paste(rt, row$name, "(", row$targetInstanceCount, "*", row[,
|
||||
3], ")")
|
||||
}
|
||||
dfn[1, 12] <- rt
|
||||
colnames(dfn) <- c("name", "ID", "location", "type", "tier", "kind",
|
||||
"OS", "provState", "status", "created", "numCores", "information")
|
||||
|
||||
return(t(dfn))
|
||||
return(dfn)
|
||||
}
|
||||
|
||||
#' Create HDInsight cluster.
|
||||
|
|
@ -150,10 +142,12 @@ azureHDIConf <- function(azureActiveContext, clustername, resourceGroup,
|
|||
#' @param componentVersion Spark componentVersion. Default : 1.6.2
|
||||
#' @param vmSize Size of nodes: "Large", "Small", "Standard_D14_V2", etc.
|
||||
#' @param mode Provisioning mode, "Sync" or "Async". Use "Async" to immediately return to R session after submission of request
|
||||
#' @param debug Used for debugging purposes. If TRUE, returns json without attempting to connect to Azure
|
||||
#'
|
||||
#' @return Success message
|
||||
#' @family HDInsight functions
|
||||
#' @note See \url{https://docs.microsoft.com/en-us/azure/hdinsight/hdinsight-component-versioning} to learn about HDInsight Versions
|
||||
#' @references https://docs.microsoft.com/en-us/rest/api/hdinsight/hdinsight-cluster#create
|
||||
#' @export
|
||||
azureCreateHDI <- function(azureActiveContext, resourceGroup, location,
|
||||
clustername, kind = c("rserver", "spark", "hadoop"),
|
||||
|
|
@ -162,7 +156,7 @@ azureCreateHDI <- function(azureActiveContext, resourceGroup, location,
|
|||
workers = 2,
|
||||
adminUser, adminPassword, sshUser, sshPassword,
|
||||
hiveServer, hiveDB, hiveUser, hivePassword,
|
||||
vmSize = "Small",
|
||||
vmSize = "Large",
|
||||
subscriptionID, mode = c("Sync", "Async"),
|
||||
verbose = FALSE, debug = FALSE) {
|
||||
assert_that(is.azureActiveContext(azureActiveContext))
|
||||
|
|
@ -245,7 +239,6 @@ azureCreateHDI <- function(azureActiveContext, resourceGroup, location,
|
|||
"?api-version=2015-03-01-preview")
|
||||
|
||||
r <- PUT(URL, azureApiHeaders(azToken), body = bodyI, encode = "json", verbosity)
|
||||
#browser()
|
||||
stopWithAzureError(r)
|
||||
|
||||
azureActiveContext$resourceGroup <- resourceGroup
|
||||
|
|
@ -363,8 +356,9 @@ azureResizeHDI <- function(azureActiveContext, clustername,
|
|||
#' @inheritParams azureAuthenticate
|
||||
#' @inheritParams azureListHDI
|
||||
#'
|
||||
#' @return Returns Dataframe of HDInsight Clusters information
|
||||
#' @return Data frame with HDInsight clusters information
|
||||
#' @family HDInsight functions
|
||||
#' @references https://docs.microsoft.com/en-us/rest/api/hdinsight/hdinsight-cluster#delete
|
||||
#' @export
|
||||
azureDeleteHDI <- function(azureActiveContext, clustername, subscriptionID,
|
||||
resourceGroup, verbose = FALSE) {
|
||||
|
|
@ -382,8 +376,9 @@ azureDeleteHDI <- function(azureActiveContext, clustername, subscriptionID,
|
|||
assert_that(is_clustername(clustername))
|
||||
|
||||
URL <- paste0("https://management.azure.com/subscriptions/", subscriptionID,
|
||||
"/resourceGroups/", resourceGroup, "/providers/Microsoft.HDInsight/clusters/",
|
||||
clustername, "?api-version=2015-03-01-preview")
|
||||
"/resourceGroups/", resourceGroup,
|
||||
"/providers/Microsoft.HDInsight/clusters/", clustername,
|
||||
"?api-version=2015-03-01-preview")
|
||||
|
||||
r <- DELETE(URL, azureApiHeaders(azToken), verbosity)
|
||||
stopWithAzureError(r)
|
||||
|
|
@ -405,14 +400,17 @@ azureDeleteHDI <- function(azureActiveContext, clustername, subscriptionID,
|
|||
#' @param workerNode install on worker nodes
|
||||
#' @param edgeNode install on worker nodes
|
||||
#' @param parameters parameters
|
||||
#' @param wait If TRUE, runs script action synchronously, i.e. waits for successfull completion. If FALSE, submits the action asynchronously
|
||||
#'
|
||||
#' @return Returns Success Message
|
||||
#' @family HDInsight functions
|
||||
#' @references https://docs.microsoft.com/en-us/rest/api/hdinsight/hdinsight-cluster#run-script-actions-on-a-running-cluster-linux-cluster-only
|
||||
#' @export
|
||||
azureRunScriptAction <- function(azureActiveContext, scriptname, scriptURL,
|
||||
headNode = TRUE, workerNode = FALSE, edgeNode = FALSE,
|
||||
clustername, resourceGroup,
|
||||
parameters = "", subscriptionID, verbose = FALSE) {
|
||||
parameters = "", subscriptionID,
|
||||
wait = TRUE, verbose = FALSE) {
|
||||
assert_that(is.azureActiveContext(azureActiveContext))
|
||||
azureCheckToken(azureActiveContext)
|
||||
azToken <- azureActiveContext$Token
|
||||
|
|
@ -426,13 +424,8 @@ azureRunScriptAction <- function(azureActiveContext, scriptname, scriptURL,
|
|||
assert_that(is_subscription_id(subscriptionID))
|
||||
assert_that(is_clustername(clustername))
|
||||
|
||||
if (!length(scriptname)) {
|
||||
stop("Error: No Valid scriptname provided")
|
||||
}
|
||||
if (!length(scriptURL)) {
|
||||
stop("Error: No Valid scriptURL provided")
|
||||
}
|
||||
|
||||
if (!length(scriptname)) stop("Error: No Valid scriptname provided")
|
||||
if (!length(scriptURL)) stop("Error: No Valid scriptURL provided")
|
||||
if (!any(headNode, workerNode, edgeNode)) {
|
||||
stop("Error: No role(headNode,workerNode,edgeNode) flag set to TRUE")
|
||||
}
|
||||
|
|
@ -465,6 +458,7 @@ azureRunScriptAction <- function(azureActiveContext, scriptname, scriptURL,
|
|||
|
||||
azureActiveContext$clustername <- clustername
|
||||
message("Accepted")
|
||||
if (wait) pollStatusScriptAction(azureActiveContext, scriptname = scriptname)
|
||||
return(TRUE)
|
||||
}
|
||||
|
||||
|
|
@ -477,6 +471,7 @@ azureRunScriptAction <- function(azureActiveContext, scriptname, scriptURL,
|
|||
#'
|
||||
#' @return Dataframe of HDInsight Clusters
|
||||
#' @family HDInsight functions
|
||||
#' @references https://docs.microsoft.com/en-us/rest/api/hdinsight/hdinsight-cluster#list-all-persisted-script-actions-for-a-cluster-linux-cluster-only
|
||||
#' @export
|
||||
azureScriptActionHistory <- function(azureActiveContext, resourceGroup,
|
||||
clustername = "*", subscriptionID,
|
||||
|
|
@ -503,20 +498,26 @@ azureScriptActionHistory <- function(azureActiveContext, resourceGroup,
|
|||
stopWithAzureError(r)
|
||||
|
||||
rc <- content(r)$value
|
||||
|
||||
if (length(rc) == 0) {
|
||||
message("No script action history found")
|
||||
}
|
||||
class(rc) <- "azureScriptActionHistory"
|
||||
|
||||
dfn <- do.call(rbind, lapply(rc, function(x) {
|
||||
data.frame(
|
||||
x[c("name", "scriptExecutionId", "startTime")],
|
||||
if (is.null(x$endTime)) list(endTime = NA) else x["endTime"],
|
||||
x[c("status", "uri", "parameters")]
|
||||
)
|
||||
}))
|
||||
|
||||
azureActiveContext$clustername <- clustername
|
||||
return(dfn)
|
||||
return(rc)
|
||||
}
|
||||
|
||||
#' @export
|
||||
#' @param object azureScriptActionHistory object, created by [azureScriptActionHistory()]
|
||||
#' @param ... not used
|
||||
#' @rdname azureScriptActionHistory
|
||||
summary.azureScriptActionHistory <- function(object, ...) {
|
||||
do.call(rbind, lapply(object, function(x) {
|
||||
data.frame(
|
||||
x[c("name", "scriptExecutionId", "startTime")],
|
||||
if (is.null(x$endTime)) list(endTime = NA) else x["endTime"],
|
||||
x[c("status", "uri", "parameters")]
|
||||
)
|
||||
}))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
#' @inheritParams setAzureContext
|
||||
#' @inheritParams azureAuthenticate
|
||||
#'
|
||||
#' @return Dataframe of subscriptionID; Sets AzureContext subscriptionID
|
||||
#' @return data frame with subscriptionID; Sets AzureContext subscriptionID
|
||||
#' @family Resource group functions
|
||||
#' @export
|
||||
azureListSubscriptions <- function(azureActiveContext, verbose = FALSE) {
|
||||
|
|
@ -18,8 +18,8 @@ azureListSubscriptions <- function(azureActiveContext, verbose = FALSE) {
|
|||
stopWithAzureError(r)
|
||||
|
||||
dfs <- lapply(content(r), data.frame, stringsAsFactors = FALSE)
|
||||
df1 <- do.call(rbind, dfs))
|
||||
if (nrow(df1) == 1) azureActiveContext$subscriptionID <- df1$subscriptionID[1]
|
||||
df1 <- do.call(rbind, dfs)
|
||||
if (nrow(df1) == 1) azureActiveContext$subscriptionID <- df1$subscriptionId[1]
|
||||
return(df1)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -16,13 +16,18 @@
|
|||
#' - [azureListStorageContainers()]
|
||||
#' - [azureListStorageBlobs()]
|
||||
#' * Virtual Machines
|
||||
#' - List VMs: [azureListVM()]
|
||||
#' - To create a virtual machine, use [azureDeployTemplate()] with a suitable template
|
||||
#' - Start a VM: [azureStartVM()]
|
||||
#' - Stop a VM: [azureStopVM()]
|
||||
#' - Get status: [azureVMStatus()]
|
||||
#' - List VMs: [azureListVM()]
|
||||
#' * HDInsight (Nodes, Hive, Spark)
|
||||
#' * Azure Resource Manager
|
||||
#' * HDInsight clusters:
|
||||
#' - [azureListHDI()]
|
||||
#' - [azureCreateHDI()]
|
||||
#' - [azureResizeHDI()]
|
||||
#' - [azureDeleteHDI()]
|
||||
#' - [azureRunScriptAction()]
|
||||
#' - [azureScriptActionHistory()]
|
||||
#'
|
||||
#'
|
||||
#' @name AzureSMR
|
||||
|
|
@ -33,7 +38,6 @@
|
|||
#' @importFrom utils browseURL URLencode ls.str str
|
||||
#' @importFrom digest hmac
|
||||
#' @importFrom base64enc base64encode base64decode
|
||||
#' @importFrom plyr rbind.fill
|
||||
#' @importFrom jsonlite fromJSON
|
||||
#' @importFrom httr add_headers headers content status_code http_status authenticate
|
||||
#' @importFrom httr GET PUT DELETE POST
|
||||
|
|
|
|||
|
|
@ -79,6 +79,7 @@ azureSAGetKey <- function(azureActiveContext, storageAccount,
|
|||
#' @inheritParams azureAuthenticate
|
||||
#' @inheritParams azureSAGetKey
|
||||
#' @param location A string for the location to create storage account
|
||||
#' @param asynchronous If TRUE, submits asynchronous request to Azure. Otherwise waits until storage account is created.
|
||||
#' @family Storage account functions
|
||||
#' @export
|
||||
azureCreateStorageAccount <- function(azureActiveContext, storageAccount,
|
||||
|
|
|
|||
|
|
@ -83,7 +83,6 @@ azureListScaleSetNetwork <- function(azureActiveContext, resourceGroup, location
|
|||
lbs <- df$value$name
|
||||
dfn <- data.frame(lbname = "", publicIpAdress = "", inport = "", outport = "")
|
||||
clust <- length(lbs)
|
||||
#browser()
|
||||
if (clust > 0) {
|
||||
for (i in seq_along(lbs)) {
|
||||
lb <- lbs[i]
|
||||
|
|
@ -142,7 +141,6 @@ azureListScaleSetNetwork <- function(azureActiveContext, resourceGroup, location
|
|||
clust <- nrow(dfn)
|
||||
clust2 <- length(df2$properties$ipAddress)
|
||||
if (clust2 > 0) {
|
||||
#if (length(df2$properties$dnsSettings$fqdn) != length(df2$properties$ipAddress)) browser()
|
||||
data.frame(
|
||||
fqdn = if (is.null(df2$properties$dnsSettings$fqdn)) "" else df2$properties$dnsSettings$fqdn,
|
||||
ipAddress = df2$properties$ipAddress,
|
||||
|
|
|
|||
|
|
@ -133,9 +133,7 @@ getSig <- function(azureActiveContext, url, verb, key, storageAccount,
|
|||
|
||||
|
||||
stopWithAzureError <- function(r) {
|
||||
#if (status_code(r) %in% c(200, 201, 202, 204)) return()
|
||||
#browser()
|
||||
if(status_code(r) < 300) return()
|
||||
if (status_code(r) < 300) return()
|
||||
msg <- paste0(as.character(sys.call(1))[1], "()") # Name of calling fucntion
|
||||
addToMsg <- function(x) {
|
||||
if (!is.null(x)) x <- strwrap(x)
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ pollStatusVM <- function(azureActiveContext) {
|
|||
pollStatusHDI <- function(azureActiveContext, clustername) {
|
||||
|
||||
message("HDI request submitted: ", Sys.time())
|
||||
message("Key: (.) - in progress, S - succeeded, E - error")
|
||||
message("Key: (.) - in progress, S - succeeded, E - error, F - failed")
|
||||
iteration <- 0
|
||||
waiting <- TRUE
|
||||
while (iteration < 500 && waiting) {
|
||||
|
|
@ -100,26 +100,68 @@ pollStatusHDI <- function(azureActiveContext, clustername) {
|
|||
rc <- switch(tolower(summary),
|
||||
succeeded = "S",
|
||||
error = "E",
|
||||
failed = "F",
|
||||
inprogress = ".",
|
||||
"?"
|
||||
)
|
||||
|
||||
message(rc, appendLF = FALSE)
|
||||
if (rc %in% c("S", "E")) {
|
||||
if (rc %in% c("S", "E", "F")) {
|
||||
waiting = FALSE
|
||||
}
|
||||
if (rc == "E") {
|
||||
warning(paste("Error deploying: ", Sys.time()))
|
||||
warning(fromJSON(status$properties$error$details$message)$error$message)
|
||||
if (rc %in% c("E", "F")) {
|
||||
message("")
|
||||
warning(paste("Error deploying: ", Sys.time()), call. = FALSE, immediate. = TRUE)
|
||||
return(FALSE)
|
||||
}
|
||||
|
||||
if (rc == "?") message(status)
|
||||
if (rc == "?") message(summary)
|
||||
|
||||
iteration <- iteration + 1
|
||||
if (!rc %in% c("S", "E")) Sys.sleep(10)
|
||||
if (!rc %in% c("S", "E", "F")) Sys.sleep(10)
|
||||
}
|
||||
message("")
|
||||
message("HDI request completed: ", Sys.time())
|
||||
return(TRUE)
|
||||
}
|
||||
|
||||
|
||||
pollStatusScriptAction <- function(azureActiveContext, scriptname) {
|
||||
|
||||
message("Script action request submitted: ", Sys.time())
|
||||
message("Key: A - accepted, (.) - in progress, S - succeeded, E - error, F - failed")
|
||||
iteration <- 0
|
||||
waiting <- TRUE
|
||||
while (iteration < 500 && waiting) {
|
||||
|
||||
status <- azureScriptActionHistory(azureActiveContext)
|
||||
idx <- which(sapply(status, "[[", "name") == scriptname)[1]
|
||||
summary <- status[[idx]]$status
|
||||
rc <- switch(tolower(summary),
|
||||
accepted = "A",
|
||||
succeeded = "S",
|
||||
error = "E",
|
||||
failed = "F",
|
||||
inprogress = ".",
|
||||
"?"
|
||||
)
|
||||
|
||||
message(rc, appendLF = FALSE)
|
||||
if (rc %in% c("S", "E", "F")) {
|
||||
waiting = FALSE
|
||||
}
|
||||
if (rc %in% c("E", "F")) {
|
||||
message("")
|
||||
warning(paste("Error deploying: ", Sys.time()), call. = FALSE, immediate. = TRUE)
|
||||
return(FALSE)
|
||||
}
|
||||
|
||||
if (rc == "?") message(summary)
|
||||
|
||||
iteration <- iteration + 1
|
||||
if (!rc %in% c("S", "E", "F")) Sys.sleep(5)
|
||||
}
|
||||
message("")
|
||||
message("Script action completed: ", Sys.time())
|
||||
return(TRUE)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,14 +31,21 @@ This enables you to use and change many Azure resources. The following is an inc
|
|||
}
|
||||
\item Virtual Machines
|
||||
\itemize{
|
||||
\item List VMs: \code{\link[=azureListVM]{azureListVM()}}
|
||||
\item To create a virtual machine, use \code{\link[=azureDeployTemplate]{azureDeployTemplate()}} with a suitable template
|
||||
\item Start a VM: \code{\link[=azureStartVM]{azureStartVM()}}
|
||||
\item Stop a VM: \code{\link[=azureStopVM]{azureStopVM()}}
|
||||
\item Get status: \code{\link[=azureVMStatus]{azureVMStatus()}}
|
||||
\item List VMs: \code{\link[=azureListVM]{azureListVM()}}
|
||||
}
|
||||
\item HDInsight (Nodes, Hive, Spark)
|
||||
\item Azure Resource Manager
|
||||
\item HDInsight clusters:
|
||||
\itemize{
|
||||
\item \code{\link[=azureListHDI]{azureListHDI()}}
|
||||
\item \code{\link[=azureCreateHDI]{azureCreateHDI()}}
|
||||
\item \code{\link[=azureResizeHDI]{azureResizeHDI()}}
|
||||
\item \code{\link[=azureDeleteHDI]{azureDeleteHDI()}}
|
||||
\item \code{\link[=azureRunScriptAction]{azureRunScriptAction()}}
|
||||
\item \code{\link[=azureScriptActionHistory]{azureScriptActionHistory()}}
|
||||
}
|
||||
}
|
||||
}
|
||||
\keyword{package}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ azureBlobCD(azureActiveContext, directory, container, file, storageAccount,
|
|||
\item Local filename to store in Azure blob
|
||||
}}
|
||||
|
||||
\item{storageAccount}{Name of the azure storage account}
|
||||
\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.}
|
||||
|
||||
\item{storageKey}{Storage key associated with storage account}
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ azureBlobFind(azureActiveContext, file, storageAccount, storageKey, container,
|
|||
\item Local filename to store in Azure blob
|
||||
}}
|
||||
|
||||
\item{storageAccount}{Name of the azure storage account}
|
||||
\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.}
|
||||
|
||||
\item{storageKey}{Storage key associated with storage account}
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ azureBlobLS(azureActiveContext, directory, recursive = FALSE, storageAccount,
|
|||
|
||||
\item{recursive}{If TRUE, list blob store directories recursively}
|
||||
|
||||
\item{storageAccount}{Name of the azure storage account}
|
||||
\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.}
|
||||
|
||||
\item{storageKey}{Storage key associated with storage account}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,24 +2,27 @@
|
|||
% Please edit documentation in R/AzureHDI.R
|
||||
\name{azureCreateHDI}
|
||||
\alias{azureCreateHDI}
|
||||
\title{Create Specifed HDInsight Cluster.}
|
||||
\title{Create HDInsight cluster.}
|
||||
\usage{
|
||||
azureCreateHDI(azureActiveContext, clustername, location, kind = "spark",
|
||||
storageAccount, storageKey, version = "3.4", componentVersion = "1.6.2",
|
||||
workers = 2, adminUser, adminPassword, sshUser, sshPassword, hiveServer,
|
||||
hiveDB, hiveUser, hivePassword, resourceGroup, vmSize = "Large",
|
||||
subscriptionID, mode = "Sync", verbose = FALSE)
|
||||
azureCreateHDI(azureActiveContext, resourceGroup, location, clustername,
|
||||
kind = c("rserver", "spark", "hadoop"), storageAccount, storageKey,
|
||||
version = "3.5", componentVersion = "1.6.2", workers = 2, adminUser,
|
||||
adminPassword, sshUser, sshPassword, hiveServer, hiveDB, hiveUser,
|
||||
hivePassword, vmSize = "Large", subscriptionID, mode = c("Sync", "Async"),
|
||||
verbose = FALSE, debug = FALSE)
|
||||
}
|
||||
\arguments{
|
||||
\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}}
|
||||
|
||||
\item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
\item{resourceGroup}{Name of the resource group}
|
||||
|
||||
\item{location}{Azure region, e.g. 'westeurope' or 'southcentralus'}
|
||||
|
||||
\item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{kind}{HDinsight kind: "hadoop","spark" or "rserver"}
|
||||
|
||||
\item{storageAccount}{Name of the azure storage account}
|
||||
\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.}
|
||||
|
||||
\item{storageKey}{Storage key associated with storage account}
|
||||
|
||||
|
|
@ -45,8 +48,6 @@ azureCreateHDI(azureActiveContext, clustername, location, kind = "spark",
|
|||
|
||||
\item{hivePassword}{Hive user password}
|
||||
|
||||
\item{resourceGroup}{Name of the resource group}
|
||||
|
||||
\item{vmSize}{Size of nodes: "Large", "Small", "Standard_D14_V2", etc.}
|
||||
|
||||
\item{subscriptionID}{Subscription ID. This is obtained automatically by \code{\link[=azureAuthenticate]{azureAuthenticate()}} when only a single subscriptionID is available via Active Directory}
|
||||
|
|
@ -54,16 +55,21 @@ azureCreateHDI(azureActiveContext, clustername, location, kind = "spark",
|
|||
\item{mode}{Provisioning mode, "Sync" or "Async". Use "Async" to immediately return to R session after submission of request}
|
||||
|
||||
\item{verbose}{Print Tracing information (Default False)}
|
||||
|
||||
\item{debug}{Used for debugging purposes. If TRUE, returns json without attempting to connect to Azure}
|
||||
}
|
||||
\value{
|
||||
Success message
|
||||
}
|
||||
\description{
|
||||
Create Specifed HDInsight Cluster.
|
||||
Create HDInsight cluster.
|
||||
}
|
||||
\note{
|
||||
See \url{https://docs.microsoft.com/en-us/azure/hdinsight/hdinsight-component-versioning} to learn about HDInsight Versions
|
||||
}
|
||||
\references{
|
||||
https://docs.microsoft.com/en-us/rest/api/hdinsight/hdinsight-cluster#create
|
||||
}
|
||||
\seealso{
|
||||
Other HDInsight functions: \code{\link{azureDeleteHDI}},
|
||||
\code{\link{azureHDIConf}}, \code{\link{azureListHDI}},
|
||||
|
|
|
|||
|
|
@ -5,12 +5,13 @@
|
|||
\title{Create an Azure Storage Account.}
|
||||
\usage{
|
||||
azureCreateStorageAccount(azureActiveContext, storageAccount,
|
||||
location = "northeurope", resourceGroup, subscriptionID, verbose = FALSE)
|
||||
location = "northeurope", resourceGroup, subscriptionID,
|
||||
asynchronous = FALSE, verbose = FALSE)
|
||||
}
|
||||
\arguments{
|
||||
\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}}
|
||||
|
||||
\item{storageAccount}{Name of the azure storage account}
|
||||
\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.}
|
||||
|
||||
\item{location}{A string for the location to create storage account}
|
||||
|
||||
|
|
@ -18,6 +19,8 @@ azureCreateStorageAccount(azureActiveContext, storageAccount,
|
|||
|
||||
\item{subscriptionID}{Subscription ID. This is obtained automatically by \code{\link[=azureAuthenticate]{azureAuthenticate()}} when only a single subscriptionID is available via Active Directory}
|
||||
|
||||
\item{asynchronous}{If TRUE, submits asynchronous request to Azure. Otherwise waits until storage account is created.}
|
||||
|
||||
\item{verbose}{Print Tracing information (Default False)}
|
||||
}
|
||||
\description{
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ azureCreateStorageContainer(azureActiveContext, container, storageAccount,
|
|||
|
||||
\item{container}{Storage container name. See \code{\link[=azureListStorageContainers]{azureListStorageContainers()}}}
|
||||
|
||||
\item{storageAccount}{Name of the azure storage account}
|
||||
\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.}
|
||||
|
||||
\item{storageKey}{Storage key associated with storage account}
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ azureDeleteBlob(azureActiveContext, blob, directory, storageAccount, storageKey,
|
|||
|
||||
\item{directory}{Blob store directory to list for content}
|
||||
|
||||
\item{storageAccount}{Name of the azure storage account}
|
||||
\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.}
|
||||
|
||||
\item{storageKey}{Storage key associated with storage account}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
% Please edit documentation in R/AzureHDI.R
|
||||
\name{azureDeleteHDI}
|
||||
\alias{azureDeleteHDI}
|
||||
\title{Delete Specifed HDInsight Cluster.}
|
||||
\title{Delete HDInsight cluster.}
|
||||
\usage{
|
||||
azureDeleteHDI(azureActiveContext, clustername, subscriptionID, resourceGroup,
|
||||
verbose = FALSE)
|
||||
|
|
@ -19,10 +19,13 @@ azureDeleteHDI(azureActiveContext, clustername, subscriptionID, resourceGroup,
|
|||
\item{verbose}{Print Tracing information (Default False)}
|
||||
}
|
||||
\value{
|
||||
Returns Dataframe of HDInsight Clusters information
|
||||
Data frame with HDInsight clusters information
|
||||
}
|
||||
\description{
|
||||
Delete Specifed HDInsight Cluster.
|
||||
Delete HDInsight cluster.
|
||||
}
|
||||
\references{
|
||||
https://docs.microsoft.com/en-us/rest/api/hdinsight/hdinsight-cluster#delete
|
||||
}
|
||||
\seealso{
|
||||
Other HDInsight functions: \code{\link{azureCreateHDI}},
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ azureDeleteStorageContainer(azureActiveContext, container, storageAccount,
|
|||
|
||||
\item{container}{Storage container name. See \code{\link[=azureListStorageContainers]{azureListStorageContainers()}}}
|
||||
|
||||
\item{storageAccount}{Name of the azure storage account}
|
||||
\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.}
|
||||
|
||||
\item{storageKey}{Storage key associated with storage account}
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ azureDeletestorageAccount(azureActiveContext, storageAccount, resourceGroup,
|
|||
\arguments{
|
||||
\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}}
|
||||
|
||||
\item{storageAccount}{Name of the azure storage account}
|
||||
\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.}
|
||||
|
||||
\item{resourceGroup}{Name of the resource group}
|
||||
|
||||
|
|
|
|||
|
|
@ -5,8 +5,8 @@
|
|||
\title{Deploy a resource from an Azure Resource Manager (ARM) template.}
|
||||
\usage{
|
||||
azureDeployTemplate(azureActiveContext, deplname, templateURL, paramURL,
|
||||
templateJSON, paramJSON, mode = "Sync", resourceGroup, subscriptionID,
|
||||
verbose = FALSE)
|
||||
templateJSON, paramJSON, mode = c("Sync", "Async"), resourceGroup,
|
||||
subscriptionID, verbose = FALSE)
|
||||
}
|
||||
\arguments{
|
||||
\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}}
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ azureGetBlob(azureActiveContext, blob, directory, type = "text",
|
|||
|
||||
\item{type}{String, either "text" or "raw". Passed to \code{\link[httr:content]{httr::content()}}}
|
||||
|
||||
\item{storageAccount}{Name of the azure storage account}
|
||||
\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.}
|
||||
|
||||
\item{storageKey}{Storage key associated with storage account}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
% Please edit documentation in R/AzureHDI.R
|
||||
\name{azureHDIConf}
|
||||
\alias{azureHDIConf}
|
||||
\title{Get Configuration Information for a specified cluster name.}
|
||||
\title{Get configuration information for a specified cluster name.}
|
||||
\usage{
|
||||
azureHDIConf(azureActiveContext, clustername, resourceGroup, subscriptionID,
|
||||
name, type, location, verbose = FALSE)
|
||||
|
|
@ -28,7 +28,7 @@ azureHDIConf(azureActiveContext, clustername, resourceGroup, subscriptionID,
|
|||
Returns Dataframe of HDInsight Clusters information
|
||||
}
|
||||
\description{
|
||||
Get Configuration Information for a specified cluster name.
|
||||
Get configuration information for a specified cluster name.
|
||||
}
|
||||
\seealso{
|
||||
Other HDInsight functions: \code{\link{azureCreateHDI}},
|
||||
|
|
|
|||
|
|
@ -14,9 +14,9 @@ azureHiveSQL(azureActiveContext, CMD, clustername, hdiAdmin, hdiPassword,
|
|||
|
||||
\item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{hdiAdmin}{HDInsight admin username}
|
||||
\item{hdiAdmin}{HDInsight admin username. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{hdiPassword}{HDInsight admin password}
|
||||
\item{hdiPassword}{HDInsight admin password. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{path}{path}
|
||||
|
||||
|
|
|
|||
|
|
@ -12,9 +12,9 @@ azureHiveStatus(azureActiveContext, clustername, hdiAdmin, hdiPassword,
|
|||
|
||||
\item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{hdiAdmin}{HDInsight admin username}
|
||||
\item{hdiAdmin}{HDInsight admin username. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{hdiPassword}{HDInsight admin password}
|
||||
\item{hdiPassword}{HDInsight admin password. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{verbose}{Print Tracing information (Default False)}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,11 +25,14 @@ azureListHDI(azureActiveContext, resourceGroup, clustername = "*",
|
|||
\item{verbose}{Print Tracing information (Default False)}
|
||||
}
|
||||
\value{
|
||||
Returns Dataframe of HDInsight Clusters
|
||||
data frame with summary information of HDI clusters
|
||||
}
|
||||
\description{
|
||||
Get all HDInsight Clusters in default Subscription or details for a specified cluster name.
|
||||
}
|
||||
\references{
|
||||
https://docs.microsoft.com/en-us/rest/api/hdinsight/hdinsight-cluster#list-by-subscription
|
||||
}
|
||||
\seealso{
|
||||
Other HDInsight functions: \code{\link{azureCreateHDI}},
|
||||
\code{\link{azureDeleteHDI}}, \code{\link{azureHDIConf}},
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ azureListStorageBlobs(azureActiveContext, storageAccount, storageKey, container,
|
|||
\arguments{
|
||||
\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}}
|
||||
|
||||
\item{storageAccount}{Name of the azure storage account}
|
||||
\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.}
|
||||
|
||||
\item{storageKey}{Storage key associated with storage account}
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ azureListStorageContainers(azureActiveContext, storageAccount, storageKey,
|
|||
\arguments{
|
||||
\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}}
|
||||
|
||||
\item{storageAccount}{Name of the azure storage account}
|
||||
\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.}
|
||||
|
||||
\item{storageKey}{Storage key associated with storage account}
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ azureListSubscriptions(azureActiveContext, verbose = FALSE)
|
|||
\item{verbose}{Print Tracing information (Default False)}
|
||||
}
|
||||
\value{
|
||||
Dataframe of subscriptionID; Sets AzureContext subscriptionID
|
||||
data frame with subscriptionID; Sets AzureContext subscriptionID
|
||||
}
|
||||
\description{
|
||||
Get available subscriptions.
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ azurePutBlob(azureActiveContext, blob, contents = "", file = "", directory,
|
|||
|
||||
\item{directory}{Blob store directory to list for content}
|
||||
|
||||
\item{storageAccount}{Name of the azure storage account}
|
||||
\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.}
|
||||
|
||||
\item{storageKey}{Storage key associated with storage account}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,19 +2,20 @@
|
|||
% Please edit documentation in R/AzureHDI.R
|
||||
\name{azureResizeHDI}
|
||||
\alias{azureResizeHDI}
|
||||
\title{Resize a HDInsight Cluster role.}
|
||||
\title{Resize a HDInsight cluster role.}
|
||||
\usage{
|
||||
azureResizeHDI(azureActiveContext, clustername, role = "worker", size = 2,
|
||||
mode = "Sync", subscriptionID, resourceGroup, verbose = FALSE)
|
||||
azureResizeHDI(azureActiveContext, clustername, role = c("worker", "head",
|
||||
"edge"), size = 2, mode = c("Sync", "Async"), subscriptionID,
|
||||
resourceGroup, verbose = FALSE)
|
||||
}
|
||||
\arguments{
|
||||
\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}}
|
||||
|
||||
\item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{role}{role type: 'worker', 'head' or 'Edge'}
|
||||
\item{role}{role type: 'worker', 'head' or 'edge'}
|
||||
|
||||
\item{size}{Desired size of role type}
|
||||
\item{size}{Numeric: the number of nodes for this type of role}
|
||||
|
||||
\item{mode}{Provisioning mode, "Sync" or "Async". Use "Async" to immediately return to R session after submission of request}
|
||||
|
||||
|
|
@ -25,7 +26,7 @@ azureResizeHDI(azureActiveContext, clustername, role = "worker", size = 2,
|
|||
\item{verbose}{Print Tracing information (Default False)}
|
||||
}
|
||||
\description{
|
||||
Resize a HDInsight Cluster role.
|
||||
Resize a HDInsight cluster role.
|
||||
}
|
||||
\seealso{
|
||||
Other HDInsight functions: \code{\link{azureCreateHDI}},
|
||||
|
|
|
|||
|
|
@ -2,24 +2,25 @@
|
|||
% Please edit documentation in R/AzureHDI.R
|
||||
\name{azureRunScriptAction}
|
||||
\alias{azureRunScriptAction}
|
||||
\title{Run Script Action on HDI Cluster.}
|
||||
\title{Run script action on HDI cluster.}
|
||||
\usage{
|
||||
azureRunScriptAction(azureActiveContext, scriptname = "script1", scriptURL,
|
||||
azureRunScriptAction(azureActiveContext, scriptname, scriptURL,
|
||||
headNode = TRUE, workerNode = FALSE, edgeNode = FALSE, clustername,
|
||||
resourceGroup, parameters = "", subscriptionID, verbose = FALSE)
|
||||
resourceGroup, parameters = "", subscriptionID, wait = TRUE,
|
||||
verbose = FALSE)
|
||||
}
|
||||
\arguments{
|
||||
\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}}
|
||||
|
||||
\item{scriptname}{Identifier for Custom action scrript operation}
|
||||
\item{scriptname}{Identifier for Custom action script operation}
|
||||
|
||||
\item{scriptURL}{URL to custom action script (Sring)}
|
||||
\item{scriptURL}{URL to custom action script}
|
||||
|
||||
\item{headNode}{install on head nodes (default FALSE)}
|
||||
\item{headNode}{install on head nodes}
|
||||
|
||||
\item{workerNode}{install on worker nodes (default FALSE)}
|
||||
\item{workerNode}{install on worker nodes}
|
||||
|
||||
\item{edgeNode}{install on worker nodes (default FALSE)}
|
||||
\item{edgeNode}{install on worker nodes}
|
||||
|
||||
\item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
|
|
@ -29,13 +30,18 @@ azureRunScriptAction(azureActiveContext, scriptname = "script1", scriptURL,
|
|||
|
||||
\item{subscriptionID}{Subscription ID. This is obtained automatically by \code{\link[=azureAuthenticate]{azureAuthenticate()}} when only a single subscriptionID is available via Active Directory}
|
||||
|
||||
\item{wait}{If TRUE, runs script action synchronously, i.e. waits for successfull completion. If FALSE, submits the action asynchronously}
|
||||
|
||||
\item{verbose}{Print Tracing information (Default False)}
|
||||
}
|
||||
\value{
|
||||
Returns Success Message
|
||||
}
|
||||
\description{
|
||||
Run Script Action on HDI Cluster.
|
||||
Run script action on HDI cluster.
|
||||
}
|
||||
\references{
|
||||
https://docs.microsoft.com/en-us/rest/api/hdinsight/hdinsight-cluster#run-script-actions-on-a-running-cluster-linux-cluster-only
|
||||
}
|
||||
\seealso{
|
||||
Other HDInsight functions: \code{\link{azureCreateHDI}},
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ azureSAGetKey(azureActiveContext, storageAccount, resourceGroup, subscriptionID,
|
|||
\arguments{
|
||||
\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}}
|
||||
|
||||
\item{storageAccount}{storageAccount}
|
||||
\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.}
|
||||
|
||||
\item{resourceGroup}{Name of the resource group}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,10 +2,13 @@
|
|||
% Please edit documentation in R/AzureHDI.R
|
||||
\name{azureScriptActionHistory}
|
||||
\alias{azureScriptActionHistory}
|
||||
\title{Get all HDInsight Script Action History for a specified cluster name.}
|
||||
\alias{summary.azureScriptActionHistory}
|
||||
\title{Get all HDInsight script action history for a specified cluster name.}
|
||||
\usage{
|
||||
azureScriptActionHistory(azureActiveContext, resourceGroup, clustername = "*",
|
||||
subscriptionID, name, type, verbose = FALSE)
|
||||
|
||||
\method{summary}{azureScriptActionHistory}(object, ...)
|
||||
}
|
||||
\arguments{
|
||||
\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}}
|
||||
|
|
@ -21,12 +24,19 @@ azureScriptActionHistory(azureActiveContext, resourceGroup, clustername = "*",
|
|||
\item{type}{filter by resource type}
|
||||
|
||||
\item{verbose}{Print Tracing information (Default False)}
|
||||
|
||||
\item{object}{azureScriptActionHistory object, created by \code{\link[=azureScriptActionHistory]{azureScriptActionHistory()}}}
|
||||
|
||||
\item{...}{not used}
|
||||
}
|
||||
\value{
|
||||
Dataframe of HDInsight Clusters
|
||||
}
|
||||
\description{
|
||||
Get all HDInsight Script Action History for a specified cluster name.
|
||||
Get all HDInsight script action history for a specified cluster name.
|
||||
}
|
||||
\references{
|
||||
https://docs.microsoft.com/en-us/rest/api/hdinsight/hdinsight-cluster#list-all-persisted-script-actions-for-a-cluster-linux-cluster-only
|
||||
}
|
||||
\seealso{
|
||||
Other HDInsight functions: \code{\link{azureCreateHDI}},
|
||||
|
|
|
|||
|
|
@ -14,9 +14,9 @@ azureSparkCMD(azureActiveContext, CMD, clustername, hdiAdmin, hdiPassword,
|
|||
|
||||
\item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{hdiAdmin}{HDInsight admin username}
|
||||
\item{hdiAdmin}{HDInsight admin username. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{hdiPassword}{HDInsight admin password}
|
||||
\item{hdiPassword}{HDInsight admin password. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{sessionID}{Spark sessionID. See \code{\link[=azureSparkCMD]{azureSparkCMD()}}}
|
||||
|
||||
|
|
|
|||
|
|
@ -14,9 +14,9 @@ azureSparkJob(azureActiveContext, FILE, clustername, hdiAdmin, hdiPassword,
|
|||
|
||||
\item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{hdiAdmin}{HDInsight admin username}
|
||||
\item{hdiAdmin}{HDInsight admin username. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{hdiPassword}{HDInsight admin password}
|
||||
\item{hdiPassword}{HDInsight admin password. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{log}{log}
|
||||
|
||||
|
|
|
|||
|
|
@ -12,9 +12,9 @@ azureSparkListJobs(azureActiveContext, clustername, hdiAdmin, hdiPassword,
|
|||
|
||||
\item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{hdiAdmin}{HDInsight admin username}
|
||||
\item{hdiAdmin}{HDInsight admin username. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{hdiPassword}{HDInsight admin password}
|
||||
\item{hdiPassword}{HDInsight admin password. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{verbose}{Print Tracing information (Default False)}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,9 +12,9 @@ azureSparkListSessions(azureActiveContext, clustername, hdiAdmin, hdiPassword,
|
|||
|
||||
\item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{hdiAdmin}{HDInsight admin username}
|
||||
\item{hdiAdmin}{HDInsight admin username. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{hdiPassword}{HDInsight admin password}
|
||||
\item{hdiPassword}{HDInsight admin password. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{verbose}{Print Tracing information (Default False)}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,11 +12,11 @@ azureSparkNewSession(azureActiveContext, clustername, hdiAdmin, hdiPassword,
|
|||
|
||||
\item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{hdiAdmin}{HDInsight admin username}
|
||||
\item{hdiAdmin}{HDInsight admin username. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{hdiPassword}{HDInsight admin password}
|
||||
\item{hdiPassword}{HDInsight admin password. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{kind}{HDinsight kind: "hadoop","spark" or "pyspark"}
|
||||
\item{kind}{HDinsight kind: "hadoop","spark" or "rserver". See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{verbose}{Print Tracing information (Default False)}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,9 +12,9 @@ azureSparkStopSession(azureActiveContext, clustername, hdiAdmin, hdiPassword,
|
|||
|
||||
\item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{hdiAdmin}{HDInsight admin username}
|
||||
\item{hdiAdmin}{HDInsight admin username. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{hdiPassword}{HDInsight admin password}
|
||||
\item{hdiPassword}{HDInsight admin password. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{sessionID}{Spark sessionID. See \code{\link[=azureSparkCMD]{azureSparkCMD()}}}
|
||||
|
||||
|
|
|
|||
|
|
@ -4,9 +4,9 @@
|
|||
\alias{setAzureContext}
|
||||
\title{Updates azureActiveContext object.}
|
||||
\usage{
|
||||
setAzureContext(azureActiveContext, tenantID, clientID, authKey, azToken,
|
||||
subscriptionID, resourceGroup, storageKey, storageAccount, container, blob,
|
||||
vmName, hdiAdmin, hdiPassword, clustername, kind, sessionID)
|
||||
setAzureContext(azureActiveContext, tenantID, clientID, authKey, subscriptionID,
|
||||
resourceGroup, storageKey, storageAccount, container, blob, vmName, hdiAdmin,
|
||||
hdiPassword, clustername, kind, sessionID)
|
||||
}
|
||||
\arguments{
|
||||
\item{azureActiveContext}{A container used for caching variables used by \code{AzureSMR}}
|
||||
|
|
@ -17,15 +17,13 @@ setAzureContext(azureActiveContext, tenantID, clientID, authKey, azToken,
|
|||
|
||||
\item{authKey}{The Authentication Key provided during creation of the Active Directory application / service principal}
|
||||
|
||||
\item{azToken}{Azure authentication token, obtained by \code{\link[=azureAuthenticate]{azureAuthenticate()}}}
|
||||
|
||||
\item{subscriptionID}{Subscription ID. This is obtained automatically by \code{\link[=azureAuthenticate]{azureAuthenticate()}} when only a single subscriptionID is available via Active Directory}
|
||||
|
||||
\item{resourceGroup}{Name of the resource group}
|
||||
|
||||
\item{storageKey}{Storage key associated with storage account}
|
||||
|
||||
\item{storageAccount}{Name of the azure storage account}
|
||||
\item{storageAccount}{Name of the azure storage account. Storage account names must be between 3 and 24 characters in length and use numbers and lower-case letters only.}
|
||||
|
||||
\item{container}{Storage container name. See \code{\link[=azureListStorageContainers]{azureListStorageContainers()}}}
|
||||
|
||||
|
|
@ -33,13 +31,13 @@ setAzureContext(azureActiveContext, tenantID, clientID, authKey, azToken,
|
|||
|
||||
\item{vmName}{Name of the virtual machine}
|
||||
|
||||
\item{hdiAdmin}{HDInsight admin username}
|
||||
\item{hdiAdmin}{HDInsight admin username. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{hdiPassword}{HDInsight admin password}
|
||||
\item{hdiPassword}{HDInsight admin password. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{clustername}{Cluster name, used for HDI and Spark clusters. See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{kind}{HDinsight kind: "hadoop","spark" or "pyspark"}
|
||||
\item{kind}{HDinsight kind: "hadoop","spark" or "rserver". See \code{\link[=azureCreateHDI]{azureCreateHDI()}}}
|
||||
|
||||
\item{sessionID}{Spark sessionID. See \code{\link[=azureSparkCMD]{azureSparkCMD()}}}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,8 @@ azureAuthenticate(asc, verbose = FALSE)
|
|||
|
||||
timestamp <- format(Sys.time(), format = "%y%m%d%H%M")
|
||||
resourceGroup_name <- paste0("_AzureSMtest_", timestamp)
|
||||
cluster_name <- paste0("azuresmrhdi", timestamp)
|
||||
storage_name <- paste0("azuresmrhdist", timestamp)
|
||||
|
||||
test_that("Can create resource group", {
|
||||
skip_if_missing_config(settingsfile)
|
||||
|
|
@ -46,58 +48,63 @@ test_that("Can create HDI cluster", {
|
|||
"storageAccount"
|
||||
)
|
||||
expect_message(
|
||||
azureCreateHDI(asc, resourceGroup = resourceGroup_name, clustername = "azuresmr_hdi_test",
|
||||
storageAccount = "azuresmrhditest",
|
||||
adminUser = "Azuresmr_test1", adminPassword = "Password_1",
|
||||
sshUser = "sssUser_test1", sshPassword = "Password_1",
|
||||
debug = TRUE
|
||||
azureCreateHDI(asc, resourceGroup = resourceGroup_name,
|
||||
clustername = cluster_name,
|
||||
storageAccount = storage_name,
|
||||
adminUser = "Azuresmr_test1", adminPassword = "Password_1",
|
||||
sshUser = "sssUser_test1", sshPassword = "Password_1",
|
||||
debug = TRUE
|
||||
),
|
||||
"creating storage account"
|
||||
)
|
||||
|
||||
expect_error(
|
||||
azureCreateHDI(asc, resourceGroup = resourceGroup_name, clustername = "azuresmr_hdi_test",
|
||||
storageAccount = "azuresmrhditest",
|
||||
adminUser = "Azuresmr_test1", adminPassword = "Azuresmr_test1",
|
||||
sshUser = "sssUser_test1", sshPassword = "sssUser_test1",
|
||||
debug = FALSE
|
||||
azureCreateHDI(asc, resourceGroup = resourceGroup_name,
|
||||
clustername = cluster_name,
|
||||
storageAccount = storage_name,
|
||||
adminUser = "Azuresmr_test1", adminPassword = "Azuresmr_test1",
|
||||
sshUser = "sssUser_test1", sshPassword = "sssUser_test1",
|
||||
debug = FALSE
|
||||
),
|
||||
"should not contain 3 consecutive letters from the username"
|
||||
)
|
||||
|
||||
# debug - default
|
||||
expect_is(
|
||||
azureCreateHDI(asc, resourceGroup = resourceGroup_name, clustername = "azuresmr_hdi_test",
|
||||
storageAccount = "azuresmrhditest",
|
||||
azureCreateHDI(asc, resourceGroup = resourceGroup_name,
|
||||
clustername = cluster_name,
|
||||
storageAccount = storage_name,
|
||||
adminUser = "x", adminPassword = "Azuresmr_test1",
|
||||
sshUser = "sssUser_test1", sshPassword = "sshUser_test1",
|
||||
debug = TRUE
|
||||
),
|
||||
"list"
|
||||
),
|
||||
"list"
|
||||
)
|
||||
|
||||
# debug - rserver
|
||||
expect_is(
|
||||
azureCreateHDI(asc, resourceGroup = resourceGroup_name, clustername = "azuresmr_hdi_test",
|
||||
storageAccount = "azuresmrhditest",
|
||||
azureCreateHDI(asc, resourceGroup = resourceGroup_name,
|
||||
clustername = cluster_name,
|
||||
storageAccount = storage_name,
|
||||
adminUser = "x", adminPassword = "Azuresmr_test1",
|
||||
sshUser = "sssUser_test1", sshPassword = "sshUser_test1",
|
||||
kind = "rserver",
|
||||
debug = TRUE
|
||||
),
|
||||
"list"
|
||||
),
|
||||
"list"
|
||||
)
|
||||
|
||||
# debug - hadoop
|
||||
expect_is(
|
||||
azureCreateHDI(asc, resourceGroup = resourceGroup_name, clustername = "azuresmr_hdi_test",
|
||||
storageAccount = "azuresmrhditest",
|
||||
azureCreateHDI(asc, resourceGroup = resourceGroup_name,
|
||||
clustername = cluster_name,
|
||||
storageAccount = storage_name,
|
||||
adminUser = "x", adminPassword = "Azuresmr_test1",
|
||||
sshUser = "sssUser_test1", sshPassword = "sshUser_test1",
|
||||
kind = "hadoop",
|
||||
debug = TRUE
|
||||
),
|
||||
"list"
|
||||
),
|
||||
"list"
|
||||
)
|
||||
})
|
||||
|
||||
|
|
@ -109,14 +116,15 @@ test_that("can create HDI cluster", {
|
|||
|
||||
# create the actual instance - rserver
|
||||
expect_true(
|
||||
azureCreateHDI(asc, resourceGroup = resourceGroup_name, clustername = "azuresmrhditest",
|
||||
storageAccount = paste0("azuresmrhdi", timestamp),
|
||||
adminUser = "x", adminPassword = "Azuresmr_test1",
|
||||
sshUser = "sssUser_test1", sshPassword = "sshUser_test1",
|
||||
kind = "rserver",
|
||||
debug = FALSE
|
||||
)
|
||||
azureCreateHDI(asc, resourceGroup = resourceGroup_name,
|
||||
clustername = cluster_name,
|
||||
storageAccount = storage_name,
|
||||
adminUser = "x", adminPassword = "Azuresmr_test1",
|
||||
sshUser = "sssUser_test1", sshPassword = "sshUser_test1",
|
||||
kind = "rserver",
|
||||
debug = FALSE
|
||||
)
|
||||
)
|
||||
})
|
||||
|
||||
|
||||
|
|
@ -128,15 +136,15 @@ test_that("can run action scripts", {
|
|||
# run an action script
|
||||
expect_true(
|
||||
azureRunScriptAction(asc,
|
||||
scriptname = "installPackages",
|
||||
scriptURL = "http://mrsactionscripts.blob.core.windows.net/rpackages-v01/InstallRPackages.sh",
|
||||
workerNode = TRUE, edgeNode = TRUE,
|
||||
parameters = "useCRAN stringr")
|
||||
scriptname = "installPackages",
|
||||
scriptURL = "http://mrsactionscripts.blob.core.windows.net/rpackages-v01/InstallRPackages.sh",
|
||||
workerNode = TRUE, edgeNode = TRUE,
|
||||
parameters = "useCRAN stringr")
|
||||
)
|
||||
|
||||
# retrieve action script history
|
||||
z <- azureScriptActionHistory(asc)
|
||||
expect_is(z, "data.frame")
|
||||
expect_is(z, "azureScriptActionHistory")
|
||||
})
|
||||
|
||||
# --------
|
||||
|
|
@ -148,9 +156,14 @@ test_that("can delete HDI cluster", {
|
|||
z <- azureListHDI(asc)
|
||||
expect_is(z, "data.frame")
|
||||
|
||||
z <- azureListHDI(asc,
|
||||
clustername = cluster_name,
|
||||
resourceGroup = resourceGroup_name)
|
||||
expect_is(z, "data.frame")
|
||||
|
||||
# delete cluster
|
||||
expect_true(
|
||||
azureDeleteHDI(asc, clustername = "azuresmrhditest")
|
||||
azureDeleteHDI(asc, clustername = cluster_name)
|
||||
)
|
||||
|
||||
azureDeleteResourceGroup(asc, resourceGroup = resourceGroup_name)
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче