Merge branch 'master' into erikms/master/test

.gitignore

@@ -34,3 +34,9 @@ vignettes/*.pdf
 
 # Shiny token, see https://shiny.rstudio.com/articles/shinyapps.html
 rsconnect/
+.Rproj.user
+
+#Python Cache files
+__pycache__/
+.cache/
+.idea/

.travis.yml

@@ -0,0 +1,10 @@
+matrix:
+  include:
+    - language: r
+      sudo: required
+      cache: packages
+      before_install:
+        - cd R
+        - ls
+      before_script:
+        - sudo bash "../Travis-CI/installODBC.sh"

Python/requirements.txt

@@ -0,0 +1,6 @@
+pip
+pymssql
+dill
+pkginfo
+requirements-parser
+pandas

Python/sqlmlutils/connectioninfo.py

@@ -46,7 +46,7 @@ class ConnectionInfo:
 
     @property
     def connection_string(self):
-        return "Driver={{driver}};Server={server};Database={database};{auth};".format(
+        return "Driver={driver};Server={server};Database={database};{auth};".format(
             driver=self._driver,
             server=self._server,
             database=self._database,

Python/tests/conftest.py

@@ -0,0 +1,15 @@
+import os
+from sqlmlutils import ConnectionInfo
+
+driver = os.environ['DRIVER'] if 'DRIVER' in os.environ else "SQL Server"
+server = os.environ['SERVER'] if 'SERVER' in os.environ else "localhost"
+database = os.environ['DATABASE'] if 'DATABASE' in os.environ else "AirlineTestDB"
+uid = os.environ['USER'] if 'USER' in os.environ else ""
+pwd = os.environ['PASSWORD'] if 'PASSWORD' in os.environ else ""
+
+
+connection = ConnectionInfo(driver=driver,
+                            server=server,
+                            database=database,
+                            uid=uid,
+                            pwd=pwd)

Python/tests/execute_function_test.py

@@ -9,10 +9,18 @@ import os
 from sqlmlutils import SQLPythonExecutor
 from sqlmlutils import ConnectionInfo
 from pandas import DataFrame
+from conftest import driver, server, database, uid, pwd
+
+connection = ConnectionInfo(driver=driver,
+                            server=server,
+                            database=database,
+                            uid=uid,
+                            pwd=pwd)
 
 current_dir = os.path.dirname(__file__)
 script_dir = os.path.join(current_dir, "scripts")
-connection = ConnectionInfo(server="localhost", database="AirlineTestDB")
+
+print(connection)
 sqlpy = SQLPythonExecutor(connection)
 
 

Python/tests/package_management_file_test.py

@@ -15,7 +15,8 @@ from package_helper_functions import _get_sql_package_table, _get_package_names_
 from sqlmlutils.packagemanagement.scope import Scope
 from sqlmlutils.packagemanagement.pipdownloader import PipDownloader
 
-connection = sqlmlutils.ConnectionInfo(server="localhost", database="AirlineTestDB")
+from conftest import connection
+
 path_to_packages = os.path.join((os.path.dirname(os.path.realpath(__file__))), "scripts", "test_packages")
 _SUCCESS_TOKEN = "SUCCESS"
 

Python/tests/package_management_pypi_test.py

@@ -10,6 +10,7 @@ from package_helper_functions import _get_sql_package_table, _get_package_names_
 import io
 from contextlib import redirect_stdout
 
+from conftest import connection
 
 def _drop_all_ddl_packages(conn):
     pkgs = _get_sql_package_table(conn)
@@ -19,12 +20,6 @@ def _drop_all_ddl_packages(conn):
         except Exception:
             pass
 
-
-server = os.environ.get("SQLPY_TEST_SERVER", "localhost")
-database = os.environ.get("SQLPY_TEST_DB", "AirlineTestDB")
-uid = os.environ.get("SQLPY_TEST_UID", "")
-pwd = os.environ.get("SQLPY_TEST_PWD", "")
-connection = sqlmlutils.ConnectionInfo(server=server, database=database, uid=uid, pwd=pwd)
 pyexecutor = SQLPythonExecutor(connection)
 pkgmanager = SQLPackageManager(connection)
 _drop_all_ddl_packages(connection)

Python/tests/stored_procedure_test.py

@@ -9,10 +9,11 @@ from pandas import DataFrame
 import io
 import os
 
+from conftest import connection
+
 current_dir = os.path.dirname(__file__)
 script_dir = os.path.join(current_dir, "scripts")
-conn = sqlmlutils.ConnectionInfo(database="AirlineTestDB")
-sqlpy = sqlmlutils.SQLPythonExecutor(conn)
+sqlpy = sqlmlutils.SQLPythonExecutor(connection)
 
 
 ###################
@@ -262,7 +263,7 @@ def test_in_param_out_param():
     # Out params don't currently work so we use sqlcmd to test the output param sproc
     sql_str = "DECLARE @res nvarchar(max)  EXEC test_in_param_out_param @t2 = 213, @t1 = N'Hello', " \
               "@t3 = N'select top 10 * from airline5000', @res = @res OUTPUT SELECT @res as N'@res'"
-    p = Popen(["sqlcmd", "-S", conn.server, "-E", "-d", conn.database, "-Q", sql_str],
+    p = Popen(["sqlcmd", "-S", connection.server, "-E", "-d", connection.database, "-Q", sql_str],
               shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT)
     output = p.stdout.read()
     assert "Hello Hello" in output.decode()
@@ -375,7 +376,7 @@ def test_script_out_param():
     # Out params don't currently work so we use sqlcmd to test the output param sproc
     sql_str = "DECLARE @res nvarchar(max)  EXEC test_script_out_param @t2 = 123, @t1 = N'Hello', " \
               "@t3 = N'select top 10 * from airline5000', @res = @res OUTPUT SELECT @res as N'@res'"
-    p = Popen(["sqlcmd", "-S", conn.server, "-E", "-d", conn.database, "-Q", sql_str],
+    p = Popen(["sqlcmd", "-S", connection.server, "-E", "-d", connection.database, "-Q", sql_str],
               shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT)
     output = p.stdout.read()
     assert "Hello123" in output.decode()

R/.Rbuildignore

@@ -0,0 +1,2 @@
+^.*\.Rproj$
+^\.Rproj\.user$

R/DESCRIPTION

@@ -14,6 +14,6 @@ Description: Provides a set of functions allowing the user
     development environment.
 License: MIT + file LICENSE
 Copyright: Copyright 2016 Microsoft Corporation
-RoxygenNote: 6.0.1
+RoxygenNote: 6.1.0
 Suggests: testthat,
     roxygen2

R/R/sqlPackage.R

@@ -142,7 +142,7 @@ sql_remove.packages <- function(connectionString, pkgs, dependencies = TRUE, che
     }
 
     if(verbose){
-        write(sprintf("%s  starting package removal on SQL server (%s)...", pkgTime(), connectionString), stdout())
+        write(sprintf("%s  Starting package removal on SQL server (%s)...", pkgTime(), connectionString), stdout())
     } else {
         write(sprintf("(package removal may take a few minutes, set verbose=TRUE for progress report)"), stdout())
     }
@@ -389,7 +389,7 @@ sqlRemoteExecuteFun <- function(connection, FUN, ..., useRemoteFun = FALSE, asus
                              argList <- as.list(unserialize(binArgList))
                              result <- do.call(%s, argList)
                              }, error = function(err) {
-                             funerror <<- err
+                             funerror <<- err$message
                              }, warning = function(warn) {
                              funwarnings <<- c(funwarnings, warn$message)
                              }
@@ -1682,7 +1682,7 @@ sqlQueryExternalLibrarySetupErrors <- function(hodbc, externalLibraryIds, queryU
                     " ) ELSE (SELECT 'OBJECT_NOT_FOUND' AS OBJECT_NOT_FOUND);"
     )
 
-    sqlResult <- sqlExecute(hodbc, query = query, fetch = TRUE, error = TRUE, stringsAsFactors = FALSE)
+    sqlResult <- sqlExecute(hodbc, query = query, fetch = TRUE, errors = TRUE, stringsAsFactors = FALSE)
 
     if (is.data.frame(sqlResult))
     {

R/buildandinstall.cmd

@@ -1,5 +1,6 @@
 pushd .
 cd ..
+R -e "install.packages('RODBCext', repos='https://cran.rstudio.com')"
 R CMD INSTALL --build R
 mv sqlmlutils_0.5.0.zip R/dist
 popd

R/man/sql_install.packages.Rd

@@ -4,9 +4,8 @@
 \alias{sql_install.packages}
 \title{sql_install.packages}
 \usage{
-sql_install.packages(connectionString, pkgs, skipMissing = FALSE,
-  repos = getOption("repos"), verbose = getOption("verbose"),
-  scope = "private", owner = "")
+sql_install.packages(connectionString, pkgs, skipMissing = FALSE, repos,
+  verbose = getOption("verbose"), scope = "private", owner = "")
 }
 \arguments{
 \item{connectionString}{ODBC connection string to Microsoft SQL Server database.}

R/man/sql_installed.packages.Rd

@@ -4,8 +4,9 @@
 \alias{sql_installed.packages}
 \title{sql_installed.packages}
 \usage{
-sql_installed.packages(connectionString, priority = NULL, noCache = FALSE,
-  fields = "Package", subarch = NULL, scope = "private", owner = "")
+sql_installed.packages(connectionString, priority = NULL,
+  noCache = FALSE, fields = "Package", subarch = NULL,
+  scope = "private", owner = "")
 }
 \arguments{
 \item{connectionString}{ODBC connection string to Microsoft SQL Server database.}
@@ -14,9 +15,7 @@ sql_installed.packages(connectionString, priority = NULL, noCache = FALSE,
 
 \item{noCache}{logical. If TRUE, do not use cached information, nor cache it.}
 
-\item{fields}{a character vector giving the fields to extract from each package's DESCRIPTION file, or NULL. If NULL, the following fields are used:
-"Package", "LibPath", "Version", "Priority", "Depends", "Imports", "LinkingTo", "Suggests", "Enhances", "License", "License_is_FOSS", "License_restricts_use", "OS_type", "MD5sum", "NeedsCompilation", and "Built".
-Unavailable fields result in NA values.}
+\item{fields}{a character vector giving the fields to extract from each package's DESCRIPTION file, or NULL. If NULL, the following fields are used: "Package", "LibPath", "Version", "Priority", "Depends", "Imports", "LinkingTo", "Suggests", "Enhances", "License", "License_is_FOSS", "License_restricts_use", "OS_type", "MD5sum", "NeedsCompilation", and "Built". Unavailable fields result in NA values.}
 
 \item{subarch}{character string or NULL. If non-null and non-empty, used to select packages which are installed for that sub-architecture}
 

R/man/sql_remove.packages.Rd

@@ -27,7 +27,7 @@ sql_remove.packages(connectionString, pkgs, dependencies = TRUE,
 invisible(NULL)
 }
 \description{
-sql_remove.packages
+Removes R packages from a SQL Server database.
 }
 \seealso{
 {

R/tests/testthat/helper-Setup.R

@@ -7,9 +7,25 @@ library(testthat)
 
 options(keep.source = TRUE)
 Sys.setenv(TZ='GMT')
-Server <- ''
+
+Driver <- Sys.getenv("DRIVER")
+if (Driver == '') Driver <- "SQL Server"
+
+Server <- Sys.getenv("SERVER")
 if (Server == '') Server <- "."
-cnnstr <- connectionInfo(server=Server, database="AirlineTestDB")
+
+Database <- Sys.getenv("DATABASE")
+if (Database == '') Database <- "AirlineTestDB"
+
+Uid <- Sys.getenv("USER")
+Pwd <- Sys.getenv("PASSWORD")
+if(Uid == '') Uid = NULL
+if(Pwd == '') Pwd = NULL
+
+sqlcmd_path <- Sys.getenv("SQLCMD")
+if (sqlcmd_path == '') sqlcmd_path <- "sqlcmd"
+
+cnnstr <- connectionInfo(driver=Driver, server=Server, database=Database, uid=Uid, pwd=Pwd)
 
 testthatDir <- getwd()
 R_Root <- file.path(testthatDir, "../..")
@@ -20,7 +36,13 @@ TestArgs <- list(
     gitRoot = R_Root,
     testDirectory = testthatDir,
     scriptDirectory = scriptDirectory,
-    connectionString = cnnstr
+    driver=Driver,
+    server=Server,
+    database=Database,
+    uid=Uid, 
+    pwd=Pwd,
+    connectionString = cnnstr,
+    sqlcmd = sqlcmd_path
 )
 
 options(TestArgs = TestArgs)

R/tests/testthat/test.executeInSqlTests.R

@@ -8,8 +8,6 @@ TestArgs <- options("TestArgs")$TestArgs
 connection <- TestArgs$connectionString
 scriptDir <- TestArgs$scriptDirectory
 
-
-
 test_that("Test with named args", {
     funcWithArgs <- function(arg1, arg2){
         print(arg1)
@@ -110,6 +108,7 @@ test_that("Print, Warning, Return test", {
     expect_warning(expect_output(result <- executeFunctionInSQL(connection, returnString), "hello"), "uh oh")
 
     expect_equal(result , "bar")
+    
 })
 
 test_that("Print, Warning, Return test, with args", {

R/tests/testthat/test.storedProcedureTests.R

@@ -7,6 +7,7 @@ context("Stored Procedure tests")
 TestArgs <- options('TestArgs')$TestArgs
 connection <- TestArgs$connectionString
 scriptDir <- TestArgs$scriptDirectory
+sqlcmd_path <- TestArgs$sqlcmd
 
 dropIfExists <- function(connectionString, name) {
     if(checkSproc(connectionString, name))
@@ -218,7 +219,20 @@ test_that("Only OuputParams test", {
 
     #Use T-SQL to verify
     sql_str = "DECLARE @res nvarchar(max)  EXEC outsFunc @arg1_outer = N'T-SQL', @res_outer = @res OUTPUT SELECT @res as N'@res'"
-    out <- system2("sqlcmd.exe", c("-S", "localhost", "-E", "-d","AirlineTestDB", "-Q", paste0('"', sql_str, '"')), stdout=TRUE)
+    if(TestArgs$uid != "") {
+        out <- system2(sqlcmd_path, c(  "-S", TestArgs$server,
+                                        "-d", TestArgs$database, 
+                                        "-Q", paste0('"', sql_str, '"'),
+                                        "-U", TestArgs$uid,
+                                        "-P", TestArgs$pwd), 
+                                        stdout=TRUE)
+    } else {
+        out <- system2(sqlcmd_path, c(  "-S", TestArgs$server,
+                                        "-d", TestArgs$database, 
+                                        "-Q", paste0('"', sql_str, '"'),
+                                        "-E"),
+                                        stdout=TRUE)
+    }
     expect_true(any(grepl("Hello T-SQL!", out)))
     #executeSproc(name, connectionString = connection, out1 = "Asd", out2 = "wqe")
 
@@ -233,20 +247,36 @@ test_that("OutputDataSet and OuputParams test", {
         list(df = df, out1 = "Hello", out2 = "World")
     }
     name = "outDataParam"
+    
+    dropIfExists(name, connectionString = connection)
+    expect_false(checkSproc(name, connectionString = connection))
 
     outputParams <- list(df = "dataframe", out2 = "character", out1 = "character")
 
     createSprocFromFunction(name, outDataParam, connectionString = connection, outputParams = outputParams)
-    stopifnot(checkSproc(name, connectionString = connection))
+    expect_true(checkSproc(name, connectionString = connection))
 
     #Use T-SQL to verify
     sql_str = "DECLARE @out1 nvarchar(max),@out2 nvarchar(max)  EXEC outDataParam @out1_outer = @out1 OUTPUT, @out2_outer = @out2 OUTPUT SELECT @out1 as N'@out1'"
-    out <- system2("sqlcmd.exe", c("-S", "localhost", "-E", "-d","AirlineTestDB", "-Q", paste0('"', sql_str, '"')), stdout=TRUE)
+    if(TestArgs$uid != "") {
+        out <- system2(sqlcmd_path, c(  "-S", TestArgs$server,
+                                        "-d", TestArgs$database, 
+                                        "-Q", paste0('"', sql_str, '"'),
+                                        "-U", TestArgs$uid,
+                                        "-P", TestArgs$pwd), 
+                                        stdout=TRUE)
+    } else {
+        out <- system2(sqlcmd_path, c(  "-S", TestArgs$server,
+                                        "-d", TestArgs$database, 
+                                        "-Q", paste0('"', sql_str, '"'),
+                                        "-E"),
+                                        stdout=TRUE)
+    }
     expect_true(any(grepl("Hello", out)))
     #res <- executeSproc(connectionString = connection, name)
 
-    dropSproc(name, connectionString = connection)
-    stopifnot(!checkSproc(name, connectionString = connection))
+    dropIfExists(name, connectionString = connection)
+    expect_false(checkSproc(name, connectionString = connection))
 })
 
 context("Sproc Negative Tests")

Travis-CI/installODBC.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+sudo curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add -
+
+#Download appropriate package for the OS version
+#Choose only ONE of the following, corresponding to your OS version
+
+#Ubuntu 14.04
+sudo curl https://packages.microsoft.com/config/ubuntu/14.04/prod.list > /etc/apt/sources.list.d/mssql-release.list
+
+sudo apt-get update
+sudo ACCEPT_EULA=Y apt-get install msodbcsql17
+
+# optional: for bcp and sqlcmd
+sudo ACCEPT_EULA=Y apt-get install mssql-tools