Add external language support (#82)
Add external language support - users can add a language_name to run on an already installed external language.
This commit is contained in:
Jonathan Zhu
GitHub
Родитель
2070d75918
Коммит
c6ff0841aa
|
|
@ -1,3 +1,4 @@
|
|||
del /q dist\*
|
||||
python.exe setup.py sdist --formats=zip
|
||||
python.exe -m pip install --upgrade --upgrade-strategy only-if-needed --find-links=dist sqlmlutils
|
||||
python.exe setup.py bdist_wheel
|
||||
python.exe -m pip install --upgrade --upgrade-strategy only-if-needed --find-links=dist sqlmlutils
|
||||
|
|
|
|||
Двоичный файл не отображается.
Двоичный файл не отображается.
|
|
@ -6,7 +6,7 @@ from setuptools import setup
|
|||
setup(
|
||||
name='sqlmlutils',
|
||||
packages=['sqlmlutils', 'sqlmlutils/packagemanagement'],
|
||||
version='1.0.3',
|
||||
version='1.1.0',
|
||||
url='https://github.com/Microsoft/sqlmlutils/Python',
|
||||
license='MIT License',
|
||||
description='A client side package for working with SQL Server',
|
||||
|
|
|
|||
|
|
@ -9,8 +9,9 @@ from sqlmlutils.packagemanagement.scope import Scope
|
|||
|
||||
class CreateLibraryBuilder(SQLBuilder):
|
||||
|
||||
def __init__(self, pkg_name: str, pkg_filename: str, scope: Scope):
|
||||
def __init__(self, pkg_name: str, pkg_filename: str, scope: Scope, language_name: str):
|
||||
self._name = clean_library_name(pkg_name)
|
||||
self._language_name = language_name
|
||||
self._filename = pkg_filename
|
||||
self._scope = scope
|
||||
|
||||
|
|
@ -23,9 +24,8 @@ class CreateLibraryBuilder(SQLBuilder):
|
|||
|
||||
@property
|
||||
def base_script(self) -> str:
|
||||
sqlpkgname = self._name
|
||||
authorization = _get_authorization(self._scope)
|
||||
dummy_spees = _get_dummy_spees()
|
||||
dummy_spees = _get_dummy_spees(self._language_name)
|
||||
|
||||
return """
|
||||
set NOCOUNT on
|
||||
|
|
@ -38,30 +38,39 @@ END CATCH
|
|||
|
||||
-- Create the library
|
||||
CREATE EXTERNAL LIBRARY [{sqlpkgname}] {authorization}
|
||||
FROM (CONTENT = ?) WITH (LANGUAGE = 'Python');
|
||||
FROM (CONTENT = ?) WITH (LANGUAGE = '{language_name}');
|
||||
|
||||
-- Dummy SPEES
|
||||
{dummy_spees}
|
||||
""".format(
|
||||
sqlpkgname=sqlpkgname,
|
||||
sqlpkgname=self._name,
|
||||
authorization=authorization,
|
||||
dummy_spees=dummy_spees
|
||||
dummy_spees=dummy_spees,
|
||||
language_name=self._language_name
|
||||
)
|
||||
|
||||
|
||||
class CheckLibraryBuilder(SQLBuilder):
|
||||
|
||||
def __init__(self, pkg_name: str, scope: Scope):
|
||||
def __init__(self, pkg_name: str, scope: Scope, language_name: str):
|
||||
self._name = clean_library_name(pkg_name)
|
||||
self._language_name = language_name
|
||||
self._scope = scope
|
||||
|
||||
if self._language_name == "Python":
|
||||
self._private_path_env = "MRS_EXTLIB_USER_PATH"
|
||||
self._public_path_env = "MRS_EXTLIB_SHARED_PATH"
|
||||
else:
|
||||
self._private_path_env = "PRIVATELIBPATH"
|
||||
self._public_path_env = "PUBLICLIBPATH"
|
||||
|
||||
@property
|
||||
def params(self):
|
||||
return """
|
||||
import os
|
||||
import re
|
||||
_ENV_NAME_USER_PATH = "MRS_EXTLIB_USER_PATH"
|
||||
_ENV_NAME_SHARED_PATH = "MRS_EXTLIB_SHARED_PATH"
|
||||
_ENV_NAME_USER_PATH = "{private_path_env}"
|
||||
_ENV_NAME_SHARED_PATH = "{public_path_env}"
|
||||
|
||||
def _is_dist_info_file(name, file):
|
||||
return re.match(name + r"-.*egg", file) or re.match(name + r"-.*dist-info", file)
|
||||
|
|
@ -92,15 +101,18 @@ def package_exists_in_scope(sql_package_name: str, scope=None) -> bool:
|
|||
# Check that the package exists in scope.
|
||||
# For some reason this check works but there is a bug in pyODBC when asserting this is True.
|
||||
assert package_exists_in_scope("{name}", "{scope}") != False
|
||||
""".format(name=self._name, scope=self._scope._name)
|
||||
""".format(private_path_env=self._private_path_env,
|
||||
public_path_env=self._public_path_env,
|
||||
name=self._name,
|
||||
scope=self._scope._name)
|
||||
|
||||
@property
|
||||
def base_script(self) -> str:
|
||||
return """
|
||||
-- Check to make sure the package was installed
|
||||
BEGIN TRY
|
||||
exec sp_execute_external_script
|
||||
@language = N'Python',
|
||||
EXEC sp_execute_external_script
|
||||
@language = N'{language_name}',
|
||||
@script = ?
|
||||
print('Package successfully installed.')
|
||||
END TRY
|
||||
|
|
@ -108,13 +120,14 @@ BEGIN CATCH
|
|||
print('Package installation failed.');
|
||||
THROW;
|
||||
END CATCH
|
||||
"""
|
||||
""".format(language_name = self._language_name)
|
||||
|
||||
|
||||
class DropLibraryBuilder(SQLBuilder):
|
||||
|
||||
def __init__(self, sql_package_name: str, scope: Scope):
|
||||
def __init__(self, sql_package_name: str, scope: Scope, language_name: str):
|
||||
self._name = clean_library_name(sql_package_name)
|
||||
self._language_name = language_name
|
||||
self._scope = scope
|
||||
|
||||
@property
|
||||
|
|
@ -126,10 +139,9 @@ DROP EXTERNAL LIBRARY [{name}] {auth}
|
|||
""".format(
|
||||
name=self._name,
|
||||
auth=_get_authorization(self._scope),
|
||||
dummy_spees=_get_dummy_spees()
|
||||
dummy_spees=_get_dummy_spees(self._language_name)
|
||||
)
|
||||
|
||||
|
||||
def clean_library_name(pkgname: str):
|
||||
return pkgname.replace("-", "_").lower()
|
||||
|
||||
|
|
@ -138,9 +150,9 @@ def _get_authorization(scope: Scope) -> str:
|
|||
return "AUTHORIZATION dbo" if scope == Scope.public_scope() else ""
|
||||
|
||||
|
||||
def _get_dummy_spees() -> str:
|
||||
def _get_dummy_spees(language_name: str) -> str:
|
||||
return """
|
||||
exec sp_execute_external_script
|
||||
@language = N'Python',
|
||||
EXEC sp_execute_external_script
|
||||
@language = N'{language_name}',
|
||||
@script = N''
|
||||
"""
|
||||
""".format(language_name = language_name)
|
||||
|
|
|
|||
|
|
@ -12,11 +12,12 @@ from sqlmlutils.packagemanagement import servermethods
|
|||
|
||||
class PipDownloader:
|
||||
|
||||
def __init__(self, connection: ConnectionInfo, downloaddir: str, targetpackage: str):
|
||||
def __init__(self, connection: ConnectionInfo, downloaddir: str, targetpackage: str, language_name: str):
|
||||
self._connection = connection
|
||||
self._downloaddir = downloaddir
|
||||
self._targetpackage = targetpackage
|
||||
server_info = SQLPythonExecutor(connection).execute_function_in_sql(servermethods.get_server_info)
|
||||
self._language_name = language_name
|
||||
server_info = SQLPythonExecutor(connection, self._language_name).execute_function_in_sql(servermethods.get_server_info)
|
||||
globals().update(server_info)
|
||||
|
||||
def download(self):
|
||||
|
|
|
|||
|
|
@ -6,9 +6,6 @@ import re
|
|||
|
||||
from sqlmlutils.packagemanagement.scope import Scope
|
||||
|
||||
_ENV_NAME_USER_PATH = "MRS_EXTLIB_USER_PATH"
|
||||
_ENV_NAME_SHARED_PATH = "MRS_EXTLIB_SHARED_PATH"
|
||||
|
||||
def show_installed_packages():
|
||||
import pkg_resources
|
||||
return [(d.project_name, d.version) for d in pkg_resources.working_set]
|
||||
|
|
@ -30,40 +27,3 @@ def get_server_info():
|
|||
"abi_tag": pep425tags.get_abi_tag(), #'cp37m'
|
||||
"platform": sysconfig.get_platform().replace("-","_") #'win_amd64', 'linux_x86_64'
|
||||
}
|
||||
|
||||
|
||||
def check_package_install_success(sql_package_name: str) -> bool:
|
||||
return package_exists_in_scope(sql_package_name)
|
||||
|
||||
|
||||
def package_files_in_scope(scope=Scope.private_scope()):
|
||||
envdir = _ENV_NAME_SHARED_PATH if scope == Scope.public_scope() or os.environ.get(_ENV_NAME_USER_PATH, "") == "" \
|
||||
else _ENV_NAME_USER_PATH
|
||||
path = os.environ.get(envdir, "")
|
||||
if os.path.isdir(path):
|
||||
return os.listdir(path)
|
||||
return []
|
||||
|
||||
|
||||
def package_exists_in_scope(sql_package_name: str, scope=None) -> bool:
|
||||
if scope is None:
|
||||
# default to user path for every user but DBOs
|
||||
scope = Scope.public_scope() if (os.environ.get(_ENV_NAME_USER_PATH, "") == "") else Scope.private_scope()
|
||||
package_files = package_files_in_scope(scope)
|
||||
return any([_is_package_match(sql_package_name, package_file) for package_file in package_files])
|
||||
|
||||
|
||||
def _is_dist_info_file(name, file):
|
||||
return re.match(name + r'-.*egg', file) or re.match(name + r'-.*dist-info', file)
|
||||
|
||||
|
||||
def _is_package_match(package_name, file):
|
||||
package_name = package_name.lower()
|
||||
file = file.lower()
|
||||
return file == package_name or file == package_name + ".py" or \
|
||||
_is_dist_info_file(package_name, file) or \
|
||||
("-" in package_name and
|
||||
(package_name.split("-")[0] == file or _is_dist_info_file(package_name.replace("-", "_"), file)))
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -19,9 +19,15 @@ from sqlmlutils.sqlqueryexecutor import execute_query, SQLQueryExecutor
|
|||
|
||||
class SQLPackageManager:
|
||||
|
||||
def __init__(self, connection_info: ConnectionInfo):
|
||||
def __init__(self, connection_info: ConnectionInfo, language_name: str = "Python"):
|
||||
"""Initialize a SQLPackageManager to manage packages on the SQL Server.
|
||||
|
||||
:param connection_info: The ConnectionInfo object that holds the connection string and other information.
|
||||
:param language_name: The name of the language to be executed in sp_execute_external_script, if using EXTERNAL LANGUAGE.
|
||||
"""
|
||||
self._connection_info = connection_info
|
||||
self._pyexecutor = SQLPythonExecutor(connection_info)
|
||||
self._pyexecutor = SQLPythonExecutor(connection_info, language_name=language_name)
|
||||
self._language_name = language_name
|
||||
|
||||
def install(self,
|
||||
package: str,
|
||||
|
|
@ -104,29 +110,32 @@ class SQLPackageManager:
|
|||
return Scope.public_scope() if is_sysadmin == 1 else Scope.private_scope()
|
||||
|
||||
def _get_packages_by_user(self, owner='', scope: Scope=Scope.private_scope()):
|
||||
has_user = (owner != '')
|
||||
scope_num = 1 if scope == Scope.private_scope() else 0
|
||||
|
||||
if scope_num == 0 and owner == '':
|
||||
owner = "dbo"
|
||||
|
||||
query = "DECLARE @principalId INT; \
|
||||
DECLARE @currentUser NVARCHAR(128); \
|
||||
SELECT @currentUser = "
|
||||
|
||||
if has_user:
|
||||
if owner != '':
|
||||
query += "?;\n"
|
||||
else:
|
||||
query += "CURRENT_USER;\n"
|
||||
|
||||
scope_num = 1 if scope == Scope.private_scope() else 0
|
||||
|
||||
query += "SELECT @principalId = USER_ID(@currentUser); \
|
||||
SELECT name, language, scope \
|
||||
FROM sys.external_libraries AS elib \
|
||||
WHERE elib.principal_id=@principalId \
|
||||
AND elib.language='Python' AND elib.scope={scope_num} \
|
||||
ORDER BY elib.name ASC;".format(scope_num=scope_num)
|
||||
AND elib.language='{language_name}' AND elib.scope={scope_num} \
|
||||
ORDER BY elib.name ASC; \
|
||||
GO".format(language_name=self._language_name,
|
||||
scope_num=scope_num)
|
||||
return self._pyexecutor.execute_sql_query(query, owner)
|
||||
|
||||
def _drop_sql_package(self, sql_package_name: str, scope: Scope, out_file: str = None):
|
||||
builder = DropLibraryBuilder(sql_package_name=sql_package_name, scope=scope)
|
||||
builder = DropLibraryBuilder(sql_package_name=sql_package_name, scope=scope, language_name=self._language_name)
|
||||
execute_query(builder, self._connection_info, out_file)
|
||||
|
||||
# TODO: Support not dependencies
|
||||
|
|
@ -146,7 +155,7 @@ class SQLPackageManager:
|
|||
target_package = target_package + "==" + version
|
||||
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
pipdownloader = PipDownloader(self._connection_info, temporary_directory, target_package)
|
||||
pipdownloader = PipDownloader(self._connection_info, temporary_directory, target_package, language_name = self._language_name)
|
||||
target_package_file = pipdownloader.download_single()
|
||||
self._install_from_file(target_package_file, scope, upgrade, out_file=out_file)
|
||||
|
||||
|
|
@ -162,7 +171,7 @@ class SQLPackageManager:
|
|||
|
||||
# Download requirements from PyPI
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
pipdownloader = PipDownloader(self._connection_info, temporary_directory, target_package_file)
|
||||
pipdownloader = PipDownloader(self._connection_info, temporary_directory, target_package_file, language_name = self._language_name)
|
||||
|
||||
# For now, we download all target package dependencies from PyPI.
|
||||
target_package_requirements, requirements_downloaded = pipdownloader.download()
|
||||
|
|
@ -189,8 +198,7 @@ class SQLPackageManager:
|
|||
sqlexecutor._cnxn.rollback()
|
||||
raise RuntimeError("Package installation failed, installed dependencies were rolled back.") from e
|
||||
|
||||
@staticmethod
|
||||
def _install_single(sqlexecutor: SQLQueryExecutor, package_file: str, scope: Scope, is_target=False, out_file: str=None):
|
||||
def _install_single(self, sqlexecutor: SQLQueryExecutor, package_file: str, scope: Scope, is_target=False, out_file: str=None):
|
||||
name = str(get_package_name_from_file(package_file))
|
||||
version = str(get_package_version_from_file(package_file))
|
||||
print("Installing {name} version: {version}".format(name=name, version=version))
|
||||
|
|
@ -200,9 +208,10 @@ class SQLPackageManager:
|
|||
with zipfile.ZipFile(prezip, 'w') as zipf:
|
||||
zipf.write(package_file, os.path.basename(package_file))
|
||||
|
||||
builder = CreateLibraryBuilder(pkg_name=name, pkg_filename=prezip, scope=scope)
|
||||
builder = CreateLibraryBuilder(pkg_name=name, pkg_filename=prezip, scope=scope, language_name=self._language_name)
|
||||
sqlexecutor.execute(builder, out_file=out_file)
|
||||
builder = CheckLibraryBuilder(pkg_name=name, scope=scope)
|
||||
|
||||
builder = CheckLibraryBuilder(pkg_name=name, scope=scope, language_name=self._language_name)
|
||||
sqlexecutor.execute(builder, out_file=out_file)
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -50,30 +50,34 @@ class SpeesBuilder(SQLBuilder):
|
|||
script: str,
|
||||
with_results_text: str = _WITH_RESULTS_TEXT,
|
||||
input_data_query: str = "",
|
||||
script_parameters_text: str = ""):
|
||||
script_parameters_text: str = "",
|
||||
language_name: str = "Python"):
|
||||
"""Instantiate a _SpeesBuilder object.
|
||||
|
||||
:param script: maps to @script parameter in the SQL query parameter
|
||||
:param with_results_text: with results text used to defined the expected data schema of the SQL query
|
||||
:param input_data_query: maps to @input_data_1 SQL query parameter
|
||||
:param script_parameters_text: maps to @params SQL query parameter
|
||||
:param language_name: name of the language to be executed in sp_execute_external_script, if using EXTERNAL LANGUAGE
|
||||
"""
|
||||
self._script = self.modify_script(script)
|
||||
self._input_data_query = input_data_query
|
||||
self._script_parameters_text = script_parameters_text
|
||||
self._with_results_text = with_results_text
|
||||
self._language_name = language_name
|
||||
|
||||
@property
|
||||
def base_script(self):
|
||||
return """
|
||||
exec sp_execute_external_script
|
||||
@language = N'Python',
|
||||
EXEC sp_execute_external_script
|
||||
@language = N'{language_name}',
|
||||
@script = ?,
|
||||
@input_data_1 = ?
|
||||
{script_parameters_text}
|
||||
{with_results_text}
|
||||
""".format(script_parameters_text=self._script_parameters_text,
|
||||
with_results_text=self._with_results_text)
|
||||
with_results_text=self._with_results_text,
|
||||
language_name=self._language_name)
|
||||
|
||||
@property
|
||||
def params(self):
|
||||
|
|
@ -112,12 +116,13 @@ class SpeesBuilderFromFunction(SpeesBuilder):
|
|||
stderr=STDERR_COLUMN_NAME
|
||||
)
|
||||
|
||||
def __init__(self, func: Callable, input_data_query: str = "", *args, **kwargs):
|
||||
def __init__(self, func: Callable, language_name: str, input_data_query: str = "", *args, **kwargs):
|
||||
"""Instantiate a _SpeesBuilderFromFunction object.
|
||||
|
||||
:param func: function to execute_function_in_sql on the SQL Server.
|
||||
The spees query is built based on this function.
|
||||
:param input_data_query: query text for @input_data_1 parameter
|
||||
:param language_name: name of the language to be executed in sp_execute_external_script, if using EXTERNAL LANGUAGE
|
||||
:param args: positional arguments to function call in SPEES
|
||||
:param kwargs: keyword arguments to function call in SPEES
|
||||
"""
|
||||
|
|
@ -125,7 +130,8 @@ class SpeesBuilderFromFunction(SpeesBuilder):
|
|||
self._function_text = self._build_wrapper_python_script(func, with_inputdf, *args, **kwargs)
|
||||
super().__init__(script=self._function_text,
|
||||
with_results_text=self._WITH_RESULTS_TEXT,
|
||||
input_data_query=input_data_query)
|
||||
input_data_query=input_data_query,
|
||||
language_name=language_name)
|
||||
|
||||
# Generates a Python script that encapsulates a user defined function and the arguments to that function.
|
||||
# This script is "shipped" over the SQL Server machine.
|
||||
|
|
@ -185,7 +191,12 @@ OutputDataSet["{returncol}"] = [dill.dumps({returncol}).hex()]
|
|||
|
||||
class StoredProcedureBuilder(SQLBuilder):
|
||||
|
||||
def __init__(self, name: str, script: str, input_params: dict = None, output_params: dict = None):
|
||||
def __init__(self,
|
||||
name: str,
|
||||
script: str,
|
||||
input_params: dict = None,
|
||||
output_params: dict = None,
|
||||
language_name: str = "Python"):
|
||||
|
||||
"""StoredProcedureBuilder SQL stored procedures based on Python functions.
|
||||
|
||||
|
|
@ -193,6 +204,7 @@ class StoredProcedureBuilder(SQLBuilder):
|
|||
:param script: function to base the stored procedure on
|
||||
:param input_params: input parameters type annotation dictionary for the stored procedure
|
||||
:param output_params: output parameters type annotation dictionary from the stored procedure
|
||||
:param language_name: name of the language to be executed in sp_execute_external_script, if using EXTERNAL LANGUAGE
|
||||
"""
|
||||
if input_params is None:
|
||||
input_params = {}
|
||||
|
|
@ -206,6 +218,7 @@ class StoredProcedureBuilder(SQLBuilder):
|
|||
self._name = name
|
||||
self._input_params = input_params
|
||||
self._output_params = output_params
|
||||
self._language_name = language_name
|
||||
self._param_declarations = ""
|
||||
|
||||
names_of_input_args = list(self._input_params)
|
||||
|
|
@ -228,7 +241,7 @@ CREATE PROCEDURE {name}
|
|||
AS
|
||||
SET NOCOUNT ON;
|
||||
EXEC sp_execute_external_script
|
||||
@language = N'Python',
|
||||
@language = N'{language_name}',
|
||||
@script = N'
|
||||
from io import StringIO
|
||||
import sys
|
||||
|
|
@ -243,13 +256,19 @@ sys.stderr = _stderr
|
|||
""".format(
|
||||
name=self._name,
|
||||
param_declarations=self._param_declarations,
|
||||
language_name=self._language_name,
|
||||
script=self._script,
|
||||
stdout=STDOUT_COLUMN_NAME,
|
||||
stderr=STDERR_COLUMN_NAME,
|
||||
script_parameter_text=self._script_parameter_text
|
||||
)
|
||||
|
||||
def script_parameter_text(self, in_names: List[str], in_types: dict, out_names: List[str], out_types: dict) -> str:
|
||||
def script_parameter_text(self,
|
||||
in_names: List[str],
|
||||
in_types: dict,
|
||||
out_names: List[str],
|
||||
out_types: dict) -> str:
|
||||
|
||||
if not in_names and not out_names:
|
||||
return ""
|
||||
|
||||
|
|
@ -362,7 +381,7 @@ class StoredProcedureBuilderFromFunction(StoredProcedureBuilder):
|
|||
|
||||
create procedure MyStoredProcedure @arg1 varchar(MAX), @arg2 varchar(MAX), @arg3 varchar(MAX) as
|
||||
|
||||
exec sp_execute_external_script
|
||||
EXEC sp_execute_external_script
|
||||
@language = N'Python',
|
||||
@script=N'
|
||||
def foobar(arg1, arg2, arg3):
|
||||
|
|
@ -375,15 +394,19 @@ class StoredProcedureBuilderFromFunction(StoredProcedureBuilder):
|
|||
@arg3 = @arg3
|
||||
"""
|
||||
|
||||
def __init__(self, name: str, func: Callable,
|
||||
input_params: dict = None, output_params: dict = None):
|
||||
def __init__(self,
|
||||
name: str, func: Callable,
|
||||
input_params: dict = None,
|
||||
output_params: dict = None,
|
||||
language_name: str = "Python"):
|
||||
"""StoredProcedureBuilderFromFunction SQL stored procedures based on Python functions.
|
||||
|
||||
:param name: name of the stored procedure
|
||||
:param func: function to base the stored procedure on
|
||||
:param input_params: input parameters type annotation dictionary for the stored procedure
|
||||
Can you function type annotations instead; if both, they must match
|
||||
Can use function type annotations instead; if both, they must match
|
||||
:param output_params: output parameters type annotation dictionary from the stored procedure
|
||||
:param language_name: name of the language to be executed in sp_execute_external_script, if using EXTERNAL LANGUAGE
|
||||
"""
|
||||
if input_params is None:
|
||||
input_params = {}
|
||||
|
|
@ -396,6 +419,7 @@ class StoredProcedureBuilderFromFunction(StoredProcedureBuilder):
|
|||
self._func = func
|
||||
self._name = name
|
||||
self._output_params = output_params
|
||||
self._language_name = language_name
|
||||
|
||||
# Get function text and escape single quotes
|
||||
function_text = textwrap.dedent(inspect.getsource(self._func)).replace("'","''")
|
||||
|
|
|
|||
|
|
@ -17,8 +17,14 @@ from .sqlbuilder import RETURN_COLUMN_NAME, STDOUT_COLUMN_NAME, STDERR_COLUMN_NA
|
|||
|
||||
class SQLPythonExecutor:
|
||||
|
||||
def __init__(self, connection_info: ConnectionInfo):
|
||||
def __init__(self, connection_info: ConnectionInfo, language_name: str = "Python"):
|
||||
"""Initialize a PythonExecutor to execute functions or queries in SQL Server.
|
||||
|
||||
:param connection_info: The ConnectionInfo object that holds the connection string and other information.
|
||||
:param language_name: The name of the language to be executed in sp_execute_external_script, if using EXTERNAL LANGUAGE.
|
||||
"""
|
||||
self._connection_info = connection_info
|
||||
self._language_name = language_name
|
||||
|
||||
def execute_function_in_sql(self,
|
||||
func: Callable, *args,
|
||||
|
|
@ -47,8 +53,15 @@ class SQLPythonExecutor:
|
|||
>>> print(ret)
|
||||
[0.28366218546322625, 0.28366218546322625]
|
||||
"""
|
||||
df, _ = execute_query(SpeesBuilderFromFunction(func, input_data_query, *args, **kwargs), self._connection_info)
|
||||
df, _ = execute_query(SpeesBuilderFromFunction(func,
|
||||
self._language_name,
|
||||
input_data_query,
|
||||
*args,
|
||||
**kwargs),
|
||||
self._connection_info)
|
||||
|
||||
results, output, error = self._get_results(df)
|
||||
|
||||
if output is not None:
|
||||
print(output)
|
||||
if error is not None:
|
||||
|
|
@ -71,7 +84,7 @@ class SQLPythonExecutor:
|
|||
content = script_file.read()
|
||||
except FileNotFoundError:
|
||||
raise FileNotFoundError("File does not exist!")
|
||||
execute_query(SpeesBuilder(content, input_data_query=input_data_query), connection=self._connection_info)
|
||||
execute_query(SpeesBuilder(content, input_data_query=input_data_query, language_name=self._language_name), connection=self._connection_info)
|
||||
|
||||
def execute_sql_query(self,
|
||||
sql_query: str,
|
||||
|
|
@ -130,7 +143,11 @@ class SQLPythonExecutor:
|
|||
out_copy = output_params.copy() if output_params is not None else None
|
||||
|
||||
# Save the stored procedure in database
|
||||
execute_query(StoredProcedureBuilderFromFunction(name, func, in_copy, out_copy),
|
||||
execute_query(StoredProcedureBuilderFromFunction(name=name,
|
||||
func=func,
|
||||
input_params=in_copy,
|
||||
output_params=out_copy,
|
||||
language_name=self._language_name),
|
||||
self._connection_info)
|
||||
return True
|
||||
|
||||
|
|
@ -173,7 +190,11 @@ class SQLPythonExecutor:
|
|||
in_copy = input_params.copy() if input_params is not None else None
|
||||
out_copy = output_params.copy() if output_params is not None else None
|
||||
|
||||
execute_query(StoredProcedureBuilder(name, content, in_copy, out_copy),
|
||||
execute_query(StoredProcedureBuilder(name=name,
|
||||
script=content,
|
||||
input_params=in_copy,
|
||||
output_params=out_copy,
|
||||
language_name=self._language_name),
|
||||
self._connection_info)
|
||||
return True
|
||||
|
||||
|
|
|
|||
|
|
@ -99,8 +99,7 @@ class SQLQueryExecutor:
|
|||
server=self._connection._server if self._connection._port == "" \
|
||||
else "{server},{port}".format(
|
||||
server=self._connection._server,
|
||||
port=self._connection._port
|
||||
)
|
||||
port=self._connection._port)
|
||||
|
||||
self._cnxn = pyodbc.connect(self._connection.connection_string,
|
||||
autocommit=True)
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ def _remove_all_new_packages(manager):
|
|||
packages = ["astor==0.8.1", "html5lib==1.0.1", "termcolor==1.1.0"]
|
||||
|
||||
for package in packages:
|
||||
pipdownloader = PipDownloader(connection, path_to_packages, package)
|
||||
pipdownloader = PipDownloader(connection, path_to_packages, package, language_name="Python")
|
||||
pipdownloader.download_single()
|
||||
|
||||
def test_install_basic_zip_package():
|
||||
|
|
|
|||
|
|
@ -39,6 +39,11 @@ def _package_no_exist(module_name: str):
|
|||
__import__(module_name)
|
||||
return True
|
||||
|
||||
def _check_version(module_name):
|
||||
"""Get the version of an installed package"""
|
||||
module = __import__(module_name)
|
||||
return module.__version__
|
||||
|
||||
def test_install_different_names():
|
||||
"""Test installing a single package with different capitalization"""
|
||||
def useit():
|
||||
|
|
@ -78,53 +83,9 @@ def test_install_version():
|
|||
finally:
|
||||
_drop_all_ddl_packages(connection, scope)
|
||||
|
||||
def test_dependency_spec():
|
||||
"""Test that the DepedencyResolver handles ~= requirement spec.
|
||||
Also tests when package name and module name are different."""
|
||||
package = "azure_cli_telemetry"
|
||||
version = "1.0.4"
|
||||
dependent = "portalocker"
|
||||
module = "azure"
|
||||
|
||||
try:
|
||||
# Install the package and its dependencies
|
||||
#
|
||||
pkgmanager.install(package, version=version)
|
||||
val = pyexecutor.execute_function_in_sql(_package_exists, module_name=module)
|
||||
assert val
|
||||
|
||||
pkgs = _get_package_names_list(connection)
|
||||
|
||||
assert package in pkgs
|
||||
assert dependent in pkgs
|
||||
|
||||
# Uninstall the top package only, not the dependencies
|
||||
#
|
||||
pkgmanager.uninstall(package)
|
||||
val = pyexecutor.execute_function_in_sql(_package_no_exist, module_name=module)
|
||||
assert val
|
||||
|
||||
pkgs = _get_package_names_list(connection)
|
||||
|
||||
assert package not in pkgs
|
||||
assert dependent in pkgs
|
||||
|
||||
# Install the package again, make sure DepedencyResolver can handle ~= Requirement spec
|
||||
#
|
||||
pkgmanager.install(package, version=version)
|
||||
val = pyexecutor.execute_function_in_sql(_package_exists, module_name=module)
|
||||
assert val
|
||||
|
||||
pkgs = _get_package_names_list(connection)
|
||||
|
||||
assert package in pkgs
|
||||
assert dependent in pkgs
|
||||
|
||||
finally:
|
||||
_drop_all_ddl_packages(connection, scope)
|
||||
|
||||
def test_upgrade_parameter():
|
||||
"""Test that "upgrade" installation parameter"""
|
||||
@pytest.mark.skipif(sys.platform.startswith("linux"), reason="Slow test, don't run on Travis-CI, which uses Linux")
|
||||
def test_no_upgrade_parameter():
|
||||
"""Test new version but no "upgrade" installation parameter"""
|
||||
try:
|
||||
pkg = "cryptography"
|
||||
|
||||
|
|
@ -146,18 +107,37 @@ def test_upgrade_parameter():
|
|||
pkgmanager.install(pkg, upgrade=False, version=second_version)
|
||||
assert "exists on server. Set upgrade to True" in output.getvalue()
|
||||
|
||||
# Make sure that the version we have on the server is still the first one
|
||||
#
|
||||
installed_version = pyexecutor.execute_function_in_sql(_check_version, pkg)
|
||||
assert first_version == installed_version
|
||||
|
||||
# Make sure nothing excess was accidentally installed
|
||||
#
|
||||
sqlpkgs = _get_sql_package_table(connection)
|
||||
assert len(sqlpkgs) == len(originalsqlpkgs)
|
||||
|
||||
#################
|
||||
finally:
|
||||
_drop_all_ddl_packages(connection, scope)
|
||||
|
||||
def check_version():
|
||||
import cryptography as cp
|
||||
return cp.__version__
|
||||
@pytest.mark.skipif(sys.platform.startswith("linux"), reason="Slow test, don't run on Travis-CI, which uses Linux")
|
||||
def test_upgrade_parameter():
|
||||
"""Test the "upgrade" installation parameter"""
|
||||
try:
|
||||
pkg = "cryptography"
|
||||
|
||||
oldversion = pyexecutor.execute_function_in_sql(check_version)
|
||||
first_version = "2.7"
|
||||
second_version = "2.8"
|
||||
|
||||
# Install package first so we can test upgrade param
|
||||
#
|
||||
pkgmanager.install(pkg, version=first_version)
|
||||
|
||||
# Get sql packages
|
||||
#
|
||||
originalsqlpkgs = _get_sql_package_table(connection)
|
||||
|
||||
oldversion = pyexecutor.execute_function_in_sql(_check_version, pkg)
|
||||
|
||||
# Test installing WITH the upgrade parameter
|
||||
#
|
||||
|
|
@ -177,6 +157,7 @@ def test_upgrade_parameter():
|
|||
finally:
|
||||
_drop_all_ddl_packages(connection, scope)
|
||||
|
||||
@pytest.mark.skipif(sys.platform.startswith("linux"), reason="Slow test, don't run on Travis-CI, which uses Linux")
|
||||
def test_already_installed_popular_ml_packages():
|
||||
"""Test packages that are preinstalled, make sure they do not install anything extra"""
|
||||
installedpackages = ["numpy", "scipy", "pandas"]
|
||||
|
|
@ -187,6 +168,41 @@ def test_already_installed_popular_ml_packages():
|
|||
newsqlpkgs = _get_sql_package_table(connection)
|
||||
assert len(sqlpkgs) == len(newsqlpkgs)
|
||||
|
||||
@pytest.mark.skipif(sys.platform.startswith("linux"), reason="Slow test, don't run on Travis-CI, which uses Linux")
|
||||
def test_dependency_spec():
|
||||
"""Test that the DepedencyResolver handles ~= requirement spec.
|
||||
Also tests when package name and module name are different."""
|
||||
package = "azure_cli_telemetry"
|
||||
version = "1.0.4"
|
||||
dependent = "portalocker"
|
||||
module = "azure"
|
||||
|
||||
try:
|
||||
# Install the package and its dependencies
|
||||
#
|
||||
pkgmanager.install(package, version=version)
|
||||
val = pyexecutor.execute_function_in_sql(_package_exists, module_name=module)
|
||||
assert val
|
||||
|
||||
pkgs = _get_package_names_list(connection)
|
||||
|
||||
assert package in pkgs
|
||||
assert dependent in pkgs
|
||||
|
||||
# Uninstall the top package only, not the dependencies
|
||||
#
|
||||
pkgmanager.uninstall(package)
|
||||
val = pyexecutor.execute_function_in_sql(_package_no_exist, module_name=module)
|
||||
assert val
|
||||
|
||||
pkgs = _get_package_names_list(connection)
|
||||
|
||||
assert package not in pkgs
|
||||
assert dependent in pkgs
|
||||
|
||||
finally:
|
||||
_drop_all_ddl_packages(connection, scope)
|
||||
|
||||
@pytest.mark.skipif(sys.platform.startswith("linux"), reason="Slow test, don't run on Travis-CI, which uses Linux")
|
||||
def test_installing_popular_ml_packages():
|
||||
"""Test a couple of popular ML packages"""
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
Package: sqlmlutils
|
||||
Type: Package
|
||||
Title: Wraps R code into executable SQL Server stored procedures
|
||||
Version: 0.7.4
|
||||
Version: 1.0.0
|
||||
Author: Microsoft Corporation
|
||||
Maintainer: Microsoft Corporation <msrpack@microsoft.com>
|
||||
Depends:
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ connectionInfo <- function(driver = "SQL Server", server = "localhost", database
|
|||
#'@param inputDataQuery character string. A string to query the database.
|
||||
#' The result of the query will be put into a data frame into the first argument in the function
|
||||
#'@param getScript boolean. Return the tsql script that would be run on the server instead of running it
|
||||
#'@param languageName string. Use a language name other than the default R, if using an EXTERNAL LANGUAGE.
|
||||
#'
|
||||
#'@return The returned value from the function
|
||||
#'
|
||||
|
|
@ -78,7 +79,7 @@ connectionInfo <- function(driver = "SQL Server", server = "localhost", database
|
|||
#'
|
||||
#'@import odbc
|
||||
#'@export
|
||||
executeFunctionInSQL <- function(connectionString, func, ..., inputDataQuery = "", getScript = FALSE)
|
||||
executeFunctionInSQL <- function(connectionString, func, ..., inputDataQuery = "", getScript = FALSE, languageName = "R")
|
||||
{
|
||||
inputDataName <- "InputDataSet"
|
||||
listArgs <- list(...)
|
||||
|
|
@ -99,7 +100,11 @@ executeFunctionInSQL <- function(connectionString, func, ..., inputDataQuery = "
|
|||
|
||||
binArgs <- serialize(listArgs, NULL)
|
||||
|
||||
spees <- speesBuilderFromFunction(func = func, inputDataQuery = inputDataQuery, inputDataName = inputDataName, binArgs)
|
||||
spees <- speesBuilderFromFunction(func = func,
|
||||
inputDataQuery = inputDataQuery,
|
||||
inputDataName = inputDataName,
|
||||
binArgs = binArgs,
|
||||
languageName = languageName)
|
||||
|
||||
if (getScript)
|
||||
{
|
||||
|
|
@ -120,6 +125,7 @@ executeFunctionInSQL <- function(connectionString, func, ..., inputDataQuery = "
|
|||
#'@param inputDataQuery character string. A string to query the database.
|
||||
#' The result of the query will be put into a data frame into the variable "InputDataSet" in the environment
|
||||
#'@param getScript boolean. Return the tsql script that would be run on the server instead of running it
|
||||
#'@param languageName string. Use a language name other than the default R, if using an EXTERNAL LANGUAGE.
|
||||
#'
|
||||
#'@return The returned value from the last line of the script
|
||||
#'
|
||||
|
|
@ -127,7 +133,7 @@ executeFunctionInSQL <- function(connectionString, func, ..., inputDataQuery = "
|
|||
#'\code{\link{executeFunctionInSQL}} to execute a user function instead of a script in SQL
|
||||
#'
|
||||
#'@export
|
||||
executeScriptInSQL <- function(connectionString, script, inputDataQuery = "", getScript = FALSE)
|
||||
executeScriptInSQL <- function(connectionString, script, inputDataQuery = "", getScript = FALSE, languageName = "R")
|
||||
{
|
||||
|
||||
if (file.exists(script))
|
||||
|
|
@ -146,8 +152,12 @@ executeScriptInSQL <- function(connectionString, script, inputDataQuery = "", ge
|
|||
eval(parse(text = script))
|
||||
}
|
||||
|
||||
executeFunctionInSQL(connectionString = connectionString, func = func,
|
||||
script = text, inputDataQuery = inputDataQuery, getScript = getScript)
|
||||
executeFunctionInSQL(connectionString = connectionString,
|
||||
func = func,
|
||||
script = text,
|
||||
inputDataQuery = inputDataQuery,
|
||||
getScript = getScript,
|
||||
languageName = languageName)
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -157,6 +167,7 @@ executeScriptInSQL <- function(connectionString, script, inputDataQuery = "", ge
|
|||
#'@param connectionString character string. The connectionString to the database
|
||||
#'@param sqlQuery character string. The query to execute
|
||||
#'@param getScript boolean. Return the tsql script that would be run on the server instead of running it
|
||||
#'@param languageName string. Use a language name other than the default R, if using an EXTERNAL LANGUAGE.
|
||||
#'
|
||||
#'@return The data frame returned by the query to the database
|
||||
#'
|
||||
|
|
@ -169,15 +180,18 @@ executeScriptInSQL <- function(connectionString, script, inputDataQuery = "", ge
|
|||
#'
|
||||
#'
|
||||
#'@export
|
||||
executeSQLQuery <- function(connectionString, sqlQuery, getScript = FALSE)
|
||||
executeSQLQuery <- function(connectionString, sqlQuery, getScript = FALSE, languageName = "R")
|
||||
{
|
||||
#We use the serialize method here instead of OutputDataSet <- InputDataSet to preserve column names
|
||||
|
||||
script <- " serializedResult <- as.character(serialize(list(result = InputDataSet), NULL))
|
||||
OutputDataSet <- data.frame(returnVal=serializedResult)
|
||||
OutputDataSet <- data.frame(returnVal=serializedResult, stringsAsFactors=FALSE)
|
||||
list(result = InputDataSet)
|
||||
"
|
||||
spees <- speesBuilder(script = script, inputDataQuery = sqlQuery, TRUE)
|
||||
spees <- speesBuilder(script = script,
|
||||
inputDataQuery = sqlQuery,
|
||||
languageName = languageName,
|
||||
withResults = TRUE)
|
||||
|
||||
if (getScript)
|
||||
{
|
||||
|
|
@ -318,18 +332,18 @@ execute <- function(connection, script, ...)
|
|||
# @param inputDataQuery The query on the database
|
||||
# @param withResults Whether to have a result set, outside of the OutputDataSet
|
||||
#
|
||||
speesBuilder <- function(script, inputDataQuery, withResults = FALSE)
|
||||
speesBuilder <- function(script, inputDataQuery, languageName, withResults = FALSE)
|
||||
{
|
||||
resultSet <- if (withResults) "with result sets((returnVal varchar(MAX)))" else ""
|
||||
|
||||
sprintf("exec sp_execute_external_script
|
||||
@language = N'R',
|
||||
@language = N'%s',
|
||||
@script = N'
|
||||
%s
|
||||
',
|
||||
@input_data_1 = N'%s'
|
||||
%s
|
||||
", script, inputDataQuery, resultSet)
|
||||
", languageName, script, inputDataQuery, resultSet)
|
||||
}
|
||||
|
||||
#
|
||||
|
|
@ -343,7 +357,7 @@ speesBuilder <- function(script, inputDataQuery, withResults = FALSE)
|
|||
# @return The spees script to execute
|
||||
# The spees script will return a data frame with the results, serialized
|
||||
#
|
||||
speesBuilderFromFunction <- function(func, inputDataQuery, inputDataName, binArgs)
|
||||
speesBuilderFromFunction <- function(func, inputDataQuery, inputDataName, binArgs, languageName)
|
||||
{
|
||||
funcName <- deparse(substitute(func))
|
||||
funcBody <- gsub('"', '\"', paste0(deparse(func), collapse = "\n"))
|
||||
|
|
@ -384,11 +398,11 @@ speesBuilderFromFunction <- function(func, inputDataQuery, inputDataName, binArg
|
|||
options(warn=oldWarn)
|
||||
|
||||
serializedResult <- as.character(serialize(list(result, output, funwarnings, funerror), NULL))
|
||||
OutputDataSet <- data.frame(returnVal=serializedResult)
|
||||
OutputDataSet <- data.frame(returnVal=serializedResult, stringsAsFactors=FALSE)
|
||||
list(result = result, output = output, warnings = funwarnings, errors = funerror)
|
||||
", funcName, funcBody, paste0(binArgs,collapse=";"), inputDataName, funcName)
|
||||
|
||||
# Call the spees builder to wrap the function; needs the returnVal resultset
|
||||
#
|
||||
speesBuilder(speesBody, inputDataQuery, withResults = TRUE)
|
||||
speesBuilder(speesBody, inputDataQuery, languageName=languageName, withResults = TRUE)
|
||||
}
|
||||
|
|
|
|||
161
R/R/sqlPackage.R
161
R/R/sqlPackage.R
|
|
@ -21,6 +21,8 @@ local(g_scriptFile <- NULL,env=install.env)
|
|||
#' @param scope character string which can be "private" or "public".
|
||||
#' @param owner character string of a user whose private packages shall be listed (availableto dbo or db_owner users only)
|
||||
#' @param scriptFile character string - a file where to record the tsql that is run by the function.
|
||||
#' @param languageName string. Use a language name other than the default R, if using an EXTERNAL LANGUAGE.
|
||||
#'
|
||||
#' @return matrix with enumerated packages
|
||||
#'
|
||||
#'@seealso{
|
||||
|
|
@ -35,7 +37,7 @@ local(g_scriptFile <- NULL,env=install.env)
|
|||
sql_installed.packages <- function(connectionString,
|
||||
priority = NULL, noCache = FALSE, fields = "Package",
|
||||
subarch = NULL, scope = "private", owner = '',
|
||||
scriptFile = NULL)
|
||||
scriptFile = NULL, languageName = "R")
|
||||
{
|
||||
|
||||
assign("g_scriptFile", scriptFile, envir = install.env)
|
||||
|
|
@ -43,7 +45,7 @@ sql_installed.packages <- function(connectionString,
|
|||
|
||||
checkOwner(owner)
|
||||
checkConnectionString(connectionString)
|
||||
checkVersion(connectionString)
|
||||
checkVersion(connectionString, languageName)
|
||||
scope <- normalizeScope(scope)
|
||||
|
||||
enumResult <- list(packages = NULL, warnings = NULL, errors = NULL)
|
||||
|
|
@ -51,7 +53,8 @@ sql_installed.packages <- function(connectionString,
|
|||
enumResult <- sqlEnumPackages(
|
||||
connectionString = connectionString,
|
||||
owner = owner, scope = scope,
|
||||
priority = priority, fields = fields, subarch = subarch)
|
||||
priority = priority, fields = fields, subarch = subarch,
|
||||
languageName = languageName)
|
||||
|
||||
if (!is.null(enumResult$errors))
|
||||
{
|
||||
|
|
@ -80,6 +83,8 @@ sql_installed.packages <- function(connectionString,
|
|||
#' @param scope character string. Should be either "public" or "private". "public" installs the packages on per database public location on SQL server which in turn can be used (referred) by multiple different users. "private" installs the packages on per database, per user private location on SQL server which is only accessible to the single user.
|
||||
#' @param owner character string. Should be either empty '' or a valid SQL database user account name. Only 'dbo' or users in 'db_owner' role for a database can specify this value to install packages on behalf of other users. A user who is member of the 'db_owner' group can set owner='dbo' to install on the "public" folder.
|
||||
#' @param scriptFile character string - a file where to record the tsql that is run by the function.
|
||||
#' @param languageName string. Use a language name other than the default R, if using an EXTERNAL LANGUAGE.
|
||||
#'
|
||||
#' @return invisible(NULL)
|
||||
#'
|
||||
#'@seealso{
|
||||
|
|
@ -95,17 +100,18 @@ sql_install.packages <- function(connectionString,
|
|||
pkgs,
|
||||
skipMissing = FALSE, repos,
|
||||
verbose = getOption("verbose"), scope = "private", owner = '',
|
||||
scriptFile = NULL)
|
||||
scriptFile = NULL, languageName = "R")
|
||||
{
|
||||
assign("g_scriptFile", scriptFile, envir = install.env)
|
||||
checkOwner(owner)
|
||||
checkConnectionString(connectionString)
|
||||
serverVersion <- checkVersion(connectionString)
|
||||
serverVersion <- checkVersion(connectionString, languageName)
|
||||
sqlInstallPackagesExtLib(connectionString,
|
||||
pkgs = pkgs,
|
||||
skipMissing = skipMissing, repos = repos,
|
||||
verbose = verbose, scope = scope, owner = owner,
|
||||
serverVersion = serverVersion)
|
||||
serverVersion = serverVersion,
|
||||
languageName = languageName)
|
||||
|
||||
return(invisible(NULL))
|
||||
}
|
||||
|
|
@ -122,6 +128,8 @@ sql_install.packages <- function(connectionString,
|
|||
#' @param scope character string. Should be either "public" or "private". "public" removes the packages from a per-database public location on SQL Server which in turn could have been used (referred) by multiple different users. "private" removes the packages from a per-database, per-user private location on SQL Server which is only accessible to the single user.
|
||||
#' @param owner character string. Should be either empty '' or a valid SQL database user account name. Only 'dbo' or users in 'db_owner' role for a database can specify this value to remove packages on behalf of other users. A user who is member of the 'db_owner' group can set owner='dbo' to remove packages from the "public" folder.
|
||||
#' @param scriptFile character string - a file where to record the tsql that is run by the function.
|
||||
#' @param languageName string. Use a language name other than the default R, if using an EXTERNAL LANGUAGE.
|
||||
#'
|
||||
#' @return invisible(NULL)
|
||||
#'
|
||||
#'@seealso{
|
||||
|
|
@ -136,12 +144,12 @@ sql_install.packages <- function(connectionString,
|
|||
#' @export
|
||||
sql_remove.packages <- function(connectionString, pkgs, dependencies = TRUE, checkReferences = TRUE,
|
||||
verbose = getOption("verbose"), scope = "private", owner = '',
|
||||
scriptFile = NULL)
|
||||
scriptFile = NULL, languageName = "R")
|
||||
{
|
||||
assign("g_scriptFile", scriptFile, envir = install.env)
|
||||
checkOwner(owner)
|
||||
checkConnectionString(connectionString)
|
||||
checkVersion(connectionString)
|
||||
checkVersion(connectionString, languageName)
|
||||
|
||||
if (length(pkgs) == 0)
|
||||
{
|
||||
|
|
@ -176,7 +184,13 @@ sql_remove.packages <- function(connectionString, pkgs, dependencies = TRUE, che
|
|||
write(sprintf("%s Enumerating installed packages on SQL server...", pkgTime()), stdout())
|
||||
}
|
||||
|
||||
installedPackages <- sql_installed.packages(connectionString = connectionString, fields = NULL, scope = scope, owner = owner, scriptFile = scriptFile)
|
||||
installedPackages <- sql_installed.packages(connectionString = connectionString,
|
||||
fields = NULL,
|
||||
scope = scope,
|
||||
owner = owner,
|
||||
scriptFile = scriptFile,
|
||||
languageName = languageName)
|
||||
|
||||
installedPackages <- data.frame(installedPackages, row.names = NULL, stringsAsFactors = FALSE)
|
||||
installedPackages <- installedPackages[installedPackages$Scope == scope,]
|
||||
|
||||
|
|
@ -187,8 +201,9 @@ sql_remove.packages <- function(connectionString, pkgs, dependencies = TRUE, che
|
|||
if (length(missingPackagesLib) > 0)
|
||||
{
|
||||
# check if package is also missing in the internal table
|
||||
tablePackages <- sqlEnumTable(connectionString, missingPackagesLib, owner, scopeint)
|
||||
missingPackages <- tablePackages[tablePackages$Package == missingPackagesLib & tablePackages$Found == FALSE,"Package",drop=FALSE]$Package
|
||||
#
|
||||
tablePackages <- sqlEnumTable(connectionString, missingPackagesLib, owner, scopeint, languageName)
|
||||
missingPackages <- tablePackages[tablePackages$Package == missingPackagesLib & tablePackages$Found == FALSE, "Package", drop=FALSE]$Package
|
||||
|
||||
if (length(missingPackages) > 0)
|
||||
{
|
||||
|
|
@ -197,15 +212,19 @@ sql_remove.packages <- function(connectionString, pkgs, dependencies = TRUE, che
|
|||
|
||||
# if a package is only in the table we still want to drop external library it
|
||||
# (e.g. package may be failing to install after a create external library outside sqlmlutils)
|
||||
pkgsToDrop <- tablePackages[tablePackages$Package == missingPackagesLib & tablePackages$Found == TRUE,"Package",drop=FALSE]$Package
|
||||
#
|
||||
pkgsToDrop <- tablePackages[tablePackages$Package == missingPackagesLib & tablePackages$Found == TRUE, "Package", drop=FALSE]$Package
|
||||
pkgs <- pkgs[pkgs %in% installedPackages$Package]
|
||||
}
|
||||
|
||||
#
|
||||
# get the dependent list of packages which is safe to remove
|
||||
#
|
||||
pkgsToUninstall <- getDependentPackagesToUninstall(pkgs, installedPackages = installedPackages,
|
||||
dependencies = dependencies, checkReferences = checkReferences, verbose = verbose)
|
||||
pkgsToUninstall <- getDependentPackagesToUninstall(pkgs,
|
||||
installedPackages = installedPackages,
|
||||
dependencies = dependencies,
|
||||
checkReferences = checkReferences,
|
||||
verbose = verbose)
|
||||
|
||||
if (is.null(pkgsToUninstall))
|
||||
{
|
||||
|
|
@ -216,7 +235,8 @@ sql_remove.packages <- function(connectionString, pkgs, dependencies = TRUE, che
|
|||
pkgs <- pkgsToUninstall$Package
|
||||
|
||||
# check if packages to uninstall are in the table as well and be drop external library
|
||||
tablePackages <- sqlEnumTable(connectionString, pkgs, owner, scopeint)
|
||||
#
|
||||
tablePackages <- sqlEnumTable(connectionString, pkgs, owner, scopeint, languageName)
|
||||
pkgsToReport <- tablePackages[tablePackages$Package == pkgs & tablePackages$Found == FALSE, "Package", drop=FALSE]$Package
|
||||
if (length(pkgsToReport) > 0)
|
||||
{
|
||||
|
|
@ -224,6 +244,7 @@ sql_remove.packages <- function(connectionString, pkgs, dependencies = TRUE, che
|
|||
# It may be scheduled to be removed with the next sp_execute_external_script call or
|
||||
# it may be failing to remove at all!
|
||||
# In any case we will track the package and report on its status to the caller
|
||||
#
|
||||
pkgs <- pkgs[!(pkgs %in% pkgsToReport)]
|
||||
}
|
||||
}
|
||||
|
|
@ -235,7 +256,7 @@ sql_remove.packages <- function(connectionString, pkgs, dependencies = TRUE, che
|
|||
write(sprintf("%s Uninstalling packages on SQL server (%s)...", pkgTime(), paste(c(pkgs, pkgsToDrop, pkgsToReport), collapse = ', ')), stdout())
|
||||
}
|
||||
|
||||
sqlHelperRemovePackages(connectionString, pkgs, pkgsToDrop, pkgsToReport, scope, owner, verbose)
|
||||
sqlHelperRemovePackages(connectionString, pkgs, pkgsToDrop, pkgsToReport, scope, owner, verbose, languageName)
|
||||
}
|
||||
|
||||
return(invisible(NULL))
|
||||
|
|
@ -258,7 +279,7 @@ sql_remove.packages <- function(connectionString, pkgs, dependencies = TRUE, che
|
|||
#
|
||||
# @return data frame returned by FUN
|
||||
#
|
||||
sqlRemoteExecuteFun <- function(connection, FUN, ..., useRemoteFun = FALSE, asuser = NULL, includeFun = list())
|
||||
sqlRemoteExecuteFun <- function(connection, FUN, ..., useRemoteFun = FALSE, asuser = NULL, includeFun = list(), languageName)
|
||||
{
|
||||
g_scriptFile <- local(g_scriptFile, install.env)
|
||||
|
||||
|
|
@ -454,7 +475,7 @@ sqlRemoteExecuteFun <- function(connection, FUN, ..., useRemoteFun = FALSE, asus
|
|||
|
||||
query <- paste0(query
|
||||
,"\nEXEC sp_execute_external_script"
|
||||
,"\n@language = N'R'"
|
||||
,"\n@language = N'", languageName, "'"
|
||||
,"\n,@script = N'",script, "';"
|
||||
)
|
||||
|
||||
|
|
@ -702,7 +723,7 @@ getRversionContribFormat <- function()
|
|||
# $rversion
|
||||
# [1] "3.4"
|
||||
#
|
||||
getserverVersion <- function(connectionString)
|
||||
getserverVersion <- function(connectionString, languageName)
|
||||
{
|
||||
checkConnectionString(connectionString)
|
||||
|
||||
|
|
@ -711,7 +732,10 @@ getserverVersion <- function(connectionString)
|
|||
return (list(sysname = Sys.info()[['sysname']], rversion = getRversionContribFormat()))
|
||||
}
|
||||
|
||||
serverVersion <- sqlRemoteExecuteFun(connectionString, getSysnameRversion, includeFun = list(getRversionContribFormat = getRversionContribFormat))
|
||||
serverVersion <- sqlRemoteExecuteFun(connectionString,
|
||||
getSysnameRversion,
|
||||
includeFun = list(getRversionContribFormat = getRversionContribFormat),
|
||||
languageName = languageName)
|
||||
|
||||
return(serverVersion)
|
||||
}
|
||||
|
|
@ -740,9 +764,9 @@ sqlSelectUser <- function(connectionString)
|
|||
# $rversion
|
||||
# [1] "3.4"
|
||||
#
|
||||
checkVersion <- function(connectionString)
|
||||
checkVersion <- function(connectionString, languageName)
|
||||
{
|
||||
serverVersion <- getserverVersion(connectionString)
|
||||
serverVersion <- getserverVersion(connectionString, languageName)
|
||||
serverIsWindows <- serverVersion[['sysname']] == 'Windows'
|
||||
|
||||
versionClass <- sqlCheckPackageManagementVersion(connectionString)
|
||||
|
|
@ -886,7 +910,7 @@ sqlServerProperties <- function(connectionString)
|
|||
#
|
||||
# Returns list containing matrix with installed packages, warnings and errors
|
||||
#
|
||||
sqlEnumPackages <- function(connectionString, owner, scope, priority, fields, subarch)
|
||||
sqlEnumPackages <- function(connectionString, owner, scope, priority, fields, subarch, languageName)
|
||||
{
|
||||
result <- list(packages = NULL, warnings = NULL, errors = NULL)
|
||||
|
||||
|
|
@ -944,7 +968,11 @@ sqlEnumPackages <- function(connectionString, owner, scope, priority, fields, su
|
|||
return (data.frame(Scope = scopes, Path = c(privatePath, publicPath, systemPath), row.names = scopes, stringsAsFactors = FALSE))
|
||||
}
|
||||
|
||||
libPaths <- sqlRemoteExecuteFun(connectionString, getScopeLibraryPaths, asuser = owner, includeFun = list(pkgGetLibraryPath = pkgGetLibraryPath))
|
||||
libPaths <- sqlRemoteExecuteFun(connectionString,
|
||||
getScopeLibraryPaths,
|
||||
asuser = owner,
|
||||
includeFun = list(pkgGetLibraryPath = pkgGetLibraryPath),
|
||||
languageName = languageName)
|
||||
|
||||
|
||||
return(libPaths)
|
||||
|
|
@ -969,7 +997,8 @@ sqlEnumPackages <- function(connectionString, owner, scope, priority, fields, su
|
|||
connectionString = connectionString,
|
||||
packages = packagesNames,
|
||||
owner = owner,
|
||||
scope = scopeint)
|
||||
scope = scopeint,
|
||||
languageName = languageName)
|
||||
|
||||
if (is.null(result) || nrow(result)<1)
|
||||
{
|
||||
|
|
@ -989,7 +1018,7 @@ sqlEnumPackages <- function(connectionString, owner, scope, priority, fields, su
|
|||
{
|
||||
packages <- sqlRemoteExecuteFun(connectionString, utils::installed.packages, lib.loc = libPath, noCache = TRUE,
|
||||
priority = priority, fields = NULL, subarch = subarch,
|
||||
useRemoteFun = TRUE, asuser = owner)
|
||||
useRemoteFun = TRUE, asuser = owner, languageName = languageName)
|
||||
},
|
||||
error = function(err)
|
||||
{
|
||||
|
|
@ -1280,7 +1309,7 @@ downloadDependentPackages <- function(pkgs, destdir, binaryPackages, sourcePacka
|
|||
}
|
||||
else
|
||||
{
|
||||
# If the server and client are NOT the same type,
|
||||
# If the server and client are NOT the same type,
|
||||
# we just download the source package and send it to the server to build
|
||||
#
|
||||
downloadedPkg <- utils::download.packages(pkg$Package, destdir = destdir,
|
||||
|
|
@ -1316,11 +1345,11 @@ buildSourcePackage <- function(name, destdir, sourcePackages)
|
|||
|
||||
# Build the source into a binary package.
|
||||
# This will also install the package to the destdir since we cannot build without installing.
|
||||
#
|
||||
#
|
||||
utils::install.packages(pkgPath, INSTALL_opts = "--build",
|
||||
repos=NULL, lib = destdir, quiet = TRUE)
|
||||
|
||||
# Find the binary (zip for Windows, tar.gz for Unix) that was created.
|
||||
# Find the binary (zip for Windows, tar.gz for Unix) that was created.
|
||||
# install.packages creates the binary file in the current working directory.
|
||||
#
|
||||
binaryFile = list.files(pattern=utils::glob2rx(paste0(name, "*zip")))[1]
|
||||
|
|
@ -1333,9 +1362,9 @@ buildSourcePackage <- function(name, destdir, sourcePackages)
|
|||
pkgMatrix = NULL
|
||||
|
||||
# Copy the binary file from the current working directory to the destdir
|
||||
# so we know exactly where it is.
|
||||
# so we know exactly where it is.
|
||||
# Construct a matrix with similar structure to download.packages return value.
|
||||
#
|
||||
#
|
||||
if(!is.na(binaryFile))
|
||||
{
|
||||
if(file.copy(from=binaryFile, to=destdir))
|
||||
|
|
@ -1355,7 +1384,8 @@ sqlInstallPackagesExtLib <- function(connectionString,
|
|||
pkgs,
|
||||
skipMissing = FALSE, repos, verbose,
|
||||
scope = "private", owner = '',
|
||||
serverVersion = serverVersion)
|
||||
serverVersion = serverVersion,
|
||||
languageName)
|
||||
{
|
||||
g_scriptFile <- local(g_scriptFile, install.env)
|
||||
|
||||
|
|
@ -1445,7 +1475,7 @@ sqlInstallPackagesExtLib <- function(connectionString,
|
|||
}
|
||||
}
|
||||
|
||||
attributePackages <- function(connectionString, packages, scopeint, owner, verbose)
|
||||
attributePackages <- function(connectionString, packages, scopeint, owner, verbose, languageName)
|
||||
{
|
||||
packagesNames <- sapply(packages, function(pkg) {pkg$name},USE.NAMES = FALSE)
|
||||
|
||||
|
|
@ -1457,7 +1487,8 @@ sqlInstallPackagesExtLib <- function(connectionString,
|
|||
result <- sqlMakeTopLevel(connectionString = connectionString,
|
||||
packages = packagesNames,
|
||||
owner = owner,
|
||||
scope = as.integer(scopeint))
|
||||
scope = as.integer(scopeint),
|
||||
languageName = languageName)
|
||||
|
||||
if (result)
|
||||
{
|
||||
|
|
@ -1568,7 +1599,13 @@ sqlInstallPackagesExtLib <- function(connectionString,
|
|||
|
||||
# get all installed packages
|
||||
#
|
||||
installedPackages <- sql_installed.packages(connectionString, fields = NULL, scope = scope, owner = owner, scriptFile = g_scriptFile)
|
||||
installedPackages <- sql_installed.packages(connectionString,
|
||||
fields = NULL,
|
||||
scope = scope,
|
||||
owner = owner,
|
||||
scriptFile = g_scriptFile,
|
||||
languageName = languageName)
|
||||
|
||||
installedPackages <- data.frame(installedPackages, row.names = NULL, stringsAsFactors = FALSE)
|
||||
|
||||
# get dependency closure of given packages
|
||||
|
|
@ -1595,7 +1632,7 @@ sqlInstallPackagesExtLib <- function(connectionString,
|
|||
|
||||
if (length(pkgsToDownload) > 0)
|
||||
{
|
||||
serverVersion <- checkVersion(connectionString)
|
||||
serverVersion <- checkVersion(connectionString, languageName)
|
||||
if (serverVersion$serverIsWindows)
|
||||
{
|
||||
pkgType = "win.binary"
|
||||
|
|
@ -1625,7 +1662,7 @@ sqlInstallPackagesExtLib <- function(connectionString,
|
|||
})
|
||||
|
||||
downloadPkgs <- cbind(downloadPkgs, Attribute = attributesVec)
|
||||
sqlHelperInstallPackages(connectionString, downloadPkgs, owner, scope, verbose)
|
||||
sqlHelperInstallPackages(connectionString, downloadPkgs, owner, scope, verbose, languageName)
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -1647,7 +1684,7 @@ sqlInstallPackagesExtLib <- function(connectionString,
|
|||
packages[[length(packages) + 1]] <- packageDescriptor
|
||||
}
|
||||
|
||||
attributePackages(connectionString, packages, scopeint, owner, verbose)
|
||||
attributePackages(connectionString, packages, scopeint, owner, verbose, languageName)
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1673,7 +1710,7 @@ sqlInstallPackagesExtLib <- function(connectionString,
|
|||
stringsAsFactors = FALSE))
|
||||
}
|
||||
|
||||
sqlHelperInstallPackages(connectionString, packages, owner, scope, verbose)
|
||||
sqlHelperInstallPackages(connectionString, packages, owner, scope, verbose, languageName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1681,7 +1718,7 @@ sqlInstallPackagesExtLib <- function(connectionString,
|
|||
#
|
||||
# Calls CREATE EXTERNAL LIBRARY on a package
|
||||
#
|
||||
sqlCreateExternalLibrary <- function(hodbc, packageName, packageFile, user = "")
|
||||
sqlCreateExternalLibrary <- function(hodbc, packageName, packageFile, user = "", languageName)
|
||||
{
|
||||
g_scriptFile <- local(g_scriptFile, install.env)
|
||||
|
||||
|
|
@ -1702,7 +1739,7 @@ sqlCreateExternalLibrary <- function(hodbc, packageName, packageFile, user = "")
|
|||
query <- paste0(query, " AUTHORIZATION ", user)
|
||||
}
|
||||
|
||||
query <- paste0(query, " FROM (CONTENT=", pkgContent ,") WITH (LANGUAGE = 'R');")
|
||||
query <- paste0(query, " FROM (CONTENT=", pkgContent ,") WITH (LANGUAGE = '", languageName,"');")
|
||||
|
||||
if(!is.null(g_scriptFile))
|
||||
{
|
||||
|
|
@ -1800,7 +1837,7 @@ sqlAddExtendedProperty <- function(hodbc, packageName, attributes, user = "")
|
|||
stop(paste(sqlResult, sep = "\n"))
|
||||
}
|
||||
|
||||
sqlMakeTopLevel <- function(connectionString, packages, owner, scope)
|
||||
sqlMakeTopLevel <- function(connectionString, packages, owner, scope, languageName)
|
||||
{
|
||||
changeTo = 1
|
||||
haveUser <- (owner != '')
|
||||
|
|
@ -1819,7 +1856,7 @@ sqlMakeTopLevel <- function(connectionString, packages, owner, scope)
|
|||
query = paste0(query, "EXEC sp_updateextendedproperty @name = N'IsTopPackage', @value=", changeTo,", @level0type=N'USER',
|
||||
@level0name=", user, ", @level1type = N'external library', @level1name=?")
|
||||
|
||||
packageList <- enumerateTopPackages(connectionString, packages, owner, scope)$name
|
||||
packageList <- enumerateTopPackages(connectionString, packages, owner, scope, languageName)$name
|
||||
|
||||
tryCatch(
|
||||
{
|
||||
|
|
@ -1863,7 +1900,7 @@ sqlMakeTopLevel <- function(connectionString, packages, owner, scope)
|
|||
#
|
||||
# Returns data frame with packages names and associated external library id |name|external_library_id|
|
||||
#
|
||||
sqlQueryExternalLibraryId <- function(hodbc, packagesNames, scopeint, queryUser)
|
||||
sqlQueryExternalLibraryId <- function(hodbc, packagesNames, scopeint, queryUser, languageName)
|
||||
{
|
||||
query <- paste0(
|
||||
" SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;", # Sets transactions isolation level to read uncommited for the current connections so we can read external library ids
|
||||
|
|
@ -1881,7 +1918,7 @@ sqlQueryExternalLibraryId <- function(hodbc, packagesNames, scopeint, queryUser)
|
|||
paste0("'", paste(packagesNames, collapse = "','"), "'"),
|
||||
")",
|
||||
" AND elib.principal_id=@principalId",
|
||||
" AND elib.language='R' AND elib.scope=", scopeint,
|
||||
" AND elib.language='", languageName,"' AND elib.scope=", scopeint,
|
||||
" ORDER BY elib.name ASC",
|
||||
" ;"
|
||||
)
|
||||
|
|
@ -1998,16 +2035,16 @@ findPackages <- function(packages, scopeint)
|
|||
#
|
||||
# Returns vector of successfully installed packages
|
||||
#
|
||||
sqlSyncAndCheckInstalledPackages <- function(hodbc, packages, user = "", queryUser, scope = "PRIVATE")
|
||||
sqlSyncAndCheckInstalledPackages <- function(hodbc, packages, user = "", queryUser, scope = "PRIVATE", languageName)
|
||||
{
|
||||
scopeint <- parseScope(scope)
|
||||
|
||||
externalLibraryIds <- sqlQueryExternalLibraryId(hodbc, packages, scopeint, queryUser)
|
||||
externalLibraryIds <- sqlQueryExternalLibraryId(hodbc, packages, scopeint, queryUser, languageName)
|
||||
|
||||
# sp_execute_external_script will first install packages to the library path
|
||||
# and the run R function to check if packages installed
|
||||
#
|
||||
checkdf <- sqlRemoteExecuteFun(hodbc, findPackages, packages, scopeint, asuser = user)
|
||||
checkdf <- sqlRemoteExecuteFun(hodbc, findPackages, packages, scopeint, asuser = user, languageName = languageName)
|
||||
|
||||
setupFailures <- sqlQueryExternalLibrarySetupErrors(hodbc, externalLibraryIds, queryUser)
|
||||
|
||||
|
|
@ -2051,7 +2088,7 @@ sqlSyncAndCheckInstalledPackages <- function(hodbc, packages, user = "", queryUs
|
|||
return(packages)
|
||||
}
|
||||
|
||||
sqlHelperInstallPackages <- function(connectionString, packages, owner = "", scope = "PRIVATE", verbose)
|
||||
sqlHelperInstallPackages <- function(connectionString, packages, owner = "", scope = "PRIVATE", verbose, languageName)
|
||||
{
|
||||
user <- "" # user argument for Create External Library
|
||||
queryUser <- "CURRENT_USER" # user argument for select to discover external_library_id
|
||||
|
|
@ -2106,7 +2143,7 @@ sqlHelperInstallPackages <- function(connectionString, packages, owner = "", sco
|
|||
write(sprintf("%s Copying package to Sql server [%d/%d] %s...", pkgTime(), packageIndex, numPkgs, packageName), stdout())
|
||||
}
|
||||
|
||||
sqlCreateExternalLibrary(hodbc, packageName, filelocation, user)
|
||||
sqlCreateExternalLibrary(hodbc, packageName, filelocation, user, languageName)
|
||||
sqlAddExtendedProperty(hodbc, packageName, attribute, user)
|
||||
}
|
||||
|
||||
|
|
@ -2115,7 +2152,7 @@ sqlHelperInstallPackages <- function(connectionString, packages, owner = "", sco
|
|||
write(sprintf("%s Installing packages to library path, this may take some time...", pkgTime()), stdout())
|
||||
}
|
||||
|
||||
packagesSuccess <- sqlSyncAndCheckInstalledPackages(hodbc, packages[,"Package"], user, queryUser, scope);
|
||||
packagesSuccess <- sqlSyncAndCheckInstalledPackages(hodbc, packages[,"Package"], user, queryUser, scope, languageName);
|
||||
dbCommit(hodbc)
|
||||
haveTransaction = FALSE
|
||||
},
|
||||
|
|
@ -2152,7 +2189,7 @@ sqlHelperInstallPackages <- function(connectionString, packages, owner = "", sco
|
|||
}
|
||||
|
||||
|
||||
sqlHelperRemovePackages <- function(connectionString, pkgs, pkgsToDrop, pkgsToReport, scope, owner, verbose)
|
||||
sqlHelperRemovePackages <- function(connectionString, pkgs, pkgsToDrop, pkgsToReport, scope, owner, verbose, languageName)
|
||||
{
|
||||
user <- "" # user argument for Drop External Library
|
||||
queryUser <- "CURRENT_USER" # user argument for select to discover external_library_id
|
||||
|
|
@ -2214,13 +2251,13 @@ sqlHelperRemovePackages <- function(connectionString, pkgs, pkgsToDrop, pkgsToRe
|
|||
# with the view sys.external_library_setup_errors for errors reported
|
||||
# by the external library uninstaller
|
||||
#
|
||||
externalLibraryIds <- sqlQueryExternalLibraryId(hodbc, pkgs, scopeint, queryUser)
|
||||
externalLibraryIds <- sqlQueryExternalLibraryId(hodbc, pkgs, scopeint, queryUser, languageName)
|
||||
lapply(pkgs, sqlDropExternalLibrary, hodbc = hodbc, user=user)
|
||||
pkgsSuccess <- c(pkgsSuccess, sqlSyncRemovePackages(hodbc, c(pkgs,pkgsToReport), externalLibraryIds, scope, user, queryUser, verbose))
|
||||
pkgsSuccess <- c(pkgsSuccess, sqlSyncRemovePackages(hodbc, c(pkgs,pkgsToReport), externalLibraryIds, scope, user, queryUser, verbose, languageName))
|
||||
}
|
||||
else if(length(pkgsToReport) > 0)
|
||||
{
|
||||
pkgsSuccess <- c(pkgsSuccess, sqlSyncRemovePackages(hodbc, pkgsToReport, externalLibraryIds = NULL, scope, user, queryUser = NULL, verbose = verbose))
|
||||
pkgsSuccess <- c(pkgsSuccess, sqlSyncRemovePackages(hodbc, pkgsToReport, externalLibraryIds = NULL, scope, user, queryUser = NULL, verbose = verbose, languageName))
|
||||
}
|
||||
|
||||
dbCommit(hodbc)
|
||||
|
|
@ -2266,7 +2303,7 @@ sqlHelperRemovePackages <- function(connectionString, pkgs, pkgsToDrop, pkgsToRe
|
|||
#
|
||||
# Returns vector of successfully removed packages
|
||||
#
|
||||
sqlSyncRemovePackages <- function(hodbc, pkgs, externalLibraryIds, scope, user, queryUser, verbose)
|
||||
sqlSyncRemovePackages <- function(hodbc, pkgs, externalLibraryIds, scope, user, queryUser, verbose, languageName)
|
||||
{
|
||||
if(verbose)
|
||||
{
|
||||
|
|
@ -2274,7 +2311,7 @@ sqlSyncRemovePackages <- function(hodbc, pkgs, externalLibraryIds, scope, user,
|
|||
}
|
||||
scopeint <- parseScope(scope)
|
||||
|
||||
checkdf <- sqlRemoteExecuteFun(hodbc, findPackages, pkgs, scopeint, asuser = user)
|
||||
checkdf <- sqlRemoteExecuteFun(hodbc, findPackages, pkgs, scopeint, asuser = user, languageName = languageName)
|
||||
|
||||
if(!(is.null(externalLibraryIds) || is.null(queryUser)))
|
||||
{
|
||||
|
|
@ -2326,7 +2363,7 @@ sqlSyncRemovePackages <- function(hodbc, pkgs, externalLibraryIds, scope, user,
|
|||
# All submitted packages will be listed.
|
||||
# If a package was found in the database, find value will be TRUE otherwise FALSE
|
||||
#
|
||||
sqlEnumTable <- function(connectionString, packagesNames, owner, scopeint)
|
||||
sqlEnumTable <- function(connectionString, packagesNames, owner, scopeint, languageName)
|
||||
{
|
||||
g_scriptFile <- local(g_scriptFile, install.env)
|
||||
queryUser <- "CURRENT_USER"
|
||||
|
|
@ -2363,7 +2400,7 @@ sqlEnumTable <- function(connectionString, packagesNames, owner, scopeint)
|
|||
paste0("'", paste(packagesNames, collapse = "','"), "'"),
|
||||
")",
|
||||
" AND elib.principal_id=@principalId",
|
||||
" AND elib.language='R' AND elib.scope=", scopeint,
|
||||
" AND elib.language='", languageName,"' AND elib.scope=", scopeint,
|
||||
" ORDER BY elib.name ASC",
|
||||
" ;"
|
||||
)
|
||||
|
|
@ -2566,7 +2603,7 @@ getDependentPackagesToUninstall <- function(pkgs, installedPackages, dependencie
|
|||
|
||||
# Returns dataframe |name (package name)|IsTopPackage (-1,0,1)|
|
||||
#
|
||||
enumerateTopPackages <- function(connectionString, packages, owner, scope)
|
||||
enumerateTopPackages <- function(connectionString, packages, owner, scope, languageName)
|
||||
{
|
||||
haveUser <- (owner != '')
|
||||
|
||||
|
|
@ -2612,9 +2649,9 @@ enumerateTopPackages <- function(connectionString, packages, owner, scope)
|
|||
INNER JOIN eprop
|
||||
ON eprop.major_id = elib.external_library_id AND elib.name in (%s)
|
||||
AND elib.principal_id=@principalId
|
||||
AND elib.language='R' AND elib.scope=?
|
||||
AND elib.language='%s' AND elib.scope=?
|
||||
ORDER BY elib.name ASC
|
||||
;", pkgcsv))
|
||||
;", pkgcsv, languageName))
|
||||
|
||||
tryCatch(
|
||||
{
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
#'@param outputParams named list. The types of the outputs,
|
||||
#'where the names are the arguments and the values are the types
|
||||
#'@param getScript boolean. Return the tsql script that would be run on the server instead of running it
|
||||
#'@param languageName string. Use a language name other than the default R, if using an EXTERNAL LANGUAGE.
|
||||
#'
|
||||
#'@section Warning:
|
||||
#'You can add output parameters to the stored procedure
|
||||
|
|
@ -62,7 +63,7 @@
|
|||
#'@export
|
||||
createSprocFromFunction <- function (connectionString, name, func,
|
||||
inputParams = NULL, outputParams = NULL,
|
||||
getScript = FALSE)
|
||||
getScript = FALSE, languageName = "R")
|
||||
{
|
||||
possibleTypes <- c("posixct", "numeric", "character", "integer", "logical", "raw", "dataframe")
|
||||
|
||||
|
|
@ -83,7 +84,7 @@ createSprocFromFunction <- function (connectionString, name, func,
|
|||
stop("inputParams and function arguments do not match!")
|
||||
}
|
||||
|
||||
procScript <- generateTSQL(func = func, spName = name, inputParams = inputParams, outputParams = outputParams)
|
||||
procScript <- generateTSQL(func = func, spName = name, inputParams = inputParams, outputParams = outputParams, languageName = languageName)
|
||||
|
||||
if (getScript)
|
||||
{
|
||||
|
|
@ -106,7 +107,7 @@ createSprocFromFunction <- function (connectionString, name, func,
|
|||
#'@export
|
||||
createSprocFromScript <- function (connectionString, name, script,
|
||||
inputParams = NULL, outputParams = NULL,
|
||||
getScript = FALSE)
|
||||
getScript = FALSE, languageName = "R")
|
||||
{
|
||||
if (file.exists(script))
|
||||
{
|
||||
|
|
@ -131,7 +132,7 @@ createSprocFromScript <- function (connectionString, name, script,
|
|||
if (!tolower(x) %in% possibleTypes) stop("Possible output types are POSIXct, numeric, character, integer, logical, raw, and DataFrame.")
|
||||
})
|
||||
|
||||
procScript <- generateTSQLFromScript(script = text, spName = name, inputParams = inputParams, outputParams = outputParams)
|
||||
procScript <- generateTSQLFromScript(script = text, spName = name, inputParams = inputParams, outputParams = outputParams, languageName = languageName)
|
||||
|
||||
if (getScript)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -14,14 +14,14 @@ getSqlType <- function(rType)
|
|||
|
||||
# creates the top part of the sql script (up to R code)
|
||||
#
|
||||
getHeader <- function(spName, inputParams, outputParams)
|
||||
getHeader <- function(spName, inputParams, outputParams, languageName)
|
||||
{
|
||||
header <- c(paste0 ("CREATE PROCEDURE ", spName),
|
||||
header <- c(paste0("CREATE PROCEDURE ", spName),
|
||||
handleHeadParams(inputParams, outputParams),
|
||||
"AS",
|
||||
"BEGIN TRY",
|
||||
"exec sp_execute_external_script",
|
||||
"@language = N'R',","@script = N'")
|
||||
paste0("@language = N'", languageName,"',"),"@script = N'")
|
||||
|
||||
return(paste0(header, collapse = "\n"))
|
||||
}
|
||||
|
|
@ -54,11 +54,11 @@ handleHeadParams <- function(inputParams, outputParams)
|
|||
return(paste0(paramString, collapse = ",\n"))
|
||||
}
|
||||
|
||||
generateTSQL <- function(func, spName, inputParams = NULL, outputParams = NULL )
|
||||
generateTSQL <- function(func, spName, inputParams, outputParams, languageName)
|
||||
{
|
||||
# header to drop and create a stored procedure
|
||||
#
|
||||
header <- getHeader(spName, inputParams, outputParams)
|
||||
header <- getHeader(spName, inputParams = inputParams, outputParams = outputParams, languageName = languageName)
|
||||
|
||||
# vector containing R code
|
||||
#
|
||||
|
|
@ -71,11 +71,11 @@ generateTSQL <- function(func, spName, inputParams = NULL, outputParams = NULL )
|
|||
return(paste0(header, rCode, tail, sep = "\n"))
|
||||
}
|
||||
|
||||
generateTSQLFromScript <- function(script, spName, inputParams, outputParams)
|
||||
generateTSQLFromScript <- function(script, spName, inputParams, outputParams, languageName)
|
||||
{
|
||||
# header to drop and create a stored procedure
|
||||
#
|
||||
header <- getHeader(spName, inputParams = inputParams, outputParams = outputParams)
|
||||
header <- getHeader(spName, inputParams = inputParams, outputParams = outputParams, languageName = languageName)
|
||||
|
||||
# vector containing R code
|
||||
#
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ sqlmlutils is an R package to help execute R code on a SQL database (SQL Server
|
|||
From command prompt, run
|
||||
```
|
||||
R.exe -e "install.packages('odbc')"
|
||||
R.exe CMD INSTALL dist/sqlmlutils_0.7.4.zip
|
||||
R.exe CMD INSTALL dist/sqlmlutils_1.0.0.zip
|
||||
```
|
||||
OR
|
||||
To build a new package file and install, run
|
||||
|
|
@ -19,7 +19,7 @@ To build a new package file and install, run
|
|||
### Linux
|
||||
```
|
||||
R.exe -e "install.packages('odbc')"
|
||||
R.exe CMD INSTALL dist/sqlmlutils_0.7.4.tar.gz
|
||||
R.exe CMD INSTALL dist/sqlmlutils_1.0.0.tar.gz
|
||||
```
|
||||
|
||||
# Getting started
|
||||
|
|
|
|||
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
|
|
@ -11,7 +11,8 @@ createSprocFromFunction(
|
|||
func,
|
||||
inputParams = NULL,
|
||||
outputParams = NULL,
|
||||
getScript = FALSE
|
||||
getScript = FALSE,
|
||||
languageName = "R"
|
||||
)
|
||||
|
||||
createSprocFromScript(
|
||||
|
|
@ -20,7 +21,8 @@ createSprocFromScript(
|
|||
script,
|
||||
inputParams = NULL,
|
||||
outputParams = NULL,
|
||||
getScript = FALSE
|
||||
getScript = FALSE,
|
||||
languageName = "R"
|
||||
)
|
||||
}
|
||||
\arguments{
|
||||
|
|
@ -38,6 +40,8 @@ where the names are the arguments and the values are the types}
|
|||
|
||||
\item{getScript}{boolean. Return the tsql script that would be run on the server instead of running it}
|
||||
|
||||
\item{languageName}{string. Use a language name other than the default R, if using an EXTERNAL LANGUAGE.}
|
||||
|
||||
\item{script}{character string. The path to the script to wrap in the stored procedure}
|
||||
}
|
||||
\value{
|
||||
|
|
|
|||
|
|
@ -9,7 +9,8 @@ executeFunctionInSQL(
|
|||
func,
|
||||
...,
|
||||
inputDataQuery = "",
|
||||
getScript = FALSE
|
||||
getScript = FALSE,
|
||||
languageName = "R"
|
||||
)
|
||||
}
|
||||
\arguments{
|
||||
|
|
@ -23,6 +24,8 @@ executeFunctionInSQL(
|
|||
The result of the query will be put into a data frame into the first argument in the function}
|
||||
|
||||
\item{getScript}{boolean. Return the tsql script that would be run on the server instead of running it}
|
||||
|
||||
\item{languageName}{string. Use a language name other than the default R, if using an EXTERNAL LANGUAGE.}
|
||||
}
|
||||
\value{
|
||||
The returned value from the function
|
||||
|
|
|
|||
|
|
@ -4,7 +4,12 @@
|
|||
\alias{executeSQLQuery}
|
||||
\title{Execute a script in SQL}
|
||||
\usage{
|
||||
executeSQLQuery(connectionString, sqlQuery, getScript = FALSE)
|
||||
executeSQLQuery(
|
||||
connectionString,
|
||||
sqlQuery,
|
||||
getScript = FALSE,
|
||||
languageName = "R"
|
||||
)
|
||||
}
|
||||
\arguments{
|
||||
\item{connectionString}{character string. The connectionString to the database}
|
||||
|
|
@ -12,6 +17,8 @@ executeSQLQuery(connectionString, sqlQuery, getScript = FALSE)
|
|||
\item{sqlQuery}{character string. The query to execute}
|
||||
|
||||
\item{getScript}{boolean. Return the tsql script that would be run on the server instead of running it}
|
||||
|
||||
\item{languageName}{string. Use a language name other than the default R, if using an EXTERNAL LANGUAGE.}
|
||||
}
|
||||
\value{
|
||||
The data frame returned by the query to the database
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@ executeScriptInSQL(
|
|||
connectionString,
|
||||
script,
|
||||
inputDataQuery = "",
|
||||
getScript = FALSE
|
||||
getScript = FALSE,
|
||||
languageName = "R"
|
||||
)
|
||||
}
|
||||
\arguments{
|
||||
|
|
@ -20,6 +21,8 @@ executeScriptInSQL(
|
|||
The result of the query will be put into a data frame into the variable "InputDataSet" in the environment}
|
||||
|
||||
\item{getScript}{boolean. Return the tsql script that would be run on the server instead of running it}
|
||||
|
||||
\item{languageName}{string. Use a language name other than the default R, if using an EXTERNAL LANGUAGE.}
|
||||
}
|
||||
\value{
|
||||
The returned value from the last line of the script
|
||||
|
|
|
|||
|
|
@ -12,7 +12,8 @@ sql_install.packages(
|
|||
verbose = getOption("verbose"),
|
||||
scope = "private",
|
||||
owner = "",
|
||||
scriptFile = NULL
|
||||
scriptFile = NULL,
|
||||
languageName = "R"
|
||||
)
|
||||
}
|
||||
\arguments{
|
||||
|
|
@ -31,6 +32,8 @@ sql_install.packages(
|
|||
\item{owner}{character string. Should be either empty '' or a valid SQL database user account name. Only 'dbo' or users in 'db_owner' role for a database can specify this value to install packages on behalf of other users. A user who is member of the 'db_owner' group can set owner='dbo' to install on the "public" folder.}
|
||||
|
||||
\item{scriptFile}{character string - a file where to record the tsql that is run by the function.}
|
||||
|
||||
\item{languageName}{string. Use a language name other than the default R, if using an EXTERNAL LANGUAGE.}
|
||||
}
|
||||
\value{
|
||||
invisible(NULL)
|
||||
|
|
|
|||
|
|
@ -12,7 +12,8 @@ sql_installed.packages(
|
|||
subarch = NULL,
|
||||
scope = "private",
|
||||
owner = "",
|
||||
scriptFile = NULL
|
||||
scriptFile = NULL,
|
||||
languageName = "R"
|
||||
)
|
||||
}
|
||||
\arguments{
|
||||
|
|
@ -31,6 +32,8 @@ sql_installed.packages(
|
|||
\item{owner}{character string of a user whose private packages shall be listed (availableto dbo or db_owner users only)}
|
||||
|
||||
\item{scriptFile}{character string - a file where to record the tsql that is run by the function.}
|
||||
|
||||
\item{languageName}{string. Use a language name other than the default R, if using an EXTERNAL LANGUAGE.}
|
||||
}
|
||||
\value{
|
||||
matrix with enumerated packages
|
||||
|
|
|
|||
|
|
@ -12,7 +12,8 @@ sql_remove.packages(
|
|||
verbose = getOption("verbose"),
|
||||
scope = "private",
|
||||
owner = "",
|
||||
scriptFile = NULL
|
||||
scriptFile = NULL,
|
||||
languageName = "R"
|
||||
)
|
||||
}
|
||||
\arguments{
|
||||
|
|
@ -31,6 +32,8 @@ sql_remove.packages(
|
|||
\item{owner}{character string. Should be either empty '' or a valid SQL database user account name. Only 'dbo' or users in 'db_owner' role for a database can specify this value to remove packages on behalf of other users. A user who is member of the 'db_owner' group can set owner='dbo' to remove packages from the "public" folder.}
|
||||
|
||||
\item{scriptFile}{character string - a file where to record the tsql that is run by the function.}
|
||||
|
||||
\item{languageName}{string. Use a language name other than the default R, if using an EXTERNAL LANGUAGE.}
|
||||
}
|
||||
\value{
|
||||
invisible(NULL)
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ helper_isLinux <- function()
|
|||
|
||||
helper_isServerLinux <- function()
|
||||
{
|
||||
return (sqlmlutils:::sqlRemoteExecuteFun(helper_getSetting("connectionStringDBO"), helper_isLinux))
|
||||
return (sqlmlutils:::sqlRemoteExecuteFun(helper_getSetting("connectionStringDBO"), helper_isLinux, languageName="R"))
|
||||
}
|
||||
|
||||
#
|
||||
|
|
@ -54,7 +54,7 @@ helper_isServerLinux <- function()
|
|||
#
|
||||
helper_remote.require <- function(connectionString, packageName)
|
||||
{
|
||||
return (suppressWarnings((sqlmlutils:::sqlRemoteExecuteFun(connectionString, require, package = packageName, useRemoteFun = TRUE ))))
|
||||
return (suppressWarnings((sqlmlutils:::sqlRemoteExecuteFun(connectionString, require, package = packageName, useRemoteFun = TRUE , languageName="R"))))
|
||||
}
|
||||
|
||||
helper_checkPackageStatusRequire <- function(connectionString, packageName, expectedInstallStatus)
|
||||
|
|
@ -70,7 +70,7 @@ helper_checkPackageStatusRequire <- function(connectionString, packageName, expe
|
|||
#
|
||||
helper_remote.find.package <- function(connectionString, packageName)
|
||||
{
|
||||
findResult <- sqlmlutils:::sqlRemoteExecuteFun(connectionString, find.package, package = packageName, quiet = TRUE, useRemoteFun = TRUE )
|
||||
findResult <- sqlmlutils:::sqlRemoteExecuteFun(connectionString, find.package, package = packageName, quiet = TRUE, useRemoteFun = TRUE, languageName="R" )
|
||||
|
||||
return (is.character(findResult) && (length(findResult) > 0))
|
||||
}
|
||||
|
|
@ -85,7 +85,7 @@ helper_checkPackageStatusFind <- function(connectionString, packageName, expecte
|
|||
|
||||
helper_checkSqlLibPaths <- function(connectionString, minimumCount)
|
||||
{
|
||||
sqlLibPaths = sqlmlutils:::sqlRemoteExecuteFun(connectionString, .libPaths, useRemoteFun = TRUE )
|
||||
sqlLibPaths = sqlmlutils:::sqlRemoteExecuteFun(connectionString, .libPaths, useRemoteFun = TRUE, languageName="R" )
|
||||
cat(paste0( "INFO: lib paths = ", sqlLibPaths, colapse = "\r\n"))
|
||||
expect_true(length(sqlLibPaths) >= minimumCount)
|
||||
}
|
||||
|
|
@ -97,7 +97,7 @@ helper_ExecuteSQLDDL <- function(connectionString, sqlDDL)
|
|||
sqlmlutils:::execute(connectionString, sqlDDL)
|
||||
}
|
||||
|
||||
helper_CreateExternalLibrary <- function(connectionString, packageName, authorization=NULL, content)
|
||||
helper_CreateExternalLibrary <- function(connectionString, packageName, authorization=NULL, content, languageName="R")
|
||||
{
|
||||
# 1. issue 'CREATE EXTERNAL LIBRARY'
|
||||
createExtLibDDLString = paste0("CREATE EXTERNAL LIBRARY [", packageName, "]")
|
||||
|
|
@ -108,22 +108,22 @@ helper_CreateExternalLibrary <- function(connectionString, packageName, authoriz
|
|||
|
||||
if (substr(content, 0, 2) == "0x")
|
||||
{
|
||||
createExtLibDDLString = paste0(createExtLibDDLString, " FROM (content = ", content, ") WITH (LANGUAGE = 'R')")
|
||||
createExtLibDDLString = paste0(createExtLibDDLString, " FROM (content = ", content, ") WITH (LANGUAGE = '", languageName,"')")
|
||||
}
|
||||
else
|
||||
{
|
||||
createExtLibDDLString = paste0(createExtLibDDLString, " FROM (content = '", content, "') WITH (LANGUAGE = 'R')")
|
||||
createExtLibDDLString = paste0(createExtLibDDLString, " FROM (content = '", content, "') WITH (LANGUAGE = '", languageName,"')")
|
||||
}
|
||||
|
||||
helper_ExecuteSQLDDL(connectionString = connectionString, sqlDDL = createExtLibDDLString)
|
||||
}
|
||||
|
||||
helper_callDummySPEES <- function(connectionString)
|
||||
helper_callDummySPEES <- function(connectionString, languageName="R")
|
||||
{
|
||||
cat(sprintf("\nINFO: call dummy sp_execute_external_library to trigger install.\r\n"))
|
||||
speesStr = "EXECUTE sp_execute_external_script
|
||||
@LANGUAGE = N'R',
|
||||
@SCRIPT = N'invisible(NULL)'"
|
||||
speesStr = paste0("EXECUTE sp_execute_external_script
|
||||
@LANGUAGE = N'", languageName,"',
|
||||
@SCRIPT = N'invisible(NULL)'")
|
||||
|
||||
sqlmlutils:::execute(connectionString, speesStr)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ pip install sqlmlutils
|
|||
```
|
||||
To install from file:
|
||||
```
|
||||
pip install Python/dist/sqlmlutils-1.0.1.zip
|
||||
pip install Python/dist/sqlmlutils-1.1.0.zip
|
||||
```
|
||||
|
||||
R:
|
||||
|
|
@ -29,7 +29,7 @@ Windows:
|
|||
From command prompt, run
|
||||
```
|
||||
R.exe -e "install.packages('odbc')"
|
||||
R.exe CMD INSTALL dist/sqlmlutils_0.7.4.zip
|
||||
R.exe CMD INSTALL dist/sqlmlutils_1.0.0.zip
|
||||
```
|
||||
OR
|
||||
To build a new package file and install, run
|
||||
|
|
@ -40,7 +40,7 @@ To build a new package file and install, run
|
|||
Linux
|
||||
```
|
||||
R.exe -e "install.packages('odbc')"
|
||||
R.exe CMD INSTALL dist/sqlmlutils_0.7.4.tar.gz
|
||||
R.exe CMD INSTALL dist/sqlmlutils_1.0.0.tar.gz
|
||||
```
|
||||
|
||||
# Details
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче