Merge pull request #48 from microsoft/JonathanZhu11/outFile
Jonathan zhu11/out file
This commit is contained in:
Jonathan Zhu
GitHub
Коммит
0703884ee8
|
|
@ -4,20 +4,19 @@ sqlmlutils is a python package to help execute Python code on a SQL Server machi
|
|||
|
||||
# Installation
|
||||
|
||||
Download the zip package file from the dist folder.
|
||||
From a command prompt, run
|
||||
```
|
||||
python.exe -m pip install --upgrade --upgrade-strategy only-if-needed dist/sqlmlutils-0.6.1.zip
|
||||
```
|
||||
OR
|
||||
To build a new package file and install (windows), run
|
||||
```
|
||||
.\buildandinstall.cmd
|
||||
python.exe -m pip install --upgrade --upgrade-strategy only-if-needed sqlmlutils-0.7.0.zip
|
||||
```
|
||||
|
||||
Note: If you encounter errors installing the pymssql dependency and your client is a Windows machine, consider
|
||||
installing the .whl file at the below link (download the file for your Python version and run pip install):
|
||||
https://www.lfd.uci.edu/~gohlke/pythonlibs/#pymssql
|
||||
|
||||
|
||||
If you are developing on your own branch and want to rebuild and install the package, you can use the buildandinstall.cmd script that is included.
|
||||
|
||||
# Getting started
|
||||
|
||||
Shown below are the important functions sqlmlutils provides:
|
||||
|
|
|
|||
|
|
@ -1,2 +1,2 @@
|
|||
python.exe setup.py sdist --formats=zip
|
||||
python.exe -m pip install --upgrade --upgrade-strategy only-if-needed dist\sqlmlutils-0.6.1.zip
|
||||
python.exe -m pip install --upgrade --upgrade-strategy only-if-needed dist\sqlmlutils-0.7.0.zip
|
||||
|
|
|
|||
Двоичный файл не отображается.
|
|
@ -6,7 +6,7 @@ from distutils.core import setup
|
|||
setup(
|
||||
name='sqlmlutils',
|
||||
packages=['sqlmlutils', 'sqlmlutils/packagemanagement'],
|
||||
version='0.6.1',
|
||||
version='0.7.0',
|
||||
url='https://github.com/Microsoft/sqlmlutils',
|
||||
license='MIT License',
|
||||
description='A client side package for working with SQL Machine Learning Python Services. '
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ _ENV_NAME_SHARED_PATH = "MRS_EXTLIB_SHARED_PATH"
|
|||
|
||||
|
||||
def _is_dist_info_file(name, file):
|
||||
return re.match(name + r'-.*egg', file) or re.match(name + r'-.*dist-info', file)
|
||||
return re.match(name + r"-.*egg", file) or re.match(name + r"-.*dist-info", file)
|
||||
|
||||
|
||||
def _is_package_match(package_name, file):
|
||||
|
|
@ -34,8 +34,8 @@ def _is_package_match(package_name, file):
|
|||
("-" in package_name and
|
||||
(package_name.split("-")[0] == file or _is_dist_info_file(package_name.replace("-", "_"), file)))
|
||||
|
||||
def package_files_in_scope(scope='private'):
|
||||
envdir = _ENV_NAME_SHARED_PATH if scope == 'public' or os.environ.get(_ENV_NAME_USER_PATH, "") == "" \
|
||||
def package_files_in_scope(scope="private"):
|
||||
envdir = _ENV_NAME_SHARED_PATH if scope == "public" or os.environ.get(_ENV_NAME_USER_PATH, "") == "" \
|
||||
else _ENV_NAME_USER_PATH
|
||||
path = os.environ.get(envdir, "")
|
||||
if os.path.isdir(path):
|
||||
|
|
@ -45,7 +45,7 @@ def package_files_in_scope(scope='private'):
|
|||
def package_exists_in_scope(sql_package_name: str, scope=None) -> bool:
|
||||
if scope is None:
|
||||
# default to user path for every user but DBOs
|
||||
scope = 'public' if (os.environ.get(_ENV_NAME_USER_PATH, "") == "") else 'private'
|
||||
scope = "public" if (os.environ.get(_ENV_NAME_USER_PATH, "") == "") else "private"
|
||||
package_files = package_files_in_scope(scope)
|
||||
return any([_is_package_match(sql_package_name, package_file) for package_file in package_files])
|
||||
|
||||
|
|
@ -57,11 +57,7 @@ assert package_exists_in_scope("{sqlpkgname}", "{scopestr}")
|
|||
|
||||
@property
|
||||
def base_script(self) -> str:
|
||||
return """
|
||||
-- Wrap this in a transaction
|
||||
DECLARE @TransactionName varchar(30) = 'SqlPackageTransaction';
|
||||
BEGIN TRAN @TransactionName
|
||||
|
||||
return """
|
||||
-- Drop the library if it exists
|
||||
BEGIN TRY
|
||||
DROP EXTERNAL LIBRARY [{sqlpkgname}] {authorization}
|
||||
|
|
@ -84,13 +80,9 @@ BEGIN TRY
|
|||
exec sp_execute_external_script
|
||||
@language = N'Python',
|
||||
@script = %s
|
||||
-- Installation succeeded, commit the transaction
|
||||
COMMIT TRAN @TransactionName
|
||||
print('Package successfully installed.')
|
||||
END TRY
|
||||
BEGIN CATCH
|
||||
-- Installation failed, rollback the transaction
|
||||
ROLLBACK TRAN @TransactionName
|
||||
print('Package installation failed.');
|
||||
THROW;
|
||||
END CATCH
|
||||
|
|
|
|||
|
|
@ -8,23 +8,9 @@ import re
|
|||
_ENV_NAME_USER_PATH = "MRS_EXTLIB_USER_PATH"
|
||||
_ENV_NAME_SHARED_PATH = "MRS_EXTLIB_SHARED_PATH"
|
||||
|
||||
|
||||
def show_installed_packages():
|
||||
from distutils.version import LooseVersion
|
||||
import pip
|
||||
if LooseVersion(pip.__version__) > LooseVersion("10"):
|
||||
from pip._internal.operations import freeze
|
||||
else:
|
||||
from pip.operations import freeze
|
||||
|
||||
packages = []
|
||||
for package in list(freeze.freeze()):
|
||||
val = package.split("==")
|
||||
name = val[0]
|
||||
version = val[1]
|
||||
packages.append((name, version))
|
||||
return packages
|
||||
|
||||
import pkg_resources
|
||||
return [(d.project_name, d.version) for d in pkg_resources.working_set]
|
||||
|
||||
def get_server_info():
|
||||
from distutils.version import LooseVersion
|
||||
|
|
|
|||
|
|
@ -30,7 +30,8 @@ class SQLPackageManager:
|
|||
upgrade: bool = False,
|
||||
version: str = None,
|
||||
install_dependencies: bool = True,
|
||||
scope: Scope = None):
|
||||
scope: Scope = None,
|
||||
out_file: str = None):
|
||||
"""Install Python package into a SQL Server Python Services environment using pip.
|
||||
|
||||
:param package: Package name to install on the SQL Server. Can also be a filename.
|
||||
|
|
@ -44,6 +45,7 @@ class SQLPackageManager:
|
|||
This installs packages into a private path for the SQL principal you connect as. If your principal has the
|
||||
db_owner role, you can also specify scope as public. This will install packages into a public path for all
|
||||
users. Note: if you connect as dbo, you can only install packages into the public path.
|
||||
:param out_file: INSTEAD of running the actual installation, print the t-sql commands to a text file to use as script.
|
||||
|
||||
>>> from sqlmlutils import ConnectionInfo, SQLPythonExecutor, SQLPackageManager
|
||||
>>> connection = ConnectionInfo(server="localhost", database="AirlineTestsDB")
|
||||
|
|
@ -63,16 +65,18 @@ class SQLPackageManager:
|
|||
if not install_dependencies:
|
||||
raise ValueError("Dependencies will always be installed - "
|
||||
"single package install without dependencies not yet supported.")
|
||||
|
||||
if scope is None:
|
||||
scope = self._get_default_scope()
|
||||
|
||||
if os.path.isfile(package):
|
||||
self._install_from_file(package, scope, upgrade)
|
||||
self._install_from_file(package, scope, upgrade, out_file=out_file)
|
||||
else:
|
||||
self._install_from_pypi(package, upgrade, version, install_dependencies, scope)
|
||||
self._install_from_pypi(package, upgrade, version, install_dependencies, scope, out_file=out_file)
|
||||
|
||||
def uninstall(self, package_name: str, scope: Scope = None):
|
||||
def uninstall(self,
|
||||
package_name: str,
|
||||
scope: Scope = None,
|
||||
out_file: str = None):
|
||||
"""Remove Python package from a SQL Server Python environment.
|
||||
|
||||
:param package_name: Package name to remove on the SQL Server.
|
||||
|
|
@ -80,13 +84,14 @@ class SQLPackageManager:
|
|||
This uninstalls packages from a private path for the SQL principal you connect as. If your principal has the
|
||||
db_owner role, you can also specify scope as public. This will uninstall packages from a public path for all
|
||||
users. Note: if you connect as dbo, you can only uninstall packages from the public path.
|
||||
:param out_file: INSTEAD of running the actual installation, print the t-sql commands to a text file to use as script.
|
||||
"""
|
||||
|
||||
|
||||
if scope is None:
|
||||
scope = self._get_default_scope()
|
||||
|
||||
print("Uninstalling " + package_name + " only, not dependencies")
|
||||
self._drop_sql_package(package_name, scope)
|
||||
self._drop_sql_package(package_name, scope, out_file)
|
||||
|
||||
def list(self):
|
||||
"""List packages installed on server, similar to output of pip freeze.
|
||||
|
|
@ -120,9 +125,9 @@ class SQLPackageManager:
|
|||
ORDER BY elib.name ASC;".format(1 if scope == Scope.private_scope() else 0)
|
||||
return self._pyexecutor.execute_sql_query(query, owner)
|
||||
|
||||
def _drop_sql_package(self, sql_package_name: str, scope: Scope):
|
||||
def _drop_sql_package(self, sql_package_name: str, scope: Scope, out_file: str):
|
||||
builder = DropLibraryBuilder(sql_package_name=sql_package_name, scope=scope)
|
||||
execute_query(builder, self._connection_info)
|
||||
execute_query(builder, self._connection_info, out_file)
|
||||
|
||||
# TODO: Support not dependencies
|
||||
def _install_from_pypi(self,
|
||||
|
|
@ -130,7 +135,8 @@ class SQLPackageManager:
|
|||
upgrade: bool = False,
|
||||
version: str = None,
|
||||
install_dependencies: bool = True,
|
||||
scope: Scope = Scope.private_scope()):
|
||||
scope: Scope = Scope.private_scope(),
|
||||
out_file: str = None):
|
||||
|
||||
if not install_dependencies:
|
||||
raise ValueError("Dependencies will always be installed - "
|
||||
|
|
@ -142,9 +148,9 @@ class SQLPackageManager:
|
|||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
pipdownloader = PipDownloader(self._connection_info, temporary_directory, target_package)
|
||||
target_package_file = pipdownloader.download_single()
|
||||
self._install_from_file(target_package_file, scope, upgrade)
|
||||
self._install_from_file(target_package_file, scope, upgrade, out_file=out_file)
|
||||
|
||||
def _install_from_file(self, target_package_file: str, scope: Scope, upgrade: bool = False):
|
||||
def _install_from_file(self, target_package_file: str, scope: Scope, upgrade: bool = False, out_file: str = None):
|
||||
name = get_package_name_from_file(target_package_file)
|
||||
version = get_package_version_from_file(target_package_file)
|
||||
|
||||
|
|
@ -164,10 +170,10 @@ class SQLPackageManager:
|
|||
# Resolve which package dependencies need to be installed or upgraded on server.
|
||||
required_installs = resolver.get_required_installs(target_package_requirements)
|
||||
dependencies_to_install = self._get_required_files_to_install(requirements_downloaded, required_installs)
|
||||
|
||||
self._install_many(target_package_file, dependencies_to_install, scope, out_file=out_file)
|
||||
|
||||
self._install_many(target_package_file, dependencies_to_install, scope)
|
||||
|
||||
def _install_many(self, target_package_file: str, dependency_files, scope: Scope):
|
||||
def _install_many(self, target_package_file: str, dependency_files, scope: Scope, out_file:str=None):
|
||||
target_name = get_package_name_from_file(target_package_file)
|
||||
|
||||
with SQLQueryExecutor(connection=self._connection_info) as sqlexecutor:
|
||||
|
|
@ -175,15 +181,15 @@ class SQLPackageManager:
|
|||
transaction.begin()
|
||||
try:
|
||||
for pkgfile in dependency_files:
|
||||
self._install_single(sqlexecutor, pkgfile, scope)
|
||||
self._install_single(sqlexecutor, target_package_file, scope, True)
|
||||
self._install_single(sqlexecutor, pkgfile, scope, out_file=out_file)
|
||||
self._install_single(sqlexecutor, target_package_file, scope, True, out_file=out_file)
|
||||
transaction.commit()
|
||||
except Exception:
|
||||
except Exception as e:
|
||||
transaction.rollback()
|
||||
raise RuntimeError("Package installation failed, installed dependencies were rolled back.")
|
||||
raise RuntimeError("Package installation failed, installed dependencies were rolled back.") from e
|
||||
|
||||
@staticmethod
|
||||
def _install_single(sqlexecutor: SQLQueryExecutor, package_file: str, scope: Scope, is_target=False):
|
||||
def _install_single(sqlexecutor: SQLQueryExecutor, package_file: str, scope: Scope, is_target=False, out_file: str=None):
|
||||
name = get_package_name_from_file(package_file)
|
||||
version = get_package_version_from_file(package_file)
|
||||
|
||||
|
|
@ -193,7 +199,7 @@ class SQLPackageManager:
|
|||
zipf.write(package_file, os.path.basename(package_file))
|
||||
|
||||
builder = CreateLibraryBuilder(pkg_name=name, pkg_filename=prezip, scope=scope)
|
||||
sqlexecutor.execute(builder)
|
||||
sqlexecutor.execute(builder, out_file=out_file, getResults=False)
|
||||
|
||||
@staticmethod
|
||||
def _get_required_files_to_install(pkgfiles, requirements):
|
||||
|
|
|
|||
|
|
@ -14,9 +14,9 @@ It is mostly setup to work with SQLBuilder objects as defined in sqlbuilder.
|
|||
# This function is best used to execute_function_in_sql a one off query
|
||||
# (the SQL connection is closed after the query completes).
|
||||
# If you need to keep the SQL connection open in between queries, you can use the _SQLQueryExecutor class below.
|
||||
def execute_query(builder, connection: ConnectionInfo):
|
||||
def execute_query(builder, connection: ConnectionInfo, out_file:str=None):
|
||||
with SQLQueryExecutor(connection=connection) as executor:
|
||||
return executor.execute(builder)
|
||||
return executor.execute(builder, out_file=out_file)
|
||||
|
||||
|
||||
def execute_raw_query(conn: ConnectionInfo, query, params=()):
|
||||
|
|
@ -38,15 +38,38 @@ class SQLQueryExecutor:
|
|||
def __init__(self, connection: ConnectionInfo):
|
||||
self._connection = connection
|
||||
|
||||
def execute(self, builder: SQLBuilder):
|
||||
def execute(self, builder: SQLBuilder, out_file=None, getResults=True):
|
||||
try:
|
||||
self._mssqlconn.set_msghandler(_sql_msg_handler)
|
||||
self._mssqlconn.execute_query(builder.base_script, builder.params)
|
||||
return [row for row in self._mssqlconn]
|
||||
if out_file is not None:
|
||||
with open(out_file,"a") as f:
|
||||
if builder.params is not None:
|
||||
script = builder.base_script.replace("%s", "N'%s'")
|
||||
f.write(script % builder.params)
|
||||
else:
|
||||
f.write(builder.base_script)
|
||||
f.write("GO\n")
|
||||
f.write("-----------------------------")
|
||||
else:
|
||||
self._mssqlconn.set_msghandler(_sql_msg_handler)
|
||||
if getResults:
|
||||
self._mssqlconn.execute_query(builder.base_script, builder.params)
|
||||
return [row for row in self._mssqlconn]
|
||||
else:
|
||||
self._mssqlconn.execute_non_query(builder.base_script, builder.params)
|
||||
return []
|
||||
except Exception as e:
|
||||
raise RuntimeError(str.format("Error in SQL Execution: {error}", error=str(e)))
|
||||
raise RuntimeError("Error in SQL Execution") from e
|
||||
|
||||
def execute_query(self, query, params):
|
||||
def execute_query(self, query, params, out_file=None):
|
||||
if out_file is not None:
|
||||
with open(out_file, "a") as f:
|
||||
if params is not None:
|
||||
script = query.replace("%s", "'%s'")
|
||||
f.write(script % params)
|
||||
else:
|
||||
f.write(query)
|
||||
f.write("GO\n")
|
||||
f.write("-----------------------------")
|
||||
self._mssqlconn.execute_query(query, params)
|
||||
return [row for row in self._mssqlconn]
|
||||
|
||||
|
|
|
|||
|
|
@ -7,19 +7,19 @@ Currently, only the R version of sqlmlutils is supported in Azure SQL Database.
|
|||
|
||||
# Installation
|
||||
|
||||
To install sqlmlutils from this repository, run the following commands from the root folder:
|
||||
To install sqlmlutils from this repository, simply download the zip package for the language you want from the corresponding dist folder: Python/dist or R/dist.
|
||||
|
||||
Python:
|
||||
1. If your client is a Linux machine, you can skip this step. If your client is a Windows machine: go to https://www.lfd.uci.edu/~gohlke/pythonlibs/#pymssql and download the correct version of pymssql for your client. Run ```pip install pymssql-2.1.4.dev5-cpXX-cpXXm-win_amd64.whl``` on that file to install pymssql.
|
||||
2. Run
|
||||
```
|
||||
python.exe -m pip install --upgrade --upgrade-strategy only-if-needed Python/dist/sqlmlutils-0.6.0.zip
|
||||
python.exe -m pip install --upgrade --upgrade-strategy only-if-needed sqlmlutils-0.7.0.zip
|
||||
```
|
||||
|
||||
R:
|
||||
```
|
||||
R -e "install.packages('RODBCext', repos='https://cran.microsoft.com')"
|
||||
R CMD INSTALL R/dist/sqlmlutils_0.7.1.zip
|
||||
R CMD INSTALL sqlmlutils_0.7.1.zip
|
||||
```
|
||||
|
||||
# Details
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче