Merge pull request #48 from microsoft/JonathanZhu11/outFile

Jonathan zhu11/out file

Python/README.md

@@ -4,20 +4,19 @@ sqlmlutils is a python package to help execute Python code on a SQL Server machi
 
 # Installation
 
+Download the zip package file from the dist folder.
 From a command prompt, run
 ```
-python.exe -m pip install --upgrade --upgrade-strategy only-if-needed dist/sqlmlutils-0.6.1.zip
+python.exe -m pip install --upgrade --upgrade-strategy only-if-needed sqlmlutils-0.7.0.zip
-```
-OR
-To build a new package file and install (windows), run
-```
-.\buildandinstall.cmd
 ```
 
 Note: If you encounter errors installing the pymssql dependency and your client is a Windows machine, consider 
 installing the .whl file at the below link (download the file for your Python version and run pip install): 
 https://www.lfd.uci.edu/~gohlke/pythonlibs/#pymssql
 
+
+If you are developing on your own branch and want to rebuild and install the package, you can use the buildandinstall.cmd script that is included.
+
 # Getting started
 
 Shown below are the important functions sqlmlutils provides:

Python/buildandinstall.cmd

@@ -1,2 +1,2 @@
 python.exe setup.py sdist --formats=zip
-python.exe -m pip install --upgrade --upgrade-strategy only-if-needed dist\sqlmlutils-0.6.1.zip
+python.exe -m pip install --upgrade --upgrade-strategy only-if-needed dist\sqlmlutils-0.7.0.zip

Python/setup.py

@@ -6,7 +6,7 @@ from distutils.core import setup
 setup(
     name='sqlmlutils',
     packages=['sqlmlutils', 'sqlmlutils/packagemanagement'],
-    version='0.6.1',
+    version='0.7.0',
     url='https://github.com/Microsoft/sqlmlutils',
     license='MIT License',
     description='A client side package for working with SQL Machine Learning Python Services. '

Python/sqlmlutils/packagemanagement/packagesqlbuilder.py

@@ -23,7 +23,7 @@ _ENV_NAME_SHARED_PATH = "MRS_EXTLIB_SHARED_PATH"
 
 
 def _is_dist_info_file(name, file):
-    return re.match(name + r'-.*egg', file) or re.match(name + r'-.*dist-info', file)
+    return re.match(name + r"-.*egg", file) or re.match(name + r"-.*dist-info", file)
 
 
 def _is_package_match(package_name, file):
@@ -34,8 +34,8 @@ def _is_package_match(package_name, file):
            ("-" in package_name and
             (package_name.split("-")[0] == file or _is_dist_info_file(package_name.replace("-", "_"), file)))
 
-def package_files_in_scope(scope='private'):
+def package_files_in_scope(scope="private"):
-    envdir = _ENV_NAME_SHARED_PATH if scope == 'public' or os.environ.get(_ENV_NAME_USER_PATH, "") == "" \
+    envdir = _ENV_NAME_SHARED_PATH if scope == "public" or os.environ.get(_ENV_NAME_USER_PATH, "") == "" \
         else _ENV_NAME_USER_PATH
     path = os.environ.get(envdir, "")
     if os.path.isdir(path):
@@ -45,7 +45,7 @@ def package_files_in_scope(scope='private'):
 def package_exists_in_scope(sql_package_name: str, scope=None) -> bool:
     if scope is None:
         # default to user path for every user but DBOs
-        scope = 'public' if (os.environ.get(_ENV_NAME_USER_PATH, "") == "") else 'private'
+        scope = "public" if (os.environ.get(_ENV_NAME_USER_PATH, "") == "") else "private"
     package_files = package_files_in_scope(scope)
     return any([_is_package_match(sql_package_name, package_file) for package_file in package_files])
 
@@ -57,11 +57,7 @@ assert package_exists_in_scope("{sqlpkgname}", "{scopestr}")
 
     @property
     def base_script(self) -> str:
-        return """ 
+        return """         
--- Wrap this in a transaction  
-DECLARE @TransactionName varchar(30) = 'SqlPackageTransaction';
-BEGIN TRAN @TransactionName
-        
 -- Drop the library if it exists
 BEGIN TRY
 DROP EXTERNAL LIBRARY [{sqlpkgname}] {authorization}
@@ -84,13 +80,9 @@ BEGIN TRY
     exec sp_execute_external_script
     @language = N'Python',
     @script = %s
-    -- Installation succeeded, commit the transaction
-    COMMIT TRAN @TransactionName
     print('Package successfully installed.')
 END TRY
 BEGIN CATCH
-    -- Installation failed, rollback the transaction
-    ROLLBACK TRAN @TransactionName
     print('Package installation failed.');
     THROW;
 END CATCH

Python/sqlmlutils/packagemanagement/servermethods.py

@@ -8,23 +8,9 @@ import re
 _ENV_NAME_USER_PATH = "MRS_EXTLIB_USER_PATH"
 _ENV_NAME_SHARED_PATH = "MRS_EXTLIB_SHARED_PATH"
 
-
 def show_installed_packages():
-    from distutils.version import LooseVersion
+    import pkg_resources
-    import pip
+    return [(d.project_name, d.version) for d in pkg_resources.working_set]
-    if LooseVersion(pip.__version__) > LooseVersion("10"):
-        from pip._internal.operations import freeze
-    else:
-        from pip.operations import freeze
-
-    packages = []
-    for package in list(freeze.freeze()):
-        val = package.split("==")
-        name = val[0]
-        version = val[1]
-        packages.append((name, version))
-    return packages
-
 
 def get_server_info():
     from distutils.version import LooseVersion

Python/sqlmlutils/packagemanagement/sqlpackagemanager.py

@@ -30,7 +30,8 @@ class SQLPackageManager:
                 upgrade: bool = False,
                 version: str = None,
                 install_dependencies: bool = True,
-                scope: Scope = None):
+                scope: Scope = None,
+                out_file: str = None):
         """Install Python package into a SQL Server Python Services environment using pip.
 
         :param package: Package name to install on the SQL Server. Can also be a filename.
@@ -44,6 +45,7 @@ class SQLPackageManager:
         This installs packages into a private path for the SQL principal you connect as. If your principal has the
         db_owner role, you can also specify scope as public. This will install packages into a public path for all
         users. Note: if you connect as dbo, you can only install packages into the public path.
+        :param out_file: INSTEAD of running the actual installation, print the t-sql commands to a text file to use as script.
 
         >>> from sqlmlutils import ConnectionInfo, SQLPythonExecutor, SQLPackageManager
         >>> connection = ConnectionInfo(server="localhost", database="AirlineTestsDB")
@@ -63,16 +65,18 @@ class SQLPackageManager:
         if not install_dependencies:
             raise ValueError("Dependencies will always be installed - "
                              "single package install without dependencies not yet supported.")
-        
         if scope is None:
             scope = self._get_default_scope()
         
         if os.path.isfile(package):
-            self._install_from_file(package, scope, upgrade)
+            self._install_from_file(package, scope, upgrade, out_file=out_file)
         else:
-            self._install_from_pypi(package, upgrade, version, install_dependencies, scope)
+            self._install_from_pypi(package, upgrade, version, install_dependencies, scope, out_file=out_file)
 
-    def uninstall(self, package_name: str, scope: Scope = None):
+    def uninstall(self, 
+                package_name: str, 
+                scope: Scope = None,
+                out_file: str = None):
         """Remove Python package from a SQL Server Python environment.
 
         :param package_name: Package name to remove on the SQL Server.
@@ -80,13 +84,14 @@ class SQLPackageManager:
         This uninstalls packages from a private path for the SQL principal you connect as. If your principal has the
         db_owner role, you can also specify scope as public. This will uninstall packages from a public path for all
         users. Note: if you connect as dbo, you can only uninstall packages from the public path.
+        :param out_file: INSTEAD of running the actual installation, print the t-sql commands to a text file to use as script.
         """
-        
+            
         if scope is None:
             scope = self._get_default_scope()
             
         print("Uninstalling " + package_name + " only, not dependencies")
-        self._drop_sql_package(package_name, scope)
+        self._drop_sql_package(package_name, scope, out_file)
 
     def list(self):
         """List packages installed on server, similar to output of pip freeze.
@@ -120,9 +125,9 @@ class SQLPackageManager:
                        ORDER BY elib.name ASC;".format(1 if scope == Scope.private_scope() else 0)
         return self._pyexecutor.execute_sql_query(query, owner)
 
-    def _drop_sql_package(self, sql_package_name: str, scope: Scope):
+    def _drop_sql_package(self, sql_package_name: str, scope: Scope, out_file: str):
         builder = DropLibraryBuilder(sql_package_name=sql_package_name, scope=scope)
-        execute_query(builder, self._connection_info)
+        execute_query(builder, self._connection_info, out_file)
 
     # TODO: Support not dependencies
     def _install_from_pypi(self,
@@ -130,7 +135,8 @@ class SQLPackageManager:
                            upgrade: bool = False,
                            version: str = None,
                            install_dependencies: bool = True,
-                           scope: Scope = Scope.private_scope()):
+                           scope: Scope = Scope.private_scope(),
+                           out_file: str = None):
 
         if not install_dependencies:
             raise ValueError("Dependencies will always be installed - "
@@ -142,9 +148,9 @@ class SQLPackageManager:
         with tempfile.TemporaryDirectory() as temporary_directory:
             pipdownloader = PipDownloader(self._connection_info, temporary_directory, target_package)
             target_package_file = pipdownloader.download_single()
-            self._install_from_file(target_package_file, scope, upgrade)
+            self._install_from_file(target_package_file, scope, upgrade, out_file=out_file)
 
-    def _install_from_file(self, target_package_file: str, scope: Scope, upgrade: bool = False):
+    def _install_from_file(self, target_package_file: str, scope: Scope, upgrade: bool = False, out_file: str = None):
         name = get_package_name_from_file(target_package_file)
         version = get_package_version_from_file(target_package_file)
 
@@ -164,10 +170,10 @@ class SQLPackageManager:
             # Resolve which package dependencies need to be installed or upgraded on server.
             required_installs = resolver.get_required_installs(target_package_requirements)
             dependencies_to_install = self._get_required_files_to_install(requirements_downloaded, required_installs)
+            
+            self._install_many(target_package_file, dependencies_to_install, scope, out_file=out_file)
 
-            self._install_many(target_package_file, dependencies_to_install, scope)
+    def _install_many(self, target_package_file: str, dependency_files, scope: Scope, out_file:str=None):
-
-    def _install_many(self, target_package_file: str, dependency_files, scope: Scope):
         target_name = get_package_name_from_file(target_package_file)
 
         with SQLQueryExecutor(connection=self._connection_info) as sqlexecutor:
@@ -175,15 +181,15 @@ class SQLPackageManager:
             transaction.begin()
             try:
                 for pkgfile in dependency_files:
-                    self._install_single(sqlexecutor, pkgfile, scope)
+                    self._install_single(sqlexecutor, pkgfile, scope, out_file=out_file)
-                self._install_single(sqlexecutor, target_package_file, scope, True)
+                self._install_single(sqlexecutor, target_package_file, scope, True, out_file=out_file)
                 transaction.commit()
-            except Exception:
+            except Exception as e:
                 transaction.rollback()
-                raise RuntimeError("Package installation failed, installed dependencies were rolled back.")
+                raise RuntimeError("Package installation failed, installed dependencies were rolled back.") from e
 
     @staticmethod
-    def _install_single(sqlexecutor: SQLQueryExecutor, package_file: str, scope: Scope, is_target=False):
+    def _install_single(sqlexecutor: SQLQueryExecutor, package_file: str, scope: Scope, is_target=False, out_file: str=None):
         name = get_package_name_from_file(package_file)
         version = get_package_version_from_file(package_file)
 
@@ -193,7 +199,7 @@ class SQLPackageManager:
                 zipf.write(package_file, os.path.basename(package_file))
 
             builder = CreateLibraryBuilder(pkg_name=name, pkg_filename=prezip, scope=scope)
-            sqlexecutor.execute(builder)
+            sqlexecutor.execute(builder, out_file=out_file, getResults=False)
 
     @staticmethod
     def _get_required_files_to_install(pkgfiles, requirements):

Python/sqlmlutils/sqlqueryexecutor.py

@@ -14,9 +14,9 @@ It is mostly setup to work with SQLBuilder objects as defined in sqlbuilder.
 # This function is best used to execute_function_in_sql a one off query
 # (the SQL connection is closed after the query completes).
 # If you need to keep the SQL connection open in between queries, you can use the _SQLQueryExecutor class below.
-def execute_query(builder, connection: ConnectionInfo):
+def execute_query(builder, connection: ConnectionInfo, out_file:str=None):
     with SQLQueryExecutor(connection=connection) as executor:
-        return executor.execute(builder)
+        return executor.execute(builder, out_file=out_file)
 
 
 def execute_raw_query(conn: ConnectionInfo, query, params=()):
@@ -38,15 +38,38 @@ class SQLQueryExecutor:
     def __init__(self, connection: ConnectionInfo):
         self._connection = connection
 
-    def execute(self, builder: SQLBuilder):
+    def execute(self, builder: SQLBuilder, out_file=None, getResults=True):
         try:
-            self._mssqlconn.set_msghandler(_sql_msg_handler)
+            if out_file is not None:
-            self._mssqlconn.execute_query(builder.base_script, builder.params)
+                with open(out_file,"a") as f:
-            return [row for row in self._mssqlconn]
+                    if builder.params is not None:
+                        script = builder.base_script.replace("%s", "N'%s'")
+                        f.write(script % builder.params)
+                    else:
+                        f.write(builder.base_script)
+                    f.write("GO\n")
+                    f.write("-----------------------------")
+            else:    
+                self._mssqlconn.set_msghandler(_sql_msg_handler)
+                if getResults:
+                    self._mssqlconn.execute_query(builder.base_script, builder.params)
+                    return [row for row in self._mssqlconn]
+                else:
+                    self._mssqlconn.execute_non_query(builder.base_script, builder.params)
+                    return []
         except Exception as e:
-            raise RuntimeError(str.format("Error in SQL Execution: {error}", error=str(e)))
+            raise RuntimeError("Error in SQL Execution") from e
 
-    def execute_query(self, query, params):
+    def execute_query(self, query, params, out_file=None):
+        if out_file is not None:
+            with open(out_file, "a") as f:
+                if params is not None:
+                    script = query.replace("%s", "'%s'")
+                    f.write(script % params)
+                else:
+                    f.write(query)
+                f.write("GO\n")
+                f.write("-----------------------------")
         self._mssqlconn.execute_query(query, params)
         return [row for row in self._mssqlconn]
 

README.md

@@ -7,19 +7,19 @@ Currently, only the R version of sqlmlutils is supported in Azure SQL Database.
 
 # Installation
 
-To install sqlmlutils from this repository, run the following commands from the root folder:
+To install sqlmlutils from this repository, simply download the zip package for the language you want from the corresponding dist folder: Python/dist or R/dist.
 
 Python:
 1. If your client is a Linux machine, you can skip this step. If your client is a Windows machine: go to https://www.lfd.uci.edu/~gohlke/pythonlibs/#pymssql and download the correct version of pymssql for your client. Run ```pip install pymssql-2.1.4.dev5-cpXX-cpXXm-win_amd64.whl``` on that file to install pymssql.
 2. Run
 ```
-python.exe -m pip install --upgrade --upgrade-strategy only-if-needed Python/dist/sqlmlutils-0.6.0.zip
+python.exe -m pip install --upgrade --upgrade-strategy only-if-needed sqlmlutils-0.7.0.zip
 ```
 
 R:
 ```
 R -e "install.packages('RODBCext', repos='https://cran.microsoft.com')"
-R CMD INSTALL R/dist/sqlmlutils_0.7.1.zip
+R CMD INSTALL sqlmlutils_0.7.1.zip
 ```
 
 # Details