Merging dev to master for release 1.0.0a20 (#141)

* Use SqlToolsService built on .NET Core 2.0 and a build script updates (#131) * Bump version to 1.0.0a19 * Use .NET Core 2.0 RTM built sqltoolsservice * Add build script to upload to azure blob storage * Upgrade to VS 2017 * Remove 3.3 as supported Python version * Fix perf issue where main event loop takes 100% of CPU (#132) Fix perf issue where main event loop takes 100% of CPU We have a 2 threads: Thread #1 runs in a loop polling the response queue Thread #2 runs in a loop decoding responses from the sqltoolsservice over stdout and posting them to the response queue Since thread #1 doesn't sleep, it's takes 100% CPU. In addition, running python 2.7 on windows, #2 doesn’t preempt the CPU due to #1 taking all of the CPU cycles, so no response is processed. Fix is simple – thread #1 needs to sleep so thread #2 can get scheduled and get it’s work done. * Refine event loop perf fix in main.py Refine event loop perf fix in main.py * Fixing regular expression Previous regex would result in release:a1 and release_version: 12. Modified the regex for part Release to only pick up lower case letters. * Adding missing forward slash on test pypi url * fixing typos/grammar (#138) fixing typos/grammar. * Updating to release version 1.0.0a20.
2017-09-01 18:56:18 -04:00 · 2017-09-01 18:56:18 -04:00 · ad37035666
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@ -1,6 +1,6 @@
 [bumpversion]
-current_version = 1.0.0a18
-parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)((?P<release>.*))(?P<release_version>\d+)
+current_version = 1.0.0a20
+parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)((?P<release>[a-z]+))(?P<release_version>\d+)
 serialize = 
 	{major}.{minor}.{patch}{release}{release_version}

--- a/.travis.yml
+++ b/.travis.yml
@ -10,8 +10,6 @@ matrix:
      env: TOXENV=py27
    - os: linux
      python: "2.7"
-    - os: linux
-      python: "3.3"
    - os: linux
      python: "3.4"
    - os: linux
--- a/appveyor.yml
+++ b/appveyor.yml
@ -3,8 +3,6 @@ environment:
  matrix:
    - TOXENV: "py27"
      PYTHON: "C:\\Python27"
-    - TOXENV: "py33"
-      PYTHON: "C:\\Python33"
    - TOXENV: "py34"
      PYTHON: "C:\\Python34"
    - TOXENV: "py35"
--- a/build.py
+++ b/build.py
@ -0,0 +1,124 @@
+#!/usr/bin/env python
+
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+from __future__ import print_function
+import os
+import re
+import sys
+import tempfile
+import utility
+from azure.storage.blob import BlockBlobService, ContentSettings
+
+AZURE_STORAGE_CONNECTION_STRING = os.environ.get('AZURE_STORAGE_CONNECTION_STRING')
+BLOB_CONTAINER_NAME = 'simple'
+UPLOADED_PACKAGE_LINKS = [] 
+
+
+def print_heading(heading, f=None):
+    print('{0}\n{1}\n{0}'.format('=' * len(heading), heading), file=f)
+
+
+def upload_index_file(service, blob_name, title, links):
+    print('Uploading index file {}'.format(blob_name))
+    service.create_blob_from_text(
+        container_name=BLOB_CONTAINER_NAME,
+        blob_name=blob_name,
+        text="<html><head><title>{0}</title></head><body><h1>{0}</h1>{1}</body></html>"
+            .format(title, '\n'.join(
+                ['<a href="{0}">{0}</a><br/>'.format(link) for link in links])),
+        content_settings=ContentSettings(
+            content_type='text/html',
+            content_disposition=None,
+            content_encoding=None,
+            content_language=None,
+            content_md5=None,
+            cache_control=None
+        )
+    )
+
+
+def gen_pkg_index_html(service, pkg_name):
+    links = []
+    index_file_name = pkg_name+'/'
+    for blob in list(service.list_blobs(BLOB_CONTAINER_NAME, prefix=index_file_name)):
+        if blob.name == index_file_name:
+            # Exclude the index file from being added to the list
+            continue
+        links.append(blob.name.replace(index_file_name, ''))
+    upload_index_file(service, index_file_name, 'Links for {}'.format(pkg_name), links)
+    UPLOADED_PACKAGE_LINKS.append(index_file_name)
+
+
+def upload_package(service, file_path, pkg_name):
+    print('Uploading {}'.format(file_path))
+    file_name = os.path.basename(file_path)
+    blob_name = '{}/{}'.format(pkg_name, file_name)
+    service.create_blob_from_path(
+        container_name=BLOB_CONTAINER_NAME,
+        blob_name=blob_name,
+        file_path=file_path
+    )
+    gen_pkg_index_html(service, pkg_name)
+
+
+def build(options):
+
+    supported_actions = ['nightly']
+    action = None
+
+    if len(options) >= 1:
+        if options[0] not in supported_actions:
+            print('Please provide a supported action {}.'.format(supported_actions))
+            return
+        action = options[0]
+
+    if action == 'nightly':
+        assert AZURE_STORAGE_CONNECTION_STRING, 'Set AZURE_STORAGE_CONNECTION_STRING environment variable'
+
+    print_heading('Cleanup')
+
+    # clean
+    utility.clean_up(utility.MSSQLSCRIPTER_DIST_DIRECTORY)
+    utility.clean_up(utility.MSSQLTOOLSSERVICE_DIST_DIRECTORY)
+    utility.cleaun_up_egg_info_sub_directories(utility.ROOT_DIR)
+    utility.cleaun_up_egg_info_sub_directories(utility.MSSQLTOOLSSERVICE_DIRECTORY)
+    
+    print_heading('Running setup')
+    
+    # install general requirements.
+    utility.exec_command('pip install -r dev_requirements.txt', utility.ROOT_DIR)
+    
+    print_heading('Running mssql-scripter tests')
+    utility.exec_command('tox', utility.ROOT_DIR, continue_on_error = False)
+    
+    print_heading('Building mssql-scripter pip package')
+    utility.exec_command('python setup.py check -r -s sdist', utility.ROOT_DIR, continue_on_error = False)
+    
+    print_heading('Building mssqltoolsservice pip package')
+    utility.exec_command('python buildwheels.py', utility.MSSQLTOOLSSERVICE_DIRECTORY, continue_on_error = False)
+
+    if action == 'nightly':
+        blob_service = BlockBlobService(connection_string=AZURE_STORAGE_CONNECTION_STRING)
+    
+        print_heading('Uploading packages to blob storage ')
+        for pkg in os.listdir(utility.MSSQLSCRIPTER_DIST_DIRECTORY):
+            pkg_path = os.path.join(utility.MSSQLSCRIPTER_DIST_DIRECTORY, pkg)
+            print('Uploading package {}'.format(pkg_path))
+            upload_package(blob_service, pkg_path, 'mssql-scripter')
+    
+        for pkg in os.listdir(utility.MSSQLTOOLSSERVICE_DIST_DIRECTORY):
+            pkg_path = os.path.join(utility.MSSQLTOOLSSERVICE_DIST_DIRECTORY, pkg)
+            pkg_name = os.path.basename(pkg_path).split('-')[0].replace('_', '-').lower()
+            print('Uploading package {}'.format(pkg_name))
+            upload_package(blob_service, pkg_path, pkg_name)
+
+        # Upload the final index file
+        upload_index_file(blob_service, 'index.html', 'Simple Index', UPLOADED_PACKAGE_LINKS)
+
+
+if __name__ == '__main__':
+    build(sys.argv[1:])
--- a/dev_requirements.txt
+++ b/dev_requirements.txt
@ -12,3 +12,4 @@ pytest >= 3.0.7
 pytest-cov >= 2.5.1
 readme_renderer >= 17.2
 docutils >= 0.13.1
+azure-storage >= 0.33.0
--- a/dev_setup.py
+++ b/dev_setup.py
@ -12,18 +12,16 @@ import os
 import setup
 import utility

-root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), '..'))
-
 print('Running dev setup...')
-print('Root directory \'{}\'\n'.format(root_dir))
+print('Root directory \'{}\'\n'.format(utility.ROOT_DIR))

 # install general requirements.
-utility.exec_command('pip install -r dev_requirements.txt', root_dir)
+utility.exec_command('pip install -r dev_requirements.txt', utility.ROOT_DIR)

 # install mssqltoolsservice if this platform supports it.
 mssqltoolsservice_package_name = os.environ['MSSQLTOOLSSERVICE_PACKAGE_NAME']
 print('Installing {}...'.format(mssqltoolsservice_package_name))
 # mssqltoolsservice package name is retrieved from environment variable set by setup.py.
-utility.exec_command('pip install {}'.format(mssqltoolsservice_package_name), root_dir)
+utility.exec_command('pip install {}'.format(mssqltoolsservice_package_name), utility.ROOT_DIR)

 print('Finished dev setup.')
--- a/doc/installation_guide.md
+++ b/doc/installation_guide.md
@ -1,7 +1,7 @@
 # Installation Guide

 ## Quick Start
-mssql-scritper is installed via pip.  If you know  pip, you can install mssql-scripter using command
+mssql-scripter is installed via pip.  If you know pip, you can install mssql-scripter using command
 ```shell
 $ pip install mssql-scripter 
 ```
@ -70,7 +70,7 @@ $ sudo apt-get install python-pip
 $ sudo pip install --upgrade pip
 ```

-Install mssql-scritper using command:
+Install mssql-scripter using command:

 ```shell
 $ sudo pip install mssql-scripter 
@ -103,7 +103,7 @@ More information can be found at:

 - [Development guide](development_guide.md#Environment_Setup)

-## Error: Could not find version that satifies the requirement mssql-scripter
+## Error: Could not find version that satisfies the requirement mssql-scripter
 If you see the above error running `pip install mssql-scripter`, this means the pip version used is out-of-date.  Upgrade pip using the command:
 ```shell
 $ sudo apt-get install python-pip
@ -141,7 +141,7 @@ $ sudo apt-get install libunwind8
 ```

 ### Debian 8
-The file `/etc/apt/sources.list' needs to updated with the following line
+The file `/etc/apt/sources.list' needs to be updated with the following line
 ```
 deb http://ftp.us.debian.org/debian/ jessie main
 ```
--- a/doc/pypi_release_steps.md
+++ b/doc/pypi_release_steps.md
@ -86,7 +86,7 @@ bumpversion release_version    ->  1.0.0a<b>1</b>
 		    pypitest
 		 
 		[pypitest]
-		repository = https://test.pypi.org/legacy
+		repository = https://test.pypi.org/legacy/
 		username = your_username
 		password = your_password
        ```
--- a/mssqlscripter/init.py
+++ b/mssqlscripter/init.py
@ -3,4 +3,4 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------

-__version__ = '1.0.0a18'
+__version__ = '1.0.0a20'
--- a/mssqlscripter/jsonrpc/contracts/tests/test_scripting.py
+++ b/mssqlscripter/jsonrpc/contracts/tests/test_scripting.py
@ -362,7 +362,7 @@ class ScriptingRequestTests(unittest.TestCase):
            # Point sqltoolsservice output to file.
            with io.open(file_name, 'wb') as baseline:
                tools_service_process = subprocess.Popen(
-                    'D:\\GitHub\\sqltoolsservice\\src\\Microsoft.SqlTools.ServiceLayer\\bin\\Debug\\netcoreapp1.0\\win7-x64\\Microsoft.SqlTools.ServiceLayer.exe',
+                    'D:\\GitHub\\sqltoolsservice\\src\\Microsoft.SqlTools.ServiceLayer\\bin\\Debug\\netcoreapp2.0\\win7-x64\\MicrosoftSqlToolsServiceLayer.exe',
                    bufsize=0,
                    stdin=subprocess.PIPE,
                    stdout=baseline)
--- a/mssqlscripter/main.py
+++ b/mssqlscripter/main.py
@ -87,9 +87,11 @@ def main(args):

        while not scripting_request.completed():
            response = scripting_request.get_response()
-
            if response:
                scriptercallbacks.handle_response(response, parameters.DisplayProgress)
+            else:
+                # The sleep prevents burning up the CPU and lets other threads get scheduled.
+                time.sleep(0.1)

        # Only write to stdout if user did not provide a file path.
        logger.info('stdout current encoding: {}'.format(sys.stdout.encoding))
--- a/mssqltoolsservice/buildwheels.py
+++ b/mssqltoolsservice/buildwheels.py
@ -17,20 +17,20 @@ install_aliases()
 from urllib.request import urlopen


-DOWNLOAD_URL_BASE = 'https://mssqlscripter.blob.core.windows.net/sqltoolsservice-08-01-2017/'
+DOWNLOAD_URL_BASE = 'https://mssqlscripter.blob.core.windows.net/sqltoolsservice-08-16-2017/'

 # Supported platform key's must match those in mssqlscript's setup.py.
 SUPPORTED_PLATFORMS = {
-    'CentOS_7': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-centos-x64-netcoreapp1.0.tar.gz',
-    'Debian_8': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-debian-x64-netcoreapp1.0.tar.gz',
-    'Fedora_23': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-fedora-x64-netcoreapp1.0.tar.gz',
-    'openSUSE_13_2': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-opensuse-x64-netcoreapp1.0.tar.gz',
-    'OSX_10_11_64': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-osx-x64-netcoreapp1.0.tar.gz',
-    'RHEL_7': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-rhel-x64-netcoreapp1.0.tar.gz',
-    'Ubuntu_14': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-ubuntu14-x64-netcoreapp1.0.tar.gz',
-    'Ubuntu_16': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-ubuntu16-x64-netcoreapp1.0.tar.gz',
-    'Windows_7_64': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-win-x64-netcoreapp1.0.zip',
-    'Windows_7_86': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-win-x86-netcoreapp1.0.zip',
+    'CentOS_7': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-centos-x64-netcoreapp2.0.tar.gz',
+    'Debian_8': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-debian-x64-netcoreapp2.0.tar.gz',
+    'Fedora_23': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-fedora-x64-netcoreapp2.0.tar.gz',
+    'openSUSE_13_2': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-opensuse-x64-netcoreapp2.0.tar.gz',
+    'OSX_10_11_64': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-osx-x64-netcoreapp2.0.tar.gz',
+    'RHEL_7': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-rhel-x64-netcoreapp2.0.tar.gz',
+    'Ubuntu_14': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-ubuntu14-x64-netcoreapp2.0.tar.gz',
+    'Ubuntu_16': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-ubuntu16-x64-netcoreapp2.0.tar.gz',
+    'Windows_7_64': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-win-x64-netcoreapp2.0.zip',
+    'Windows_7_86': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-win-x86-netcoreapp2.0.zip'
 }

 CURRENT_DIRECTORY = os.path.abspath(os.path.join(os.path.abspath(__file__), '..'))
--- a/mssqltoolsservice/mssqltoolsservice/init.py
+++ b/mssqltoolsservice/mssqltoolsservice/init.py
@ -9,7 +9,7 @@
 import os
 import platform

-__version__ = '1.0.0a18'
+__version__ = '1.0.0a20'


 def get_executable_path():
@ -28,7 +28,7 @@ def get_executable_path():
                'bin'))

    # Format name based on platform.
-    mssqltoolsservice_name = u'Microsoft.SqlTools.ServiceLayer{}'.format(
+    mssqltoolsservice_name = u'MicrosoftSqlToolsServiceLayer{}'.format(
        u'.exe' if (platform.system() == u'Windows') else u'')

    mssqltoolsservice_full_path = os.path.abspath(os.path.join(mssqltoolsservice_base_path, mssqltoolsservice_name))
--- a/mssqltoolsservice/setup.py
+++ b/mssqltoolsservice/setup.py
@ -12,7 +12,7 @@ import sys

 # This version number is in place in two places and must be in sync with
 # mssqlscripter's version in setup.py.
-MSSQLTOOLSSERVICE_VERSION = '1.0.0a18'
+MSSQLTOOLSSERVICE_VERSION = '1.0.0a20'

 # If we have source, validate version numbers match to prevent
 # uploading releases with mismatched versions.
--- a/setup.py
+++ b/setup.py
@ -14,7 +14,7 @@ from setuptools import setup

 # This version number is in place in two places and must be in sync with
 # mssqltoolsservice's version in setup.py.
-MSSQLSCRIPTER_VERSION = '1.0.0a18'
+MSSQLSCRIPTER_VERSION = '1.0.0a20'

 # If we have the source, validate our setup version matches source version.
 # This will prevent uploading releases with mismatched versions. This will
@ -56,7 +56,7 @@ else:
                toolsservice_version.group(1)))
        sys.exit(1)

-MSSQLTOOLSSERVICE_PACKAGE_NAME = 'mssqltoolsservice_{}=={}'
+MSSQLTOOLSSERVICE_PACKAGE_NAME = 'mssqltoolsservice-{}=={}'
 MSSQLTOOLSSERVICE_PACKAGE_SUFFIX = [
    'CentOS_7',
    'Debian_8',
@ -204,7 +204,7 @@ def get_mssqltoolsservice_package_name(run_time_id=_get_runtime_id()):
        # set package suffix name for other uses like building wheels outside of setup.py.
        os.environ['MSSQLTOOLSSERVICE_PACKAGE_SUFFIX'] = run_time_id
        return MSSQLTOOLSSERVICE_PACKAGE_NAME.format(
-            run_time_id, MSSQLSCRIPTER_VERSION)
+            run_time_id, MSSQLSCRIPTER_VERSION).replace('_', '-').lower()

    raise EnvironmentError('mssqltoolsservice is not supported on this platform.')

@ -217,7 +217,6 @@ CLASSIFIERS = [
    'Programming Language :: Python :: 2',
    'Programming Language :: Python :: 2.7',
    'Programming Language :: Python :: 3',
-    'Programming Language :: Python :: 3.3',
    'Programming Language :: Python :: 3.4',
    'Programming Language :: Python :: 3.5',
    'Programming Language :: Python :: 3.6',
--- a/sql-xplat-cli.pyproj
+++ b/sql-xplat-cli.pyproj
@ -11,8 +11,6 @@
    <OutputPath>.</OutputPath>
    <ProjectTypeGuids>{888888a0-9f3d-457c-b088-3a5042f75d52}</ProjectTypeGuids>
    <LaunchProvider>Standard Python launcher</LaunchProvider>
-    <InterpreterId />
-    <InterpreterVersion />
    <CommandLineArguments>-S localhost -d AdventureWorks2014</CommandLineArguments>
    <EnableNativeCodeDebugging>False</EnableNativeCodeDebugging>
    <IsWindowsApplication>False</IsWindowsApplication>
@ -21,10 +19,11 @@
  <PropertyGroup Condition="'$(Configuration)' == 'Release'" />
  <PropertyGroup>
    <VisualStudioVersion Condition=" '$(VisualStudioVersion)' == '' ">10.0</VisualStudioVersion>
-    <PtvsTargetsFile>$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\Python Tools\Microsoft.PythonTools.targets</PtvsTargetsFile>
  </PropertyGroup>
  <ItemGroup>
    <Content Include=".gitignore" />
+    <Content Include="appveyor.yml" />
+    <Content Include="dev_requirements.txt" />
    <Content Include="doc\README.md" />
    <Content Include="doc\architecture_guide.md" />
    <Content Include="doc\development_guide.md" />
@ -43,13 +42,13 @@
    <Content Include="mssqltoolsservice\README.rst" />
    <Content Include="mssqltoolsservice\setup.cfg" />
    <Content Include="README.rst" />
-    <Content Include="requirements.txt" />
    <Content Include="setup.cfg" />
    <Content Include="tox.ini" />
    <Content Include=".bumpversion.cfg" />
    <Content Include=".travis.yml" />
  </ItemGroup>
  <ItemGroup>
+    <Compile Include="build.py" />
    <Compile Include="dev_setup.py" />
    <Compile Include="mssqlscripter\argparser.py" />
    <Compile Include="mssqlscripter\jsonrpc\contracts\scriptingservice.py" />
@ -87,6 +86,5 @@
    <Folder Include="mssqltoolsservice\" />
    <Folder Include="mssqltoolsservice\mssqltoolsservice\" />
  </ItemGroup>
-  <Import Project="$(PtvsTargetsFile)" Condition="Exists($(PtvsTargetsFile))" />
-  <Import Project="$(MSBuildToolsPath)\Microsoft.Common.targets" Condition="!Exists($(PtvsTargetsFile))" />
+  <Import Project="$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\Python Tools\Microsoft.PythonTools.targets" />
 </Project>
--- a/sql-xplat-cli.sln
+++ b/sql-xplat-cli.sln
@ -1,7 +1,7 @@

 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio 14
-VisualStudioVersion = 14.0.25420.1
+# Visual Studio 15
+VisualStudioVersion = 15.0.26730.0
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "sql-xplat-cli", "sql-xplat-cli.pyproj", "{F4BB6290-43F3-4F35-B26E-067C5EF8E64B}"
 EndProject
@ -17,4 +17,7 @@ Global
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {1A0F24BB-2803-4A8C-9816-CB374FAAC673}
+	EndGlobalSection
 EndGlobal
--- a/utility.py
+++ b/utility.py
@ -9,6 +9,10 @@ import os
 import shutil
 import sys

+ROOT_DIR = os.path.abspath(os.path.join(os.path.abspath(__file__), '..'))
+
+MSSQLTOOLSSERVICE_DIRECTORY = os.path.abspath(os.path.join(
+    os.path.abspath(__file__), '..', 'mssqltoolsservice'))

 MSSQLSCRIPTER_DIST_DIRECTORY = os.path.abspath(
    os.path.join(os.path.abspath(__file__), '..', 'dist'))
@ -32,6 +36,12 @@ def exec_command(command, directory, continue_on_error=True):
            pass


+def cleaun_up_egg_info_sub_directories(directory):
+    for f in os.listdir(directory):
+        if f.endswith(".egg-info"):
+            clean_up(os.path.join(directory, f))
+
+
 def clean_up(directory):
    """
        Delete directory.