msphpsql/buildscripts/buildtools.py

#!/usr/bin/python3
#########################################################################################
#
# Description:  The class BuildUtil will build Microsoft SQL Server PHP 7+ Drivers 
#               for 32 bit and 64 bit.
#
# Requirement:
#               python 3.x
#               PHP SDK and PHP Source 
#               Driver source code folder
#               Git for Windows
#               Visual Studio 2015 (PHP 7.0* and 7.1*) and Visual Studio 2017 (PHP 7.2*)
#
# Output: The drivers will be renamed and copied to the specified location.
#
#############################################################################################

import shutil
import os.path
import stat
import datetime
import urllib.request
import zipfile 
import fileinput

class BuildUtil(object):
    """Build sqlsrv and/or pdo_sqlsrv drivers with PHP source with the following properties:
    
    Attributes:
        phpver          # PHP version, e.g. 7.1.*, 7.2.* etc.
        driver          # all, sqlsrv, or pdo_sqlsrv
        arch            # x64 or x86
        thread          # nts or ts
        no_rename       # do NOT rename the drivers if True
        debug_enabled   # whether debug is enabled
    """
    
    def __init__(self, phpver, driver, arch, thread, no_rename, debug_enabled = False):
        self.phpver = phpver
        self.driver = driver.lower()
        self.arch = arch.lower()
        self.thread = thread.lower()
        self.no_rename = no_rename
        self.debug_enabled = debug_enabled
        self.vc = ''

    def major_version(self):
        """Return the major version number based on the PHP version."""
        return self.phpver[0:3]
        
    def version_label(self):
        """Return the version label based on the PHP version."""
        major_ver = self.major_version()       
        version = major_ver[0] + major_ver[2]
        return version

    def driver_name(self, driver, suffix):
        """Return the *driver* name with *suffix* after PHP is successfully compiled."""
        return 'php_' + driver + suffix

    def driver_new_name(self, driver, suffix):
        """Return the *driver* name with *suffix* based on PHP version and thread."""
        version = self.version_label()
        return 'php_' + driver + '_' + version + '_' + self.thread + suffix

    def determine_compiler(self, sdk_dir, vs_ver):
        """Return the compiler version using vswhere.exe."""
        vswhere = os.path.join(sdk_dir, 'php-sdk', 'bin', 'vswhere.exe')
        if not os.path.exists(vswhere):
            print('Could not find ' + vswhere)
            exit(1)
        
        # If both VS 2017 and VS 2019 are installed, if we check only version 15,
        # both versions are returned.
        # For example, temp.txt would have the following values (in this order):
        # 16.1.29009.5
        # 15.9.28307.344
        # But if only VS 2017 is present, temp.txt will only have one value like this:
        # 15.9.28307.344
        # Likewise, if only VS 2019 is present, temp.txt contains only the one for 16.
        # We can achieve the above by checking for version [15,16), in which case 
        # even if both compilers are present, it only returns one. If only VS 2019
        # exists, temp.txt is empty
        command = '{0} -version [{1},{2}) -property installationVersion '.format(vswhere, vs_ver, vs_ver + 1)
        os.system(command + ' > temp.txt')
        
        # Read the first line from temp.txt
        with open('temp.txt', 'r') as f:
            ver = f.readline()
            print('Version: ' + ver)
        vc = ver[:2]
        if vc == '15':
            return 'vc15'
        else: # For VS2019, it's 'vs' instead of 'vc'
            return 'vs16'
        
    def compiler_version(self, sdk_dir):
        """Return the appropriate compiler version based on PHP version."""
        if self.vc == '':
            VC = 'vc15'
            version = self.version_label()
            if version[0] == '8':     # Compiler version for PHP 8.0 or above
                VC = 'vs16'
            self.vc = VC
            print('Compiler: ' + self.vc)
        return self.vc

    def phpsrc_root(self, sdk_dir):
        """Return the path to the PHP source folder based on *sdk_dir*."""
        vc = self.compiler_version(sdk_dir)
        return os.path.join(sdk_dir, 'php-sdk', 'phpdev', vc, self.arch, 'php-'+self.phpver+'-src')
        
    def build_abs_path(self, sdk_dir):   
        """Return the absolute path to the PHP build folder based on *sdk_dir*."""
        phpsrc = self.phpsrc_root(sdk_dir)
        
        build_dir = 'Release'
        if self.debug_enabled:
            build_dir = 'Debug'
        
        if self.thread == 'ts':
            build_dir = build_dir + '_TS'
            
        if self.arch == 'x64':
            build_dir = self.arch + os.sep + build_dir
        
        return os.path.join(phpsrc, build_dir)

    def remove_old_builds(self, sdk_dir):
        """Remove the extensions, e.g. the driver subfolders in php-7.*-src\ext."""
        if not os.path.exists(os.path.join(sdk_dir, 'php-sdk')):
            print('No old builds to be removed...')
            return
    
        print('Removing old builds...')

        phpsrc = self.phpsrc_root(sdk_dir)
        ext_path = os.path.join(phpsrc, 'ext')
        if os.path.exists( ext_path ):
            shutil.rmtree(os.path.join(ext_path, 'sqlsrv'), ignore_errors=True) 
            shutil.rmtree(os.path.join(ext_path, 'pdo_sqlsrv'), ignore_errors=True) 
        
        if self.arch == 'x64':
            shutil.rmtree(os.path.join(phpsrc, self.arch), ignore_errors=True)
        else:
            shutil.rmtree(os.path.join(phpsrc, 'Debug'), ignore_errors=True)
            shutil.rmtree(os.path.join(phpsrc, 'Debug_TS'), ignore_errors=True)
            shutil.rmtree(os.path.join(phpsrc, 'Release'), ignore_errors=True)
            shutil.rmtree(os.path.join(phpsrc, 'Release_TS'), ignore_errors=True)

    def remove_prev_build(self, sdk_dir):
        """Remove all binaries and source code in the Release* or Debug* 
        folders according to the current configuration
        """
        if not os.path.exists(os.path.join(sdk_dir, 'php-sdk')):
            print('No old builds to be removed...')
            return
    
        print('Removing previous build...')
        build_dir = self.build_abs_path(sdk_dir)
        if not os.path.exists(build_dir):
            return
            
        os.chdir(build_dir)
        os.system('DEL *sqlsrv*')    
        
        # remove the extensions in the phpsrc's release* or debug* folder's ext subfolder
        release_ext_path = os.path.join(build_dir, 'ext')
        if os.path.exists( release_ext_path ):
            shutil.rmtree(os.path.join(release_ext_path, 'sqlsrv'), ignore_errors=True) 
            shutil.rmtree(os.path.join(release_ext_path, 'pdo_sqlsrv'), ignore_errors=True) 
        
        # next remove the binaries too
        os.chdir(release_ext_path)
        os.system('DEL *sqlsrv*')    
        
    @staticmethod
    def get_logfile_name():
        """Return the filename for the log file based on timestamp."""
        return 'Build_' + datetime.datetime.now().strftime("%Y%m%d_%H%M") + '.log'
    
    @staticmethod
    def update_file_content(file, search_str, new_str):
        """Find *search_str* and replace it by *new_str* in a *file*"""
        os.chmod(file, stat.S_IWRITE)
        with fileinput.FileInput(file, inplace=True) as f:
            for line in f:
                print(line.replace(search_str, new_str), end='')

    @staticmethod
    def generateMMDD():
        """Return the generated Microsoft PHP Build Version Number"""
        d = datetime.date.today()

        startYear = 2009
        startMonth = 4
        passYear = int( '%02d' % d.year ) - startYear
        passMonth = int( '%02d' % d.month ) - startMonth
        MM = passYear * 12 + passMonth
        dd = d.day

        MMDD = "" + str( MM )
        if( dd < 10 ):
            return MMDD + "0" + str( dd )
        else:
            return MMDD + str( dd )
            
    @staticmethod
    def get_driver_version(version_file):
        """Read the *version_file* and return the driver version."""
        with open(version_file) as f:
            for line in f:
                if 'SQLVERSION_MAJOR' in line:      # major version
                    major = line.split()[2]
                elif 'SQLVERSION_MINOR' in line:    # minor version  
                    minor = line.split()[2]
                elif 'SQLVERSION_PATCH' in line:    # patch  
                    patch = line.split()[2]
                    break

        return major + '.' + minor + '.' + patch 

    @staticmethod
    def write_lines_to_copy_source(driver, file):
        """Write to file the commands to copy *driver* source."""
        source = '%currDir%' + os.sep + 'Source' + os.sep + driver
        dest = '%phpSrc%' + os.sep + 'ext' + os.sep + driver
        file.write('@CALL ROBOCOPY ' + source + ' ' + dest + ' /s /xx /xo' + os.linesep)
        
        source = '%currDir%' + os.sep + 'Source' + os.sep + 'shared'
        dest = '%phpSrc%' + os.sep + 'ext' + os.sep + driver + os.sep + 'shared'
        file.write('@CALL ROBOCOPY ' + source + ' ' + dest + ' /s /xx /xo' + os.linesep)
    
    @staticmethod
    def download_msphpsql_source(repo, branch, dest_folder = 'Source'):
        """Download to *dest_folder* the msphpsql archive of the specified 
        GitHub *repo* and *branch*. The downloaded files will be removed by default.
        """
        try:
            work_dir = os.path.dirname(os.path.realpath(__file__))   

            temppath = os.path.join(work_dir, 'temp')
            # There is no need to remove tree - 
            # for Bamboo, it will be cleaned up eventually
            # for local development, this can act as a cached copy of the repo
            if not os.path.exists(temppath):
                os.makedirs(temppath)
            os.chdir(temppath)
            
            msphpsqlFolder = os.path.join(temppath, 'msphpsql-' + branch)
            
            url = 'https://github.com/' + repo + '/msphpsql.git'
            command = 'git clone ' + url + ' -b ' + branch + ' --single-branch --depth 1 ' + msphpsqlFolder
            os.system(command)
            
            source = os.path.join(msphpsqlFolder, 'source')
            os.chdir(work_dir)
            
            os.system('ROBOCOPY ' + source + '\shared ' + dest_folder + '\shared /xx /xo')
            os.system('ROBOCOPY ' + source + '\pdo_sqlsrv ' + dest_folder + '\pdo_sqlsrv /xx /xo')
            os.system('ROBOCOPY ' + source + '\sqlsrv ' + dest_folder + '\sqlsrv /xx /xo')
                
        except:
            print('Error occurred when downloading source')
            raise

    def update_driver_source(self, source_dir, driver): 
        """Update the *driver* source in *source_path* with the 
        latest version, file descriptions, etc.
        If debug is enabled, will remove the optimization flag  
        """
        driver_dir = os.path.join(source_dir, driver)
        
        if self.debug_enabled:
            # Adding linker flags for creating more debugging information in the binaries
            print('Adding linker flags for', driver)
            config_file = os.path.join(driver_dir, 'config.w32')
            if driver == 'sqlsrv':
                self.update_file_content(config_file, 'ADD_FLAG( "LDFLAGS_SQLSRV", "/NXCOMPAT /DYNAMICBASE /debug /guard:cf" );', 'ADD_FLAG( "LDFLAGS_SQLSRV", "/NXCOMPAT /DYNAMICBASE /debug /guard:cf /debugtype:cv,fixup" );')
            elif driver == 'pdo_sqlsrv':
                self.update_file_content(config_file, 'ADD_FLAG( "LDFLAGS_PDO_SQLSRV", "/NXCOMPAT /DYNAMICBASE /debug /guard:cf" );', 'ADD_FLAG( "LDFLAGS_PDO_SQLSRV", "/NXCOMPAT /DYNAMICBASE /debug /guard:cf /debugtype:cv,fixup" );')
                    
        # Update Template.rc 
        template_file = os.path.join(driver_dir, 'template.rc')
        if driver == 'sqlsrv':
            drivername = self.driver_new_name(driver, '.dll') 
            self.update_file_content(template_file, 'FILE_NAME \"\\0\"', '"' + drivername + '\\0"')
            self.update_file_content(template_file, '\"Microsoft Drivers for PHP for SQL Server\\0\"', '"Microsoft Drivers for PHP for SQL Server (SQLSRV Driver)\\0"')
        elif driver == 'pdo_sqlsrv':
            drivername = self.driver_new_name(driver, '.dll') 
            self.update_file_content(template_file, 'FILE_NAME \"\\0\"', '"' + drivername + '\\0"')
            self.update_file_content(template_file, '\"Microsoft Drivers for PHP for SQL Server\\0\"', '"Microsoft Drivers for PHP for SQL Server (PDO Driver)\\0"')
            
        # Update Version.h
        version_file = os.path.join(source_dir, 'shared', 'version.h')
        build_number = self.generateMMDD()
        self.update_file_content(version_file, 'SQLVERSION_BUILD 0', 'SQLVERSION_BUILD ' + build_number)

        # get the latest version
        version = self.get_driver_version(version_file) + '.' + build_number
        print('Driver version is: ', version)
            
        # Update CREDIT file
        credits_file = os.path.join(driver_dir, 'CREDITS')
        if driver == 'sqlsrv':
            self.update_file_content(credits_file, 'Microsoft Drivers for PHP for SQL Server', 'Microsoft Drivers ' + version + ' for PHP for SQL Server (' + self.driver.upper() + ' driver)')
        elif driver == 'pdo_sqlsrv': 
            self.update_file_content(credits_file, 'Microsoft Drivers for PHP for SQL Server (PDO driver)', 'Microsoft Drivers ' + version + ' for PHP for SQL Server (' + self.driver.upper() + ' driver)')

    def generate_build_options(self):
        """Return the generated build configuration and arguments"""
        cmd_line = ''
        if self.debug_enabled:
            cmd_line = ' --enable-debug '
            
        if self.driver == 'all':
            cmd_line = ' --enable-sqlsrv=shared --enable-pdo --with-pdo-sqlsrv=shared ' + cmd_line
        else:
            if self.driver == 'sqlsrv':
                cmd_line = ' --enable-sqlsrv=shared ' + cmd_line
            else:       # pdo_sqlsrv
                cmd_line = ' --enable-pdo --with-pdo-sqlsrv=shared ' + cmd_line
                
        cmd_line = 'cscript configure.js --disable-all --enable-cli --enable-cgi --enable-json --enable-embed --enable-mbstring --enable-ctype' + cmd_line
        if self.thread == 'nts':
            cmd_line = cmd_line + ' --disable-zts'
        return cmd_line
    
    def create_local_batch_file(self, make_clean, cmd_line, log_file):
        """Generate the batch file to be picked up by the PHP starter script."""
        filename = 'phpsdk-build-task.bat'
        print('Generating ', filename)
        try:
            file = open(filename, 'w')
            file.write('@ECHO OFF' + os.linesep)
            file.write('SET currDir=%CD%' + os.linesep)
            file.write('SET LOG_NAME=%currDir%\\' + log_file + os.linesep)       
            file.write('@CALL phpsdk_buildtree phpdev > %LOG_NAME% 2>&1' + os.linesep)
            
            # for PHP version with release tags, such as 'RC', 'beta', etc. 
            # we need to remove the hyphen '-' between the version number and tag
            # because in https://github.com/php/php-src the released tags have no hyphens
            
            php_tag = 'php-' + self.phpver.replace('-', '')
            php_src = 'php-' + self.phpver +'-src'
            
            # if not exists, check out the specified tag
            file.write('IF NOT EXIST ' + php_src + ' @CALL git clone -b ' + php_tag + ' --depth 1 --single-branch https://github.com/php/php-src.git ' + php_src +  os.linesep)        
            file.write('CD ' + php_src + os.linesep)
            file.write('SET phpSrc=%CD%' + os.linesep)
            file.write('@CALL phpsdk_deps -u >> %LOG_NAME% 2>&1' + os.linesep)
            
            # copy source files to extension
            if self.driver == 'all':
                self.write_lines_to_copy_source('sqlsrv', file)
                self.write_lines_to_copy_source('pdo_sqlsrv', file)
            else:
                self.write_lines_to_copy_source(self.driver, file)
            
            # configure and build
            file.write('@CALL buildconf --force >> %LOG_NAME% 2>&1' + os.linesep)
            file.write('@CALL ' + cmd_line + ' >> %LOG_NAME% 2>&1' + os.linesep)
            if make_clean:
                file.write('nmake clean >> %LOG_NAME% 2>&1' + os.linesep)
            file.write('nmake >> %LOG_NAME% 2>&1' + os.linesep)
            file.write('exit' + os.linesep)
            file.close()
            return filename
        except:
            print('Cannot create ', filename)

    def build_drivers(self, make_clean = False, dest = None, log_file = None):
        """Build sqlsrv/pdo_sqlsrv extensions for PHP, assuming the Source folder 
        exists in the working directory, and this folder will be removed when the build 
        is complete.
        """
        work_dir = os.path.dirname(os.path.realpath(__file__))   
        # First, update the driver source file contents
        source_dir = os.path.join(work_dir, 'Source')
        if self.driver == 'all':
            self.update_driver_source(source_dir, 'sqlsrv') 
            self.update_driver_source(source_dir, 'pdo_sqlsrv') 
        else:
            self.update_driver_source(source_dir, self.driver) 

        # Next, generate the build configuration and arguments
        cmd_line = self.generate_build_options()
        print('cmd_line: ' + cmd_line)

        # Generate a batch file based on the inputs
        if log_file is None:
            log_file = self.get_logfile_name()
        
        batch_file = self.create_local_batch_file(make_clean, cmd_line, log_file)
        
        # Reference: https://github.com/php/php-sdk-binary-tools
        # Clone the master branch of PHP sdk if the directory does not exist 
        print('Downloading the latest php SDK...')
        
        # if *dest* is None, simply use the current working directory
        sdk_dir = dest
        copy_to_ext = True      # this determines where to copy the binaries to
        if dest is None:
            sdk_dir = work_dir
            copy_to_ext = False

        phpSDK = os.path.join(sdk_dir, 'php-sdk')
        if not os.path.exists( phpSDK ):
            os.system('git clone https://github.com/php/php-sdk-binary-tools.git --branch master --single-branch --depth 1 ' + phpSDK)
        os.chdir(phpSDK)
        os.system('git pull ')
        print('Done cloning the latest php SDK...')

        # Move the generated batch file to phpSDK for the php starter script 
        print('Moving the sdk bath file over...')
        sdk_batch_file = os.path.join(phpSDK, batch_file)
        if os.path.exists(sdk_batch_file):
            os.remove(sdk_batch_file)
        shutil.move(os.path.join(work_dir, batch_file), phpSDK)
        
        print('Checking if source exists...')
        sdk_source = os.path.join(phpSDK, 'Source')
        # Sometimes, for various reasons, the Source folder from previous build 
        # might exist in phpSDK. If so, remove it first
        if os.path.exists(sdk_source):  
            os.chmod(sdk_source, stat.S_IWRITE)
            shutil.rmtree(sdk_source, ignore_errors=True) 
        shutil.move(source_dir, phpSDK)
        
        # Invoke phpsdk-<vc>-<arch>.bat
        vc = self.compiler_version(sdk_dir)
        starter_script = 'phpsdk-' + vc + '-' + self.arch + '.bat'
        print('Running starter script: ', starter_script)
        os.system(starter_script + ' -t ' + batch_file)
        
        # Now we can safely remove the Source folder, because its contents have 
        # already been modified prior to building the extensions
        shutil.rmtree(os.path.join(phpSDK, 'Source'), ignore_errors=True) 
        
        # Next, rename the newly compiled PHP extensions, if required
        if not self.no_rename:
            self.rename_binaries(sdk_dir)
        
        # Final step, copy the binaries to the right place
        ext_dir = self.copy_binaries(sdk_dir, copy_to_ext)
        
        return ext_dir

    def rename_binary(self, path, driver):
        """Rename the *driver* binary (sqlsrv or pdo_sqlsrv) (only the dlls)."""
        driver_old_name = self.driver_name(driver, '.dll')
        driver_new_name = self.driver_new_name(driver, '.dll')

        os.rename(os.path.join(path, driver_old_name), os.path.join(path, driver_new_name))

    def rename_binaries(self, sdk_dir):
        """Rename the sqlsrv and/or pdo_sqlsrv dlls according to the PHP
        version and thread.
        """
        
        # Derive the path to where the extensions are located
        ext_dir = self.build_abs_path(sdk_dir)
        print("Renaming binaries in ", ext_dir)
        
        if self.driver == 'all':
            self.rename_binary(ext_dir, 'sqlsrv')
            self.rename_binary(ext_dir, 'pdo_sqlsrv')
        else:
            self.rename_binary(ext_dir, self.driver)
                
    def copy_binary(self, from_dir, dest_dir, driver, suffix):
        """Copy sqlsrv or pdo_sqlsrv binary (based on *suffix*) to *dest_dir*."""
        if not self.no_rename and suffix == '.dll':
            binary = self.driver_new_name(driver, suffix)
        else:
            binary = self.driver_name(driver, suffix)
        shutil.copy2(os.path.join(from_dir, binary), dest_dir)
        if suffix == '.dll':
            php_ini_file = os.path.join(from_dir, 'php.ini')
            with open(php_ini_file, 'a') as php_ini:
                php_ini.write('extension=' + binary + '\n');
    
    def copy_binaries(self, sdk_dir, copy_to_ext):
        """Copy the sqlsrv and/or pdo_sqlsrv binaries, including the pdb files, 
        to the right place, depending on *copy_to_ext*. The default is to 
        copy them to the 'ext' folder.
        """
        
        # Get php.ini file from php.ini-production
        build_dir = self.build_abs_path(sdk_dir)
        php_ini_file = os.path.join(build_dir, 'php.ini')
        print('Setting up php ini file', php_ini_file)
        
        # Copy php.ini-production file to php.ini
        phpsrc = self.phpsrc_root(sdk_dir)
        shutil.copy(os.path.join(phpsrc, 'php.ini-production'), php_ini_file)
        
        # Copy run-tests.php as well
        shutil.copy(os.path.join(phpsrc, 'run-tests.php'), build_dir)
        
        print('Copying the binaries from', build_dir)
        if copy_to_ext:
            dest_dir = os.path.join(build_dir, 'ext') 
            ext_dir_line = 'extension_dir=ext\\'
        else:   
            ext_dir_line = 'extension_dir=.\\'
            # Simply make a copy of the binaries in sdk_dir
            dest_dir = sdk_dir
        
        print('Destination:', dest_dir)
        with open(php_ini_file, 'a') as php_ini:
            php_ini.write(ext_dir_line + '\n')

        # Now copy the binaries
        if self.driver == 'all':
            self.copy_binary(build_dir, dest_dir, 'sqlsrv', '.dll')
            self.copy_binary(build_dir, dest_dir, 'sqlsrv', '.pdb')
            self.copy_binary(build_dir, dest_dir, 'pdo_sqlsrv', '.dll')
            self.copy_binary(build_dir, dest_dir, 'pdo_sqlsrv', '.pdb')
        else:
            self.copy_binary(build_dir, dest_dir, self.driver, '.dll')
            self.copy_binary(build_dir, dest_dir, self.driver, '.pdb')
            
        return dest_dir