gecko-dev/addon-sdk/source/bin/integration-scripts/integration-check

#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import os
import signal
import threading
import urllib2, urllib
import zipfile
import tarfile
import subprocess
import optparse
import sys, re
#import win32api


class SDK:
    def __init__(self):
        try:
            # Take the current working directory
            self.default_path = os.getcwd()
            if sys.platform == "win32":
                self.mswindows = True
            else:
                self.mswindows = False
            # Take the default home path of the user.
            home = os.path.expanduser('~')

            # The following are the parameters that can be used to pass a dynamic URL, a specific path or a binry. The binary is not used yet. It will be used in version 2.0
            # If a dynamic path is to be mentioned, it should start with a '/'. For eg. "/Desktop"
            parser = optparse.OptionParser()
            parser.add_option('-u', '--url', dest = 'url', default = 'https://ftp.mozilla.org/pub/mozilla.org/labs/jetpack/addon-sdk-latest.zip')
            parser.add_option('-p', '--path', dest = 'path', default = self.default_path)
            parser.add_option('-b', '--binary', dest = 'binary')#, default='/Applications/Firefox.app')
            (options, args) = parser.parse_args()
      
            # Get the URL from the parameter
            self.link = options.url
            # Set the base path for the user. If the user supplies the path, use the home variable as well. Else, take the default path of this script as the installation directory.
            if options.path!=self.default_path:
                if self.mswindows:
                    self.base_path = home + str(options.path).strip() + '\\'
                else:
                    self.base_path = home + str(options.path).strip() + '/'
            else:
                if self.mswindows:
                    self.base_path = str(options.path).strip() + '\\'
                else:
                    self.base_path = str(options.path).strip() + '/'
            assert ' ' not in self.base_path, "You cannot have a space in your home path. Please remove the space before you continue."
            print('Your Base path is =' + self.base_path)
            
            # This assignment is not used in this program. It will be used in version 2 of this script.
            self.bin = options.binary
            # if app or bin is empty, dont pass anything
    
            # Search for the .zip file or tarball file in the URL.
            i = self.link.rfind('/')

            self.fname = self.link[i+1:]
            z = re.search('zip',self.fname,re.I)
            g = re.search('gz',self.fname,re.I)
            if z:
                print 'zip file present in the URL.'
                self.zip = True
                self.gz = False
            elif g:
                print 'gz file present in the URL'
                self.gz = True
                self.zip = False
            else:
                print 'zip/gz file not present. Check the URL.'
                return
            print("File name is =" + self.fname)
    
            # Join the base path and the zip/tar file name to crate a complete Local file path.
            self.fpath = self.base_path + self.fname
            print('Your local file path will be=' + self.fpath)
        except AssertionError, e:
            print e.args[0] 
            sys.exit(1)

    # Download function - to download the SDK from the URL to the local machine.
    def download(self,url,fpath,fname):
        try:
            # Start the download
            print("Downloading...Please be patient!")
            urllib.urlretrieve(url,filename = fname)
            print('Download was successful.')
        except ValueError: # Handles broken URL errors.
            print 'The URL is ether broken or the file does not exist. Please enter the correct URL.'
            raise
        except urllib2.URLError: # Handles URL errors
            print '\nURL not correct. Check again!'
            raise

    # Function to extract the downloaded zipfile.
    def extract(self, zipfilepath, extfile):
        try:
            # Timeout is set to 30 seconds. 
            timeout = 30
            # Change the directory to the location of the zip file.
            try:
                os.chdir(zipfilepath)
            except OSError:
             # Will reach here if zip file doesnt exist
                 print 'O/S Error:' + zipfilepath + 'does not exist'
                 raise

            # Get the folder name of Jetpack to get the exact version number.
            if self.zip:
                try:
                    f = zipfile.ZipFile(extfile, "r")
                except IOError as (errno, strerror): # Handles file errors
                    print "I/O error - Cannot perform extract operation: {1}".format(errno, strerror)
                    raise
                list = f.namelist()[0]
                temp_name = list.split('/')
                print('Folder Name= ' +temp_name[0])
                self.folder_name = temp_name[0]
            elif self.gz:
                try:
                    f = tarfile.open(extfile,'r')
                except IOError as (errno, strerror): # Handles file errors
                    print "I/O error - Cannot perform extract operation: {1}".format(errno, strerror)
                    raise
                list = f.getnames()[0]
                temp_name = list.split('/')
                print('Folder Name= ' +temp_name[0])
                self.folder_name = temp_name[0]

            print ('Starting to Extract...')

            # Timeout code. The subprocess.popen exeutes the command and the thread waits for a timeout. If the process does not finish within the mentioned-
            # timeout, the process is killed.
            kill_check = threading.Event()
            
            if self.zip:
            # Call the command to unzip the file.
                if self.mswindows:
                    zipfile.ZipFile.extractall(f)
                else:
                    p = subprocess.Popen('unzip '+extfile, stdout=subprocess.PIPE, shell=True)
                    pid = p.pid
            elif self.gz:
            # Call the command to untar the file.
                if self.mswindows:
                    tarfile.TarFile.extractall(f)
                else:
                    p = subprocess.Popen('tar -xf '+extfile, stdout=subprocess.PIPE, shell=True)
                    pid = p.pid
            
            #No need to handle for windows because windows automatically replaces old files with new files. It does not ask the user(as it does in Mac/Unix)
            if self.mswindows==False:
                watch = threading.Timer(timeout, kill_process, args=(pid, kill_check, self.mswindows ))
                watch.start()
                (stdout, stderr) = p.communicate()
                watch.cancel() # if it's still waiting to run
                success = not kill_check.isSet()
    
                # Abort process if process fails.
                if not success:
                    raise RuntimeError
                kill_check.clear()
            print('Extraction Successful.')
        except RuntimeError:
            print "Ending the program"
            sys.exit(1)
        except:
            print "Error during file extraction: ", sys.exc_info()[0]
            raise

    # Function to run the cfx testall comands and to make sure the SDK is not broken.
    def run_testall(self, home_path, folder_name):
        try:
            timeout = 500

            self.new_dir = home_path + folder_name
            try:
                os.chdir(self.new_dir)
            except OSError:
             # Will reach here if the jetpack 0.X directory doesnt exist
                print 'O/S Error: Jetpack directory does not exist at ' + self.new_dir
                raise
            print '\nStarting tests...'
            # Timeout code. The subprocess.popen exeutes the command and the thread waits for a timeout. If the process does not finish within the mentioned-
            # timeout, the process is killed.
            kill_check = threading.Event()

            # Set the path for the logs. They will be in the parent directory of the Jetpack SDK.
            log_path = home_path + 'tests.log'

            # Subprocess call to set up the jetpack environment and to start the tests. Also sends the output to a log file.
            if self.bin != None:
                if self.mswindows:
                    p = subprocess.Popen("bin\\activate && cfx testall -a firefox -b \"" + self.bin + "\"" , stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
                    proc_handle = p._handle
                    (stdout,stderr) = p.communicate()
                else:
                    p = subprocess.Popen('. bin/activate; cfx testall -a firefox -b ' + self.bin , stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
                    pid = p.pid
                    (stdout,stderr) = p.communicate()
            elif self.bin == None:
                if self.mswindows:
                    p=subprocess.Popen('bin\\activate && cfx testall -a firefox > '+log_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
                    proc_handle = p._handle
                    (stdout,stderr) = p.communicate()
                else:
                    p = subprocess.Popen('. bin/activate; cfx testall -a firefox > '+log_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
                    pid = p.pid
                    (stdout,stderr) = p.communicate()
                    
            #Write the output to log file
            f=open(log_path,"w")
            f.write(stdout+stderr)
            f.close()

            #Watchdog for timeout process
            if self.mswindows:
                watch = threading.Timer(timeout, kill_process, args=(proc_handle, kill_check, self.mswindows))
            else:
                watch = threading.Timer(timeout, kill_process, args=(pid, kill_check, self.mswindows))
            watch.start()
            watch.cancel() # if it's still waiting to run
            success = not kill_check.isSet()
            if not success:
                raise RuntimeError
            kill_check.clear()
        
            if p.returncode!=0:
                print('\nAll tests were not successful. Check the test-logs in the jetpack directory.')
                result_sdk(home_path)
                #sys.exit(1)
                raise RuntimeError
            else:
                ret_code=result_sdk(home_path)
                if ret_code==0:
                    print('\nAll tests were successful. Yay \o/ . Running a sample package test now...')
                else:
                    print ('\nThere were errors during the tests.Take a look at logs')
                    raise RuntimeError
        except RuntimeError:
            print "Ending the program"
            sys.exit(1)
        except:
            print "Error during the testall command execution:", sys.exc_info()[0]
            raise
        
    def package(self, example_dir):
        try:
            timeout = 30
    
            print '\nNow Running packaging tests...'
    
            kill_check = threading.Event()

            # Set the path for the example logs. They will be in the parent directory of the Jetpack SDK.
            exlog_path = example_dir + 'test-example.log'
            # Subprocess call to test the sample example for packaging.
            if self.bin!=None:
                if self.mswindows:
                    p = subprocess.Popen('bin\\activate && cfx run --pkgdir examples\\reading-data  --static-args="{\\"quitWhenDone\\":true}" -b \"" + self.bin + "\"' , stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
                    proc_handle = p._handle
                    (stdout, stderr) = p.communicate()
                else:
                    p = subprocess.Popen('. bin/activate; cfx run --pkgdir examples/reading-data  --static-args=\'{\"quitWhenDone\":true}\' -b ' + self.bin , stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
                    pid = p.pid
                    (stdout, stderr) = p.communicate()
            elif self.bin==None:
                if self.mswindows:
                    p = subprocess.Popen('bin\\activate && cfx run  --pkgdir examples\\reading-data --static-args="{\\"quitWhenDone\\":true}"', stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
                    proc_handle = p._handle
                    (stdout, stderr) = p.communicate()
                else:
                    p = subprocess.Popen('. bin/activate; cfx run --pkgdir examples/reading-data --static-args=\'{\"quitWhenDone\":true}\' ', stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
                    pid = p.pid
                    (stdout, stderr) = p.communicate()
            
            #Write the output to log file
            f=open(exlog_path,"w")
            f.write(stdout+stderr)
            f.close()
            
            #Watch dog for timeout process
            if self.mswindows:
                watch = threading.Timer(timeout, kill_process, args=(proc_handle, kill_check, self.mswindows))
            else:
                watch = threading.Timer(timeout, kill_process, args=(pid, kill_check, self.mswindows))
            watch.start()
            watch.cancel() # if it's still waiting to run
            success = not kill_check.isSet()
            if not success:
                raise RuntimeError
            kill_check.clear()

            if p.returncode != 0:
                print('\nSample tests were not executed correctly. Check the test-example log in jetpack diretory.')
                result_example(example_dir)
                raise RuntimeError
            else:
                ret_code=result_example(example_dir)
                if ret_code==0:
                    print('\nAll tests pass. The SDK is working! Yay \o/')
                else:
                    print ('\nTests passed with warning.Take a look at logs')
                    sys.exit(1)
        
        except RuntimeError:
            print "Ending program"
            sys.exit(1)
        except:
            print "Error during running sample tests:", sys.exc_info()[0]
            raise
    
def result_sdk(sdk_dir):
    log_path = sdk_dir + 'tests.log'
    print 'Results are logged at:' + log_path
    try:
        f = open(log_path,'r')
    # Handles file errors
    except IOError : 
        print 'I/O error - Cannot open test log at ' + log_path
        raise

    for line in reversed(open(log_path).readlines()):
        if line.strip()=='FAIL':
            print ('\nOverall result - FAIL. Look at the test log at '+log_path)
            return 1
    return 0
    

def result_example(sdk_dir):
    exlog_path = sdk_dir + 'test-example.log'
    print 'Sample test results are logged at:' + exlog_path
    try:
        f = open(exlog_path,'r')
    # Handles file errors
    except IOError : 
        print 'I/O error - Cannot open sample test log at ' + exlog_path
        raise
    
    #Read the file in reverse and check for the keyword 'FAIL'.
    for line in reversed(open(exlog_path).readlines()):
        if line.strip()=='FAIL':
            print ('\nOverall result for Sample tests - FAIL. Look at the test log at '+exlog_path)
            return 1
    return 0

def kill_process(process, kill_check, mswindows):
    print '\nProcess Timedout. Killing the process. Please Rerun this script.'
    if mswindows:
        win32api.TerminateProcess(process, -1)
    else:
        os.kill(process, signal.SIGKILL)
    kill_check.set()# tell the main routine to kill. Used SIGKILL to hard kill the process.
    return

if __name__ == "__main__":
    obj = SDK()
    obj.download(obj.link,obj.fpath,obj.fname)
    obj.extract(obj.base_path,obj.fname)
    obj.run_testall(obj.base_path,obj.folder_name)
    obj.package(obj.base_path)