From 55d2c4f583c87adf741b2e8515b2df45b7058fd0 Mon Sep 17 00:00:00 2001 From: Ted Mielczarek Date: Thu, 13 Nov 2008 15:37:04 +0000 Subject: [PATCH] Bug 454594: need a makefile target that can upload files via ssh. r=bsmedberg, NPOB --- browser/build.mk | 3 + build/__init__.py | 0 build/upload.py | 195 ++++++++++++++++++++++++++ build/util.py | 50 +++++++ client.py | 13 +- toolkit/mozapps/installer/packager.mk | 7 + 6 files changed, 256 insertions(+), 12 deletions(-) create mode 100644 build/__init__.py create mode 100644 build/upload.py create mode 100644 build/util.py diff --git a/browser/build.mk b/browser/build.mk index 5f108a7481f..b7e6638b62d 100644 --- a/browser/build.mk +++ b/browser/build.mk @@ -69,6 +69,9 @@ distclean:: source-package:: @$(MAKE) -C browser/installer source-package +upload:: + @$(MAKE) -C browser/installer upload + ifdef ENABLE_TESTS # Implemented in testing/testsuite-targets.mk diff --git a/build/__init__.py b/build/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/build/upload.py b/build/upload.py new file mode 100644 index 00000000000..34a360d64ce --- /dev/null +++ b/build/upload.py @@ -0,0 +1,195 @@ +#!/usr/bin/python +# +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is mozilla.org code. +# +# The Initial Developer of the Original Code is +# The Mozilla Foundation +# Portions created by the Initial Developer are Copyright (C) 2008 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Ted Mielczarek +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** +# +# When run directly, this script expects the following environment variables +# to be set: +# UPLOAD_HOST : host to upload files to +# UPLOAD_USER : username on that host +# UPLOAD_PATH : path on that host to put the files in +# +# And will use the following optional environment variables if set: +# UPLOAD_SSH_KEY : path to a ssh private key to use +# UPLOAD_PORT : port to use for ssh +# POST_UPLOAD_CMD: a commandline to run on the remote host after uploading. +# UPLOAD_PATH and the full paths of all files uploaded will +# be appended to the commandline. +# +# All files to be uploaded should be passed as commandline arguments to this +# script. The script takes one other parameter, --base-path, which you can use +# to indicate that files should be uploaded including their paths relative +# to the base path. + +import sys, os +from optparse import OptionParser +from util import check_call + +def RequireEnvironmentVariable(v): + """Return the value of the environment variable named v, or print + an error and exit if it's unset (or empty).""" + if not v in os.environ or os.environ[v] == "": + print "Error: required environment variable %s not set" % v + sys.exit(1) + return os.environ[v] + +def OptionalEnvironmentVariable(v): + """Return the value of the environment variable named v, or None + if it's unset (or empty).""" + if v in os.environ and os.environ[v] != "": + return os.environ[v] + return None + +def FixupMsysPath(path): + """MSYS helpfully translates absolute pathnames in environment variables + and commandline arguments into Windows native paths. This sucks if you're + trying to pass an absolute path on a remote server. This function attempts + to un-mangle such paths.""" + if 'OSTYPE' in os.environ and os.environ['OSTYPE'] == 'msys': + # sort of awful, find out where our shell is (should be in msys/bin) + # and strip the first part of that path out of the other path + if 'SHELL' in os.environ: + sh = os.environ['SHELL'] + msys = sh[:sh.find('/bin')] + if path.startswith(msys): + path = path[len(msys):] + return path + +def WindowsPathToMsysPath(path): + """Translate a Windows pathname to an MSYS pathname. + Necessary because we call out to ssh/scp, which are MSYS binaries + and expect MSYS paths.""" + if sys.platform != 'win32': + return path + (drive, path) = os.path.splitdrive(os.path.abspath(path)) + return "/" + drive[0] + path.replace('\\','/') + +def AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key): + """Given optional port and ssh key values, append valid OpenSSH + commandline arguments to the list cmdline if the values are not None.""" + if port is not None: + cmdline.append("-P%d" % port) + if ssh_key is not None: + cmdline.extend(["-i", WindowsPathToMsysPath(ssh_key)]) + +def DoSSHCommand(command, user, host, port=None, ssh_key=None): + """Execute command on user@host using ssh. Optionally use + port and ssh_key, if provided.""" + cmdline = ["ssh"] + AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key) + cmdline.extend(["%s@%s" % (user, host), command]) + check_call(cmdline) + +def DoSCPFile(file, remote_path, user, host, port=None, ssh_key=None): + """Upload file to user@host:remote_path using scp. Optionally use + port and ssh_key, if provided.""" + cmdline = ["scp"] + AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key) + cmdline.extend([WindowsPathToMsysPath(file), + "%s@%s:%s" % (user, host, remote_path)]) + check_call(cmdline) + +def GetRemotePath(path, local_file, base_path): + """Given a remote path to upload to, a full path to a local file, and an + optional full path that is a base path of the local file, construct the + full remote path to place the file in. If base_path is not None, include + the relative path from base_path to file.""" + if base_path is None or not local_file.startswith(base_path): + return path + dir = os.path.dirname(local_file) + # strip base_path + extra slash and make it unixy + dir = dir[len(base_path)+1:].replace('\\','/') + return path + dir + +def UploadFiles(user, host, path, files, verbose=False, port=None, ssh_key=None, base_path=None, post_upload_command=None): + """Upload each file in the list files to user@host:path. Optionally pass + port and ssh_key to the ssh commands. If base_path is not None, upload + files including their path relative to base_path. If post_upload_command + is not None, execute that command on the remote host after uploading + all files, passing it the upload path, and the full paths to all files + uploaded. If verbose is True, print status updates while working.""" + if not path.endswith("/"): + path += "/" + if base_path is not None: + base_path = os.path.abspath(base_path) + remote_files = [] + for file in files: + file = os.path.abspath(file) + if not os.path.isfile(file): + raise IOError("File not found: %s" % file) + # first ensure that path exists remotely + remote_path = GetRemotePath(path, file, base_path) + DoSSHCommand("mkdir -p " + remote_path, user, host, port=port, ssh_key=ssh_key) + if verbose: + print "Uploading " + file + DoSCPFile(file, remote_path, user, host, port=port, ssh_key=ssh_key) + remote_files.append(remote_path + '/' + os.path.basename(file)) + if post_upload_command is not None: + if verbose: + print "Running post-upload command: " + post_upload_command + file_list = '"' + '" "'.join(remote_files) + '"' + DoSSHCommand('%s "%s" %s' % (post_upload_command, path, file_list), user, host, port=port, ssh_key=ssh_key) + if verbose: + print "Upload complete" + +if __name__ == '__main__': + host = RequireEnvironmentVariable('UPLOAD_HOST') + user = RequireEnvironmentVariable('UPLOAD_USER') + path = RequireEnvironmentVariable('UPLOAD_PATH') + if sys.platform == 'win32': + path = FixupMsysPath(path) + port = int(OptionalEnvironmentVariable('UPLOAD_PORT')) + key = OptionalEnvironmentVariable('UPLOAD_SSH_KEY') + post_upload_command = OptionalEnvironmentVariable('POST_UPLOAD_CMD') + + parser = OptionParser(usage="usage: %prog [options] ") + parser.add_option("-b", "--base-path", + action="store", dest="base_path", + help="Preserve file paths relative to this path when uploading. If unset, all files will be uploaded directly to UPLOAD_PATH.") + (options, args) = parser.parse_args() + if len(args) < 1: + print "You must specify at least one file to upload" + sys.exit(1) + try: + UploadFiles(user, host, path, args, base_path=options.base_path, + port=port, ssh_key=key, post_upload_command=post_upload_command, + verbose=True) + except IOError, (strerror): + print strerror + except Exception, (err): + print err + diff --git a/build/util.py b/build/util.py new file mode 100644 index 00000000000..89a6d9621fb --- /dev/null +++ b/build/util.py @@ -0,0 +1,50 @@ +#!/usr/bin/python +# +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is mozilla.org code. +# +# The Initial Developer of the Original Code is +# The Mozilla Foundation +# Portions created by the Initial Developer are Copyright (C) 2008 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Ted Mielczarek +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** + +try: + from subprocess import check_call +except ImportError: + import subprocess + def check_call(*popenargs, **kwargs): + retcode = subprocess.call(*popenargs, **kwargs) + if retcode: + cmd = kwargs.get("args") + if cmd is None: + cmd = popenargs[0] + raise Exception("Command '%s' returned non-zero exit status %i" % (cmd, retcode)) diff --git a/client.py b/client.py index 90b13073fa8..b30dee3ec69 100644 --- a/client.py +++ b/client.py @@ -11,23 +11,12 @@ import sys import datetime import shutil from optparse import OptionParser +from build.util import check_call topsrcdir = os.path.dirname(__file__) if topsrcdir == '': topsrcdir = '.' -try: - from subprocess import check_call -except ImportError: - import subprocess - def check_call(*popenargs, **kwargs): - retcode = subprocess.call(*popenargs, **kwargs) - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise Exception("Command '%s' returned non-zero exit status %i" % (cmd, retcode)) - def check_call_noisy(cmd, *args, **kwargs): print "Executing command:", cmd check_call(cmd, *args, **kwargs) diff --git a/toolkit/mozapps/installer/packager.mk b/toolkit/mozapps/installer/packager.mk index 3d235de9d33..14ddedbbe1f 100644 --- a/toolkit/mozapps/installer/packager.mk +++ b/toolkit/mozapps/installer/packager.mk @@ -480,6 +480,13 @@ make-sdk: (cd $(DIST)/$(MOZ_APP_NAME)-sdk/lib && tar -xf -) cd $(DIST) && $(MAKE_SDK) +ifeq ($(OS_TARGET), WINNT) +INSTALLER_PACKAGE = $(DIST)/$(PKG_INST_PATH)$(PKG_INST_BASENAME).exe +endif + +upload: + $(PYTHON) $(topsrcdir)/build/upload.py --base-path $(DIST) $(DIST)/$(PACKAGE) $(INSTALLER_PACKAGE) + ifndef MOZ_PKG_SRCDIR MOZ_PKG_SRCDIR = $(topsrcdir) endif