diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..bab1a40 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,7 @@ +{ + "workbench.colorCustomizations": { + "activityBar.background": "#32068B", + "titleBar.activeBackground": "#4608C3", + "titleBar.activeForeground": "#FEFDFF" + } +} \ No newline at end of file diff --git a/dino_engine3.py b/dino_engine3.py new file mode 100755 index 0000000..b162e6c --- /dev/null +++ b/dino_engine3.py @@ -0,0 +1,437 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from http.client import HTTPMessage +import subprocess +import json +import os +import hashlib +import urllib.request, urllib.error, urllib.parse +import re +import stat +import shutil +import shlex +import pwd +import grp +import argparse +from SystemConfiguration import SCDynamicStoreCopyConsoleUser + + +# the downloader function accepts two arguments: the url of the file you are +# downloading and the filename (path) of the file you are downloading. The +# downloader reads the Content-Length portion of the header of the incoming +# file and determines the expected file size then reads the incoming file in +# chunks of 8192 bytes and displays the currently read bytes and percentage +# complete +def downloader(url, file_path): + download = urllib.request.urlopen(url) + meta = download.info() + file_size = int(meta.get("Content-Length")[0]) + print("%s is %s bytes." % (file_path, file_size)) + with open(file_path, 'wb') as code: + chunk_size = 8192 + bytes_read = 0 + while True: + data = download.read(chunk_size) + bytes_read += len(data) + code.write(data) + status = r"%10d [%3.2f%%]" % (bytes_read, bytes_read * 100 / file_size) + status = status + chr(8) * (len(status) + 1) + print("\r", status, end=' ') + if len(data) < chunk_size: + break + + +# the package installer function runs the installer binary in macOS and pipes +# stdout and stderr to the python console the return code of the package run +# can be found in the pipes object (pipes.returncode). this is the reason we +# need to run # this is the bit where we can accept an optional command with +# arguments +def pkg_install(package): + pipes = subprocess.Popen(["sudo", "installer", "-pkg", package, "-target", "/"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = pipes.communicate() + if pipes.returncode == 1: + print(stdout) + print(stderr) + exit(1) + + +# the script executer executes any .sh file using bash and pipes stdout and +# stderr to the python console. the return code of the script execution can be +# found in the pipes object (pipes.returncode). +def script_exec(script): + pipes = subprocess.Popen(["/bin/bash", "-c", script], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + #for line in iter(pipes.stdout.readline, b''): + # print("*** " + line.rstrip()) + pipes.communicate() + if pipes.returncode == 1: + exit(1) + + +# the dmg installer is by far the most complicated function, because DMGs are +# more complicated than a .app inside we take the appropriate action. we also +# have the option to specify an optional command. since sometimes we must +# execute installer .apps or pkgs buried in the .app bundle, which is annoying. +def dmg_install(filename, installer, command=None): + pipes = subprocess.Popen(["hdiutil", "attach", filename], stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = pipes.communicate() + if pipes.returncode == 1: + print(stdout) + print(stderr) + exit(1) + volume_path = re.search(r'(\/Volumes\/).*$', stdout.decode('utf-8')).group(0) + installer_path = "%s/%s" % (volume_path, installer) + if command is not None and installer == '': + command = command.replace('${volume}', volume_path).encode("utf-8") + command = shlex.split(command) + pipes = subprocess.Popen( + command, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = pipes.communicate() + if pipes.returncode == 1: + print(stdout) + print(stderr) + exit(1) + if ".pkg" in installer: + pkg_install(installer_path) + if ".app" in installer: + applications_path = "/Applications/%s" % installer.rsplit('/', 1)[-1] + if os.path.exists(applications_path): + shutil.rmtree(applications_path) + shutil.copytree(installer_path, applications_path) + # current_user - the name of the user running the script. Apple suggests using + # both methods. + # uid - the UID of the user running the script + # gid - the GID of the group "admin" which the user account is expected to be a member of + # users in macOS + current_user = (SCDynamicStoreCopyConsoleUser(None, None, None) or [None])[0] + current_user = [current_user, ""][current_user in ["loginwindow", None, ""]] + uid = pwd.getpwnam(current_user).pw_uid + gid = grp.getgrnam("admin").gr_gid + os.chown(applications_path, uid, gid) + for root, dirs, files in os.walk(applications_path): + for d in dirs: + os.chown(os.path.join(root, d), uid, gid) + for f in files: + os.chown(os.path.join(root, f), uid, gid) + pipes = subprocess.Popen(["hdiutil", "detach", volume_path], stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = pipes.communicate() + if pipes.returncode == 1: + print(stdout) + print(stderr) + exit(1) + + +# the mobileconfig_install function installs configuration profiles +def mobileconfig_install(mobileconfig): + pipes = subprocess.Popen(["/usr/bin/profiles", "-I", "-F", mobileconfig], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = pipes.communicate() + if pipes.returncode == 1: + print(stdout) + print(stderr) + exit(1) + + +# autohash-firefox helper functions +# checks given hash summary page for the line that matches given locale and returns hash +def autohash_firefox_find_hash(hash_summary, locale): + for line in hash_summary.split(b'\n'): + if locale in line: + linesplit = line.split() + # In 79.0 hashes are encoded in b'' form. + # easiest way to interpret is to split it 2nd char to 66th char, since the text is + # literal, and not read as an encoded string + # hash = linesplit[0][2:66] + # In 80.0 hashes are not in b'' form, so splitting at element 0 gets the hash + hash = linesplit[0] + return hash + return False + + +def autohash_firefox(): + # find latest firefox version by following redirect + response = urllib.request.urlopen( + 'https://download.mozilla.org/?product=firefox-latest-ssl&os=osx&lang=en-US') + version = urllib.parse.urlsplit(response.geturl()).path.split('/')[-4] + # build url to fetch official SHA256SUMS page + firefox_hash_url = "https://releases.mozilla.org/pub/firefox/releases/%s/SHA256SUMS" % version + print(("NOTICE: Manifest file is instructing us to compare against official hash for " + "Firefox %s found at %s." % (version, firefox_hash_url))) + req = urllib.request.Request(firefox_hash_url) + response = urllib.request.urlopen(req) + hash_summary = response.read() + # find the hash for given locale + hash = autohash_firefox_find_hash(hash_summary, b'mac/en-US') + return hash + + +# the hash_file function accepts two arguments: the filename that you need to +# determine the SHA256 hash of and the expected hash it returns True or False. +def hash_file(filename, man_hash): + if man_hash == "skip": + print("NOTICE: Manifest file is instructing us to SKIP hashing %s." % filename) + elif man_hash == "autohash-firefox": + hash_check = hashlib.sha256() + with open(filename, 'rb') as downloaded_file: + for chunk in iter(lambda: downloaded_file.read(4096), b""): + hash_check.update(chunk) + if hash_check.hexdigest() == autohash_firefox().decode(): + print("\rThe hash for %s matches the official Firefox hash" % filename) + else: + print(hash_check.hexdigest()) + print(autohash_firefox().decode()) + print("WARNING: The the hash for %s is unexpected." % filename) + exit(1) + else: + hash_check = hashlib.sha256() + with open(filename, 'rb') as downloaded_file: + for chunk in iter(lambda: downloaded_file.read(4096), b""): + hash_check.update(chunk) + if hash_check.hexdigest() == man_hash: + print("\rThe hash for %s match the manifest file" % filename) + else: + print("WARNING: The the hash for %s is unexpected." % filename) + exit(1) + + +# the pointer_to_json function accepts the url of the file in the github repo +# and the password to the repo. the pointer file is read from github then +# parsed and the "oid sha256" and "size" are extracted from the pointer. an +# object is returned that contains a json request for the file that the pointer +# is associated with. +def pointer_to_json(dl_url): + content_result = urllib.request.urlopen(dl_url) + output = content_result.read() + content_result.close() + oid = re.search('(?m)^oid sha256:([a-z0-9]+)$', output) + size = re.search('(?m)^size ([0-9]+)$', output) + json_data = ( + '{"operation": "download", ' + '"transfers": ["basic"], ' + '"objects": [{"oid": "%s", "size": %s}]}' % (oid.group(1), size.group(1))) + return json_data + + +# the get_lfs_url function makes a request the the lfs API of the github repo, +# receives a JSON response. then gets the download URL from the JSON response +# and returns it. +def get_lfs_url(json_input, lfs_url): + req = urllib.request.Request(lfs_url, json_input) + req.add_header("Accept", "application/vnd.git-lfs+json") + req.add_header("Content-Type", "application/vnd.git-lfs+json") + result = urllib.request.urlopen(req) + results_python = json.load(result) + file_url = results_python['objects'][0]['actions']['download']['href'] + result.close() + return file_url + + +def main(): + # local_dir - the local directory the builder will use + # org - the org that is hosting the build repository + # repo - the rep that is hosting the build + # default_branch - the default branch to build against if no --branch argument is specified + # testing + local_dir = "/var/tmp/dinobuildr" + default_org = "mozilla" + default_repo = "dinobuildr" + default_branch = "master" + default_manifest = "production_manifest.json" + + # this section parses argument(s) passed to this script + # the --branch argument specified the branch that this script will build + # against, which is useful for testing. the script will default to the master + # branch if no argument is specified. + parser = argparse.ArgumentParser() + parser.add_argument("-b", "--branch", + help="The branch name to build against. Defaults to %s" % default_branch) + parser.add_argument("-m", "--manifest", + help="The manifest to build against. Defaults to %s" % default_manifest) + parser.add_argument("-r", "--repo", + help="The repo to build against. Defaults to %s" % default_repo) + parser.add_argument("-o", "--org", + help="The org to build against. Defaults to %s" % default_org) + + args = parser.parse_args() + + if args.branch is None: + branch = default_branch + else: + branch = args.branch + + if args.manifest is None: + manifest = default_manifest + else: + manifest = args.manifest + + if args.repo is None: + repo = default_repo + else: + repo = args.repo + + if args.org is None: + org = default_org + else: + org = args.org + + # os.environ - an environment variable for the builder's local directory to be + # passed on to shells scripts + os.environ["DINOPATH"] = local_dir + + # lfs_url - the generic LFS url structure that github uses + # raw_url - the generic RAW url structure that github uses + # manifest_url - the url of the manifest file + # manifest_hash - the hash of the manifest file + # manifest_file - the expected filepath of the manifest file + lfs_url = "https://github.com/%s/%s.git/info/lfs/objects/batch" % (org, repo) + raw_url = "https://raw.githubusercontent.com/%s/%s/%s/" % (org, repo, branch) + manifest_url = "https://raw.githubusercontent.com/%s/%s/%s/%s" % (org, repo, branch, manifest) + manifest_file = "%s/%s" % (local_dir, manifest) + + # check to see if user ran with sudo , since it's required + + if os.getuid() != 0: + print("This script requires root to run, please try again with sudo.") + exit(1) + + # if the local directory doesn't exist, we make it. + if not os.path.exists(local_dir): + os.makedirs(local_dir) + + # download the manifest.json file. + print("\nDownloading the manifest file and hash-checking it.\n") + print(manifest_url) + print(manifest_file) + downloader(manifest_url, manifest_file) + + print("\n***** DINOBUILDR IS BUILDING. RAWR. *****\n") + print("Building against the [%s] branch and the %s manifest\n" % (branch, manifest)) + # we read the manifest file and examine each object in it. if the object is a + # .pkg file, then we assemble the download url of the pointer, read the pointer + # and request the file from LFS. if the file we get has a hash that matches + # what's in the manifest, we Popen the installer function if the object is a + # .sh file, we assemble the download url and download the file directly. if the + # script we get has a hash that matches what's in the manifest, we set the + # execute flag and Popen the script_exec function. same with dmgs, although + # dmgs are real complicated so we may end up running an arbitrary command, + # copying the installer or installing a pkg. + with open(manifest_file, 'r') as manifest_data: + data = json.load(manifest_data) + + for item in data['packages']: + if item['filename'] != "": + file_name = item['filename'] + else: + file_name = ( + item['url'].replace('${version}', item['version']) + ).rsplit('/', 1)[-1] + # TODO: this variable name is dumb, this is the path to the file we're + # working with + local_path = "%s/%s" % (local_dir, file_name) + + if item['type'] == "pkg-lfs": + dl_url = raw_url + item['url'] + json_data = pointer_to_json(dl_url) + lfsfile_url = get_lfs_url(json_data, lfs_url) + print("Downloading:", item['item']) + downloader(lfsfile_url, local_path) + hash_file(local_path, item['hash']) + print("Installing:", item['item']) + pkg_install(local_path) + print("\r") + + if item['type'] == "pkg": + dl_url = item['url'].replace('${version}', item['version']) + print("Downloading:", item['item']) + downloader(dl_url, local_path) + hash_file(local_path, item['hash']) + print("Installing:", item['item']) + pkg_install(local_path) + print("\r") + + if item['type'] == "shell": + dl_url = raw_url + item['url'] + print("Downloading:", item['item']) + downloader(dl_url, local_path) + hash_file(local_path, item['hash']) + print("Executing:", item['item']) + perms = os.stat(local_path) + os.chmod(local_path, perms.st_mode | stat.S_IEXEC) + script_exec(local_path) + print("\r") + + if item['type'] == "dmg": + # TODO: consisitency: there should be URL checks everywhere or do this + # TODO: dmg-installer / dmg-advanced are not being checked to allow + # for functionality that should be in a downloader function + # in the manifest generator + if item['url'] == '': + print("No URL specified for %s" % item['item']) + break + dl_url = item['url'].replace('${version}', item['version']) + print("Downloading:", item['item']) + downloader(dl_url, local_path) + hash_file(local_path, item['hash']) + if item['dmg-installer'] != '': + print("Installing:", item['dmg-installer']) + if item['dmg-advanced'] != '': + print("Getting fancy and executing:", item['dmg-advanced']) + if item['dmg-installer'] == '' and item['dmg-advanced'] == '': + print(("No installer or install command specified for %s." + "Assuming this is download only." % item['item'])) + if item['dmg-installer'] != '': + dmg_install(local_path, item['dmg-installer']) + if item['dmg-advanced'] != '': + dmg_install(local_path, '', item['dmg-advanced']) + print("\r") + + if item['type'] == "file-lfs": + if item['url'] == '': + print("No URL specified for %s" % item['item']) + break + dl_url = raw_url + item['url'] + json_data = pointer_to_json(dl_url) + lfsfile_url = get_lfs_url(json_data, lfs_url) + print("Downloading:", item['item']) + downloader(lfsfile_url, local_path) + hash_file(local_path, item['hash']) + print("File downloaded to:", local_path) + print("\r") + + if item['type'] == "file": + if item['url'] == '': + print("No URL specified for %s" % item['item']) + break + dl_url = raw_url + item['url'] + print("Downloading:", item['item']) + downloader(dl_url, local_path) + hash_file(local_path, item['hash']) + print("File downloaded to:", local_path) + print("\r") + + if item['type'] == "mobileconfig": + dl_url = raw_url + item['url'] + print("Downloading:", item['item']) + downloader(dl_url, local_path) + hash_file(local_path, item['hash']) + print("Applying Mobileconfig:", item['item']) + mobileconfig_install(local_path) + print("\r") + + # delete the temporary directory we've been downloading packages into. + print("Cleanup: Deleting %s" % local_dir) + shutil.rmtree(local_dir) + + print("Build complete!") + + +if __name__ == '__main__': + main() diff --git a/dinobuildr3.sh b/dinobuildr3.sh new file mode 100755 index 0000000..2b0f3c3 --- /dev/null +++ b/dinobuildr3.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Written by Lucius Bono and Tristan Thomas + +# This script grabs the latest config.py script from the dinobuilder +# Github repo (https://github.com/mozilla/dinobuildr). At this time, +# we blindly trust the master branch, but later we hope to put more +# elaborate checks in place to ensure that the script is legit. + +# Accepts optional parameters that specifies the branch and manifest +# to build against. + +branch=master +manifest=production_manifest.json +repo=dinobuildr +org=mozilla +dino_engine=dino_engine3.py + +while :; do + case $1 in + -b|--branch) branch=$2 + shift + ;; + -m|--manifest) manifest=$2 + shift + ;; + -r|--repo) repo=$2 + shift + ;; + -o|--org) org=$2 + shift + ;; + *) break + esac + shift +done + +if [ "$branch" != '' ]; then + if [[ "$branch" =~ [^a-zA-Z0-9{-.}] ]]; then + echo "********************************************************************" + echo "Branch name must be numbers, letters, - and . only." + exit 1 + fi +fi + +if [ "$manifest" != '' ]; then + if [[ "$manifest" =~ [^a-zA-Z0-9{._}] ]]; then + echo "********************************************************************" + echo "Manifest name must be numbers, letters, . and _ only" + exit 1 + fi +fi + +if [ "$repo" != '' ]; then + if [[ "$repo" =~ [^a-zA-Z0-9{._}] ]]; then + echo "********************************************************************" + echo "Repo name must be numbers, letters, . and _ only" + exit 1 + fi +fi + +if [ "$org" != '' ]; then + if [[ "$org" =~ [^a-zA-Z0-9{._}] ]]; then + echo "********************************************************************" + echo "Org name must be numbers, letters, . and _ only" + exit 1 + fi +fi + + +printf "\nInstalling Command Line Tools for python3 - please click through the prompt to install" +xcode-select --install + +printf "\nInstalling the pyobjc-framework-SystemConfiguration package for system management" +pip3 install pyobjc-framework-SystemConfiguration + + +printf "\nPulling down dinobuildr from the [%s] branch from the [%s] repo in the [%s] org on github. +\n\n" "$branch" "$repo" "$org" + +build_script=$(curl -f https://raw.githubusercontent.com/"$org"/"$repo"/"$branch"/"$dino_engine") +curl_status=$? + +# If curl fails for some reason, we return it's non-zero exit code so that the +# script can fail in a predictable way. + +if [ $curl_status -eq 0 ]; then + printf "\nStarting the build!\n" + if python3 -c "$build_script" -b "$branch" -m "$manifest" -r "$repo" -o "$org"; then + echo "Rebooting!" + osascript -e 'tell app "System Events" to restart' + else + echo "********************************************************************" + echo "HOUSTON WE HAVE A PROBLEM: Dinobuildr did not complete successfully." + echo "Please review the error and run the this build again." + fi +else + echo "********************************************************************" + echo "Uh oh, unable to download Dinobuildr from Github." + if [ $curl_status -eq 6 ]; then + echo "Check your internet connection and try again!" + elif [ $curl_status -eq 22 ]; then + echo "We can't find the branch or repo you're trying to use. Please try again!" + fi + exit 1 +fi