new files for py3
This commit is contained in:
Jonathan Lin 2022-03-31 17:14:21 -04:00
Родитель 77bc26679c
Коммит 4351d8a1fb
3 изменённых файлов: 554 добавлений и 0 удалений

7
.vscode/settings.json поставляемый Normal file
Просмотреть файл

@ -0,0 +1,7 @@
{
"workbench.colorCustomizations": {
"activityBar.background": "#32068B",
"titleBar.activeBackground": "#4608C3",
"titleBar.activeForeground": "#FEFDFF"
}
}

437
dino_engine3.py Executable file
Просмотреть файл

@ -0,0 +1,437 @@
#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from http.client import HTTPMessage
import subprocess
import json
import os
import hashlib
import urllib.request, urllib.error, urllib.parse
import re
import stat
import shutil
import shlex
import pwd
import grp
import argparse
from SystemConfiguration import SCDynamicStoreCopyConsoleUser
# the downloader function accepts two arguments: the url of the file you are
# downloading and the filename (path) of the file you are downloading. The
# downloader reads the Content-Length portion of the header of the incoming
# file and determines the expected file size then reads the incoming file in
# chunks of 8192 bytes and displays the currently read bytes and percentage
# complete
def downloader(url, file_path):
download = urllib.request.urlopen(url)
meta = download.info()
file_size = int(meta.get("Content-Length")[0])
print("%s is %s bytes." % (file_path, file_size))
with open(file_path, 'wb') as code:
chunk_size = 8192
bytes_read = 0
while True:
data = download.read(chunk_size)
bytes_read += len(data)
code.write(data)
status = r"%10d [%3.2f%%]" % (bytes_read, bytes_read * 100 / file_size)
status = status + chr(8) * (len(status) + 1)
print("\r", status, end=' ')
if len(data) < chunk_size:
break
# the package installer function runs the installer binary in macOS and pipes
# stdout and stderr to the python console the return code of the package run
# can be found in the pipes object (pipes.returncode). this is the reason we
# need to run # this is the bit where we can accept an optional command with
# arguments
def pkg_install(package):
pipes = subprocess.Popen(["sudo", "installer", "-pkg", package, "-target", "/"],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = pipes.communicate()
if pipes.returncode == 1:
print(stdout)
print(stderr)
exit(1)
# the script executer executes any .sh file using bash and pipes stdout and
# stderr to the python console. the return code of the script execution can be
# found in the pipes object (pipes.returncode).
def script_exec(script):
pipes = subprocess.Popen(["/bin/bash", "-c", script],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
#for line in iter(pipes.stdout.readline, b''):
# print("*** " + line.rstrip())
pipes.communicate()
if pipes.returncode == 1:
exit(1)
# the dmg installer is by far the most complicated function, because DMGs are
# more complicated than a .app inside we take the appropriate action. we also
# have the option to specify an optional command. since sometimes we must
# execute installer .apps or pkgs buried in the .app bundle, which is annoying.
def dmg_install(filename, installer, command=None):
pipes = subprocess.Popen(["hdiutil", "attach", filename], stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = pipes.communicate()
if pipes.returncode == 1:
print(stdout)
print(stderr)
exit(1)
volume_path = re.search(r'(\/Volumes\/).*$', stdout.decode('utf-8')).group(0)
installer_path = "%s/%s" % (volume_path, installer)
if command is not None and installer == '':
command = command.replace('${volume}', volume_path).encode("utf-8")
command = shlex.split(command)
pipes = subprocess.Popen(
command,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = pipes.communicate()
if pipes.returncode == 1:
print(stdout)
print(stderr)
exit(1)
if ".pkg" in installer:
pkg_install(installer_path)
if ".app" in installer:
applications_path = "/Applications/%s" % installer.rsplit('/', 1)[-1]
if os.path.exists(applications_path):
shutil.rmtree(applications_path)
shutil.copytree(installer_path, applications_path)
# current_user - the name of the user running the script. Apple suggests using
# both methods.
# uid - the UID of the user running the script
# gid - the GID of the group "admin" which the user account is expected to be a member of
# users in macOS
current_user = (SCDynamicStoreCopyConsoleUser(None, None, None) or [None])[0]
current_user = [current_user, ""][current_user in ["loginwindow", None, ""]]
uid = pwd.getpwnam(current_user).pw_uid
gid = grp.getgrnam("admin").gr_gid
os.chown(applications_path, uid, gid)
for root, dirs, files in os.walk(applications_path):
for d in dirs:
os.chown(os.path.join(root, d), uid, gid)
for f in files:
os.chown(os.path.join(root, f), uid, gid)
pipes = subprocess.Popen(["hdiutil", "detach", volume_path], stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = pipes.communicate()
if pipes.returncode == 1:
print(stdout)
print(stderr)
exit(1)
# the mobileconfig_install function installs configuration profiles
def mobileconfig_install(mobileconfig):
pipes = subprocess.Popen(["/usr/bin/profiles", "-I", "-F", mobileconfig],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = pipes.communicate()
if pipes.returncode == 1:
print(stdout)
print(stderr)
exit(1)
# autohash-firefox helper functions
# checks given hash summary page for the line that matches given locale and returns hash
def autohash_firefox_find_hash(hash_summary, locale):
for line in hash_summary.split(b'\n'):
if locale in line:
linesplit = line.split()
# In 79.0 hashes are encoded in b'' form.
# easiest way to interpret is to split it 2nd char to 66th char, since the text is
# literal, and not read as an encoded string
# hash = linesplit[0][2:66]
# In 80.0 hashes are not in b'' form, so splitting at element 0 gets the hash
hash = linesplit[0]
return hash
return False
def autohash_firefox():
# find latest firefox version by following redirect
response = urllib.request.urlopen(
'https://download.mozilla.org/?product=firefox-latest-ssl&os=osx&lang=en-US')
version = urllib.parse.urlsplit(response.geturl()).path.split('/')[-4]
# build url to fetch official SHA256SUMS page
firefox_hash_url = "https://releases.mozilla.org/pub/firefox/releases/%s/SHA256SUMS" % version
print(("NOTICE: Manifest file is instructing us to compare against official hash for "
"Firefox %s found at %s." % (version, firefox_hash_url)))
req = urllib.request.Request(firefox_hash_url)
response = urllib.request.urlopen(req)
hash_summary = response.read()
# find the hash for given locale
hash = autohash_firefox_find_hash(hash_summary, b'mac/en-US')
return hash
# the hash_file function accepts two arguments: the filename that you need to
# determine the SHA256 hash of and the expected hash it returns True or False.
def hash_file(filename, man_hash):
if man_hash == "skip":
print("NOTICE: Manifest file is instructing us to SKIP hashing %s." % filename)
elif man_hash == "autohash-firefox":
hash_check = hashlib.sha256()
with open(filename, 'rb') as downloaded_file:
for chunk in iter(lambda: downloaded_file.read(4096), b""):
hash_check.update(chunk)
if hash_check.hexdigest() == autohash_firefox().decode():
print("\rThe hash for %s matches the official Firefox hash" % filename)
else:
print(hash_check.hexdigest())
print(autohash_firefox().decode())
print("WARNING: The the hash for %s is unexpected." % filename)
exit(1)
else:
hash_check = hashlib.sha256()
with open(filename, 'rb') as downloaded_file:
for chunk in iter(lambda: downloaded_file.read(4096), b""):
hash_check.update(chunk)
if hash_check.hexdigest() == man_hash:
print("\rThe hash for %s match the manifest file" % filename)
else:
print("WARNING: The the hash for %s is unexpected." % filename)
exit(1)
# the pointer_to_json function accepts the url of the file in the github repo
# and the password to the repo. the pointer file is read from github then
# parsed and the "oid sha256" and "size" are extracted from the pointer. an
# object is returned that contains a json request for the file that the pointer
# is associated with.
def pointer_to_json(dl_url):
content_result = urllib.request.urlopen(dl_url)
output = content_result.read()
content_result.close()
oid = re.search('(?m)^oid sha256:([a-z0-9]+)$', output)
size = re.search('(?m)^size ([0-9]+)$', output)
json_data = (
'{"operation": "download", '
'"transfers": ["basic"], '
'"objects": [{"oid": "%s", "size": %s}]}' % (oid.group(1), size.group(1)))
return json_data
# the get_lfs_url function makes a request the the lfs API of the github repo,
# receives a JSON response. then gets the download URL from the JSON response
# and returns it.
def get_lfs_url(json_input, lfs_url):
req = urllib.request.Request(lfs_url, json_input)
req.add_header("Accept", "application/vnd.git-lfs+json")
req.add_header("Content-Type", "application/vnd.git-lfs+json")
result = urllib.request.urlopen(req)
results_python = json.load(result)
file_url = results_python['objects'][0]['actions']['download']['href']
result.close()
return file_url
def main():
# local_dir - the local directory the builder will use
# org - the org that is hosting the build repository
# repo - the rep that is hosting the build
# default_branch - the default branch to build against if no --branch argument is specified
# testing
local_dir = "/var/tmp/dinobuildr"
default_org = "mozilla"
default_repo = "dinobuildr"
default_branch = "master"
default_manifest = "production_manifest.json"
# this section parses argument(s) passed to this script
# the --branch argument specified the branch that this script will build
# against, which is useful for testing. the script will default to the master
# branch if no argument is specified.
parser = argparse.ArgumentParser()
parser.add_argument("-b", "--branch",
help="The branch name to build against. Defaults to %s" % default_branch)
parser.add_argument("-m", "--manifest",
help="The manifest to build against. Defaults to %s" % default_manifest)
parser.add_argument("-r", "--repo",
help="The repo to build against. Defaults to %s" % default_repo)
parser.add_argument("-o", "--org",
help="The org to build against. Defaults to %s" % default_org)
args = parser.parse_args()
if args.branch is None:
branch = default_branch
else:
branch = args.branch
if args.manifest is None:
manifest = default_manifest
else:
manifest = args.manifest
if args.repo is None:
repo = default_repo
else:
repo = args.repo
if args.org is None:
org = default_org
else:
org = args.org
# os.environ - an environment variable for the builder's local directory to be
# passed on to shells scripts
os.environ["DINOPATH"] = local_dir
# lfs_url - the generic LFS url structure that github uses
# raw_url - the generic RAW url structure that github uses
# manifest_url - the url of the manifest file
# manifest_hash - the hash of the manifest file
# manifest_file - the expected filepath of the manifest file
lfs_url = "https://github.com/%s/%s.git/info/lfs/objects/batch" % (org, repo)
raw_url = "https://raw.githubusercontent.com/%s/%s/%s/" % (org, repo, branch)
manifest_url = "https://raw.githubusercontent.com/%s/%s/%s/%s" % (org, repo, branch, manifest)
manifest_file = "%s/%s" % (local_dir, manifest)
# check to see if user ran with sudo , since it's required
if os.getuid() != 0:
print("This script requires root to run, please try again with sudo.")
exit(1)
# if the local directory doesn't exist, we make it.
if not os.path.exists(local_dir):
os.makedirs(local_dir)
# download the manifest.json file.
print("\nDownloading the manifest file and hash-checking it.\n")
print(manifest_url)
print(manifest_file)
downloader(manifest_url, manifest_file)
print("\n***** DINOBUILDR IS BUILDING. RAWR. *****\n")
print("Building against the [%s] branch and the %s manifest\n" % (branch, manifest))
# we read the manifest file and examine each object in it. if the object is a
# .pkg file, then we assemble the download url of the pointer, read the pointer
# and request the file from LFS. if the file we get has a hash that matches
# what's in the manifest, we Popen the installer function if the object is a
# .sh file, we assemble the download url and download the file directly. if the
# script we get has a hash that matches what's in the manifest, we set the
# execute flag and Popen the script_exec function. same with dmgs, although
# dmgs are real complicated so we may end up running an arbitrary command,
# copying the installer or installing a pkg.
with open(manifest_file, 'r') as manifest_data:
data = json.load(manifest_data)
for item in data['packages']:
if item['filename'] != "":
file_name = item['filename']
else:
file_name = (
item['url'].replace('${version}', item['version'])
).rsplit('/', 1)[-1]
# TODO: this variable name is dumb, this is the path to the file we're
# working with
local_path = "%s/%s" % (local_dir, file_name)
if item['type'] == "pkg-lfs":
dl_url = raw_url + item['url']
json_data = pointer_to_json(dl_url)
lfsfile_url = get_lfs_url(json_data, lfs_url)
print("Downloading:", item['item'])
downloader(lfsfile_url, local_path)
hash_file(local_path, item['hash'])
print("Installing:", item['item'])
pkg_install(local_path)
print("\r")
if item['type'] == "pkg":
dl_url = item['url'].replace('${version}', item['version'])
print("Downloading:", item['item'])
downloader(dl_url, local_path)
hash_file(local_path, item['hash'])
print("Installing:", item['item'])
pkg_install(local_path)
print("\r")
if item['type'] == "shell":
dl_url = raw_url + item['url']
print("Downloading:", item['item'])
downloader(dl_url, local_path)
hash_file(local_path, item['hash'])
print("Executing:", item['item'])
perms = os.stat(local_path)
os.chmod(local_path, perms.st_mode | stat.S_IEXEC)
script_exec(local_path)
print("\r")
if item['type'] == "dmg":
# TODO: consisitency: there should be URL checks everywhere or do this
# TODO: dmg-installer / dmg-advanced are not being checked to allow
# for functionality that should be in a downloader function
# in the manifest generator
if item['url'] == '':
print("No URL specified for %s" % item['item'])
break
dl_url = item['url'].replace('${version}', item['version'])
print("Downloading:", item['item'])
downloader(dl_url, local_path)
hash_file(local_path, item['hash'])
if item['dmg-installer'] != '':
print("Installing:", item['dmg-installer'])
if item['dmg-advanced'] != '':
print("Getting fancy and executing:", item['dmg-advanced'])
if item['dmg-installer'] == '' and item['dmg-advanced'] == '':
print(("No installer or install command specified for %s."
"Assuming this is download only." % item['item']))
if item['dmg-installer'] != '':
dmg_install(local_path, item['dmg-installer'])
if item['dmg-advanced'] != '':
dmg_install(local_path, '', item['dmg-advanced'])
print("\r")
if item['type'] == "file-lfs":
if item['url'] == '':
print("No URL specified for %s" % item['item'])
break
dl_url = raw_url + item['url']
json_data = pointer_to_json(dl_url)
lfsfile_url = get_lfs_url(json_data, lfs_url)
print("Downloading:", item['item'])
downloader(lfsfile_url, local_path)
hash_file(local_path, item['hash'])
print("File downloaded to:", local_path)
print("\r")
if item['type'] == "file":
if item['url'] == '':
print("No URL specified for %s" % item['item'])
break
dl_url = raw_url + item['url']
print("Downloading:", item['item'])
downloader(dl_url, local_path)
hash_file(local_path, item['hash'])
print("File downloaded to:", local_path)
print("\r")
if item['type'] == "mobileconfig":
dl_url = raw_url + item['url']
print("Downloading:", item['item'])
downloader(dl_url, local_path)
hash_file(local_path, item['hash'])
print("Applying Mobileconfig:", item['item'])
mobileconfig_install(local_path)
print("\r")
# delete the temporary directory we've been downloading packages into.
print("Cleanup: Deleting %s" % local_dir)
shutil.rmtree(local_dir)
print("Build complete!")
if __name__ == '__main__':
main()

110
dinobuildr3.sh Executable file
Просмотреть файл

@ -0,0 +1,110 @@
#!/bin/bash
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Written by Lucius Bono and Tristan Thomas
# This script grabs the latest config.py script from the dinobuilder
# Github repo (https://github.com/mozilla/dinobuildr). At this time,
# we blindly trust the master branch, but later we hope to put more
# elaborate checks in place to ensure that the script is legit.
# Accepts optional parameters that specifies the branch and manifest
# to build against.
branch=master
manifest=production_manifest.json
repo=dinobuildr
org=mozilla
dino_engine=dino_engine3.py
while :; do
case $1 in
-b|--branch) branch=$2
shift
;;
-m|--manifest) manifest=$2
shift
;;
-r|--repo) repo=$2
shift
;;
-o|--org) org=$2
shift
;;
*) break
esac
shift
done
if [ "$branch" != '' ]; then
if [[ "$branch" =~ [^a-zA-Z0-9{-.}] ]]; then
echo "********************************************************************"
echo "Branch name must be numbers, letters, - and . only."
exit 1
fi
fi
if [ "$manifest" != '' ]; then
if [[ "$manifest" =~ [^a-zA-Z0-9{._}] ]]; then
echo "********************************************************************"
echo "Manifest name must be numbers, letters, . and _ only"
exit 1
fi
fi
if [ "$repo" != '' ]; then
if [[ "$repo" =~ [^a-zA-Z0-9{._}] ]]; then
echo "********************************************************************"
echo "Repo name must be numbers, letters, . and _ only"
exit 1
fi
fi
if [ "$org" != '' ]; then
if [[ "$org" =~ [^a-zA-Z0-9{._}] ]]; then
echo "********************************************************************"
echo "Org name must be numbers, letters, . and _ only"
exit 1
fi
fi
printf "\nInstalling Command Line Tools for python3 - please click through the prompt to install"
xcode-select --install
printf "\nInstalling the pyobjc-framework-SystemConfiguration package for system management"
pip3 install pyobjc-framework-SystemConfiguration
printf "\nPulling down dinobuildr from the [%s] branch from the [%s] repo in the [%s] org on github.
\n\n" "$branch" "$repo" "$org"
build_script=$(curl -f https://raw.githubusercontent.com/"$org"/"$repo"/"$branch"/"$dino_engine")
curl_status=$?
# If curl fails for some reason, we return it's non-zero exit code so that the
# script can fail in a predictable way.
if [ $curl_status -eq 0 ]; then
printf "\nStarting the build!\n"
if python3 -c "$build_script" -b "$branch" -m "$manifest" -r "$repo" -o "$org"; then
echo "Rebooting!"
osascript -e 'tell app "System Events" to restart'
else
echo "********************************************************************"
echo "HOUSTON WE HAVE A PROBLEM: Dinobuildr did not complete successfully."
echo "Please review the error and run the this build again."
fi
else
echo "********************************************************************"
echo "Uh oh, unable to download Dinobuildr from Github."
if [ $curl_status -eq 6 ]; then
echo "Check your internet connection and try again!"
elif [ $curl_status -eq 22 ]; then
echo "We can't find the branch or repo you're trying to use. Please try again!"
fi
exit 1
fi