This commit is contained in:
Rehan Dalal 2017-09-07 04:29:17 -04:00
Родитель 4f2c268957
Коммит 3c16a6e454
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 410D198EEF339E0B
11 изменённых файлов: 462 добавлений и 158 удалений

1
.gitignore поставляемый
Просмотреть файл

@ -1,5 +1,6 @@
*.xpi
.morgoth.yml
/.morgoth/
/releases/
# Byte-compiled / optimized / DLL files

Просмотреть файл

@ -1,141 +0,0 @@
#!/usr/bin/env python3
import os
import json
import tempfile
import boto3
import click
from colorama import Fore, Style
from morgoth.conf import settings
from morgoth.xpi import XPI
aws_settings = settings['aws']
def close(code):
"""An alias for exit that resets the terminal styling."""
print(Style.RESET_ALL)
exit(code)
@click.group()
def cli():
pass
@cli.command()
@click.option('--profile', default=aws_settings.get('profile'))
@click.argument('xpi_file')
def mkrelease(xpi_file, profile):
prefix = aws_settings['prefix']
try:
xpi = XPI(xpi_file)
except XPI.DoesNotExist:
print(Fore.RED + 'File does not exist.')
close(1)
except XPI.BadZipfile:
print(Fore.RED + 'XPI cannot be unzipped.')
close(1)
except XPI.BadXPIfile:
print(Fore.RED + 'XPI is not properly configured.')
close(1)
else:
print(Fore.CYAN + 'Found: {}'.format(xpi.release_name))
if not click.confirm(Style.RESET_ALL + 'Is this correct?'):
print(Fore.RED + 'Release could not be auto-generated.')
close(1)
session = boto3.Session(profile_name=profile)
s3 = session.resource('s3')
bucket = s3.Bucket(settings.get('aws', {})['bucket_name'])
exists = False
for obj in bucket.objects.filter(Prefix=prefix):
if obj.key == xpi.get_ftp_path(prefix):
exists = True
uploaded = False
if exists:
tmpdir = tempfile.mkdtemp()
download_path = os.path.join(tmpdir, xpi.file_name)
bucket.download_file(xpi.get_ftp_path(prefix), download_path)
uploaded_xpi = XPI(download_path)
if uploaded_xpi.sha512sum == xpi.sha512sum:
print(Fore.GREEN + 'XPI already uploaded.')
uploaded = True
else:
print(Fore.YELLOW + 'XPI with matching filename already uploaded.')
if not uploaded:
if exists and not click.confirm(Style.RESET_ALL + 'Would you like to replace it?'):
print(Fore.RED + 'Aborting.')
close(1)
with open(xpi.path, 'rb') as data:
bucket.put_object(Key=xpi.get_ftp_path(prefix), Body=data)
print(Fore.GREEN + 'XPI uploaded successfully.')
json_path = 'releases/{}.json'.format(xpi.release_name)
if os.path.exists(json_path):
print(Fore.YELLOW + 'Release JSON file already exists.')
if not click.confirm(Style.RESET_ALL + 'Replace existing release JSON file?'):
print(Fore.RED + 'Aborting.')
close(1)
print(Style.RESET_ALL + 'Saving to: {}{}'.format(Style.BRIGHT, json_path))
os.makedirs('releases', exist_ok=True)
with open(json_path, 'w') as f:
f.write(json.dumps(
xpi.generate_release_data(aws_settings['base_url'], prefix),
indent=2, sort_keys=True))
close(0)
@cli.command()
@click.argument('releases', nargs=-1)
def mksuperblob(releases):
names = []
for release in releases:
with open(release, 'r') as f:
release_data = json.loads(f.read())
short_name = release_data['name'].split('@')[0]
for k in release_data['addons']:
if k.startswith(short_name):
version = release_data['addons'][k]['version']
names.append('{}-{}'.format(short_name, version))
if not len(names):
print(Fore.RED + 'No releases specified.')
close(1)
names.sort()
sb_name = 'Superblob-{}'.format('-'.join(names))
sb_data = {
'blobs': names,
'name': sb_name,
'schema_version': 4000
}
sb_path = 'releases/superblobs/{}.json'.format(sb_name)
os.makedirs('releases/superblobs', exist_ok=True)
with open(sb_path, 'w') as f:
f.write(json.dumps(sb_data, indent=2, sort_keys=True))
print(Style.RESET_ALL + 'Saving to: {}{}'.format(Style.BRIGHT, sb_path))
close(0)
if __name__ == '__main__':
cli()

Просмотреть файл

@ -0,0 +1,7 @@
import os
BASE_PATH = os.getcwd()
MORGOTH_PATH = os.path.join(BASE_PATH, '.morgoth')
ENVIRONMENT_PATH = os.path.join(MORGOTH_PATH, 'env')
CONFIG_PATH = os.path.join(MORGOTH_PATH, 'config')

233
morgoth/cli.py Executable file
Просмотреть файл

@ -0,0 +1,233 @@
import os
import json
import tempfile
import boto3
import click
from colorama import Fore, Style
from morgoth import ENVIRONMENT_PATH
from morgoth.environment import Environment
from morgoth.settings import GPGImproperlyConfigured, settings
from morgoth.utils import output, validate_environment
from morgoth.xpi import XPI
@click.group()
def cli():
pass
@cli.command()
@click.option('--environment', '-e', default=None)
@click.option('--username', '-u', default=None)
@click.pass_context
def init(ctx, environment, username):
"""Initialize a Morgoth repository."""
curr_env = None
if os.path.exists(ENVIRONMENT_PATH):
output('A repo has already been initialized in this directory.')
curr_env = Environment.from_file(ENVIRONMENT_PATH)
# Prompt for a password if a username is provided
password = None
if username:
password = click.prompt('Password', hide_input=True)
if environment:
environment = Environment(environment, username=username, password=password)
is_valid, message = validate_environment(environment)
if not is_valid:
output(message)
exit(1)
while not environment:
url = click.prompt('Environment URL')
environment = Environment(url, username=username, password=password)
is_valid, message = validate_environment(environment)
if not is_valid:
output(message)
environment = None
if curr_env and environment != curr_env:
if not click.confirm('Would you like to replace the existing environment?'):
exit(1)
elif curr_env:
output('Environment was unchanged.')
environment.save(ENVIRONMENT_PATH)
if username:
if click.confirm('Do you want to save your username?'):
ctx.invoke(config, key='username', value=username)
if click.confirm('Do you want to save your password?'):
ctx.invoke(config, key='password', value=password)
@cli.command()
@click.option('--username', '-u', default=None)
@click.pass_context
def auth(ctx, username):
"""Update authentication settings."""
if not username:
username = click.prompt('Username')
password = None
while not password:
password = click.prompt('Password', hide_input=True)
confirm_password = click.prompt('Confirm Password', hide_input=True)
if password != confirm_password:
output('Passwords did not match. Try again.')
password = None
ctx.invoke(config, key='username', value=username)
ctx.invoke(config, key='password', value=password)
@cli.command()
@click.option('--delete', '-d', is_flag=True)
@click.option('--list', '-l', is_flag=True)
@click.argument('key', default='')
@click.argument('value', default='')
def config(key, value, delete, list):
"""Get or set a configuration value."""
if list:
for section in settings.config:
for option in settings.config[section]:
key = '{}.{}'.format(section, option)
output('{} = {}'.format(key, settings.get(key)))
elif delete:
try:
settings.delete(key)
except KeyError:
output('Setting does not exist.')
exit(1)
elif value == '':
if settings.get(key):
output(settings.get(key))
exit(0)
elif key:
try:
settings.set(key, value)
except GPGImproperlyConfigured:
output('GPG settings improperly configured.')
exit(1)
settings.save()
@cli.command()
@click.option('--profile', default=settings.get('aws.profile'))
@click.argument('xpi_file')
def mkrelease(xpi_file, profile):
"""Make a new release from an XPI file."""
prefix = settings.get('aws.prefix')
try:
xpi = XPI(xpi_file)
except XPI.DoesNotExist:
output('File does not exist.', Fore.RED)
exit(1)
except XPI.BadZipfile:
output('XPI cannot be unzipped.', Fore.RED)
exit(1)
except XPI.BadXPIfile:
output('XPI is not properly configured.', Fore.RED)
exit(1)
else:
output('Found: {}'.format(xpi.release_name), Fore.CYAN)
if not click.confirm('Is this correct?'):
output('Release could not be auto-generated.', Fore.RED)
exit(1)
session = boto3.Session(profile_name=profile)
s3 = session.resource('s3')
bucket = s3.Bucket(settings.get('aws', {})['bucket_name'])
exists = False
for obj in bucket.objects.filter(Prefix=prefix):
if obj.key == xpi.get_ftp_path(prefix):
exists = True
uploaded = False
if exists:
tmpdir = tempfile.mkdtemp()
download_path = os.path.join(tmpdir, xpi.file_name)
bucket.download_file(xpi.get_ftp_path(prefix), download_path)
uploaded_xpi = XPI(download_path)
if uploaded_xpi.sha512sum == xpi.sha512sum:
output('XPI already uploaded.', Fore.GREEN)
uploaded = True
else:
output('XPI with matching filename already uploaded.', Fore.YELLOW)
if not uploaded:
if exists and not click.confirm('Would you like to replace it?'):
output('Aborting.', Fore.RED)
exit(1)
with open(xpi.path, 'rb') as data:
bucket.put_object(Key=xpi.get_ftp_path(prefix), Body=data)
output('XPI uploaded successfully.', Fore.GREEN)
json_path = 'releases/{}.json'.format(xpi.release_name)
if os.path.exists(json_path):
output('Release JSON file already exists.', Fore.YELLOW)
if not click.confirm('Replace existing release JSON file?'):
output('Aborting.', Fore.RED)
exit(1)
output('Saving to: {}{}'.format(Style.BRIGHT, json_path))
os.makedirs('releases', exist_ok=True)
with open(json_path, 'w') as f:
f.write(json.dumps(
xpi.generate_release_data(settings.get('aws.base_url'), prefix),
indent=2, sort_keys=True))
output('')
@cli.command()
@click.argument('releases', nargs=-1)
def mksuperblob(releases):
"""Make a new superblob from one or more releases."""
names = []
short_names = []
for release in releases:
with open(release, 'r') as f:
release_data = json.loads(f.read())
short_name = release_data['name'].split('@')[0]
for k in release_data['addons']:
if k.startswith(short_name):
version = release_data['addons'][k]['version']
names.append(release_data['name'])
short_names.append('{}-{}'.format(short_name, version))
if not len(names):
output('No releases specified.', Fore.RED)
exit(1)
names.sort()
sb_name = 'Superblob-{}'.format('-'.join(short_names))
sb_data = {
'blobs': names,
'name': sb_name,
'schema_version': 4000
}
sb_path = 'releases/superblobs/{}.json'.format(sb_name)
os.makedirs('releases/superblobs', exist_ok=True)
with open(sb_path, 'w') as f:
f.write(json.dumps(sb_data, indent=2, sort_keys=True))
output('Saving to: {}{}'.format(Style.BRIGHT, sb_path))
output('')

Просмотреть файл

@ -1,14 +0,0 @@
import os
import yaml
from colorama import Fore, Style
if not os.path.exists('.morgoth.yml'):
print(Fore.RED + 'Configuration file is missing.')
print(Style.RESET_ALL)
exit(1)
with open('.morgoth.yml', 'r') as f:
settings = yaml.safe_load(f)

59
morgoth/environment.py Normal file
Просмотреть файл

@ -0,0 +1,59 @@
import base64
import os
from urllib.error import URLError, HTTPError
from urllib.request import Request, urlopen
class Environment(object):
url = None
username = None
password = None
def __init__(self, url, **kwargs):
self.url = self._normalize_url(url)
self.username = kwargs.get('username', None)
self.password = kwargs.get('password', None)
def __eq__(self, other):
return isinstance(other, self.__class__) and self.url == other.url
@classmethod
def from_file(cls, path):
with open(path, 'r') as f:
return cls(f.read())
@staticmethod
def _normalize_url(url):
stripped = url.strip('\n\t\r /')
return '{}/'.format(stripped)
def get_url(self, endpoint):
return '{}api/{}'.format(self.url, endpoint)
def request(self, endpoint, **kwargs):
request = Request(self.get_url(endpoint), **kwargs)
if self.username and self.password:
auth = base64.encodebytes('{}:{}'.format(self.username, self.password).encode())
auth = auth.decode().replace('\n', '')
request.add_header('Authorization', 'Basic {}'.format(auth))
return urlopen(request)
def validate(self):
try:
response = self.request('rules')
except HTTPError as err:
if err.code == 401:
raise
return False
except (URLError, ValueError):
return False
return response.code == 200 and response.getheader('Content-Type') == 'application/json'
def save(self, path):
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, 'w') as f:
f.write(self.url)

93
morgoth/settings.py Normal file
Просмотреть файл

@ -0,0 +1,93 @@
import base64
import configparser
import gnupg
from morgoth import CONFIG_PATH
class GPGImproperlyConfigured(Exception):
pass
class Settings(object):
_path = None
def __init__(self, path):
self.config = configparser.ConfigParser()
self.path = path
@property
def path(self):
return self._path
@path.setter
def path(self, value):
if value != self._path:
self._path = value
with open(self._path, 'a+') as f:
f.seek(0)
self.config.read_file(f)
@property
def gpg(self):
if not self.get('gpg.fingerprint'):
raise GPGImproperlyConfigured()
return gnupg.GPG(binary=self.get('gpg.binary'), homedir=self.get('gpg.homedir'),
use_agent=True)
@staticmethod
def _parse_key(key):
keys = key.split('.', 1)
if len(keys) < 2:
keys = ['morgoth'] + keys
return keys
def _encrypt(self, value):
encrypted = self.gpg.encrypt(value, self.get('gpg.fingerprint'))
return base64.b64encode(str(encrypted).encode()).decode()
def _decrypt(self, value):
decoded = base64.b64decode(value.encode()).decode()
return self.gpg.decrypt(decoded)
def get(self, key, default=None, decrypt=False):
keys = self._parse_key(key)
try:
value = self.config[keys[0]][keys[1]]
except KeyError:
return default
if decrypt:
return self._decrypt(value)
return value
def _set(self, key, value):
keys = self._parse_key(key)
if not keys[0] in self.config:
self.config[keys[0]] = {}
self.config[keys[0]][keys[1]] = value
def set(self, key, value):
keys = self._parse_key(key)
if keys == ['morgoth', 'password']:
value = self._encrypt(value)
self._set(key, value)
def delete(self, key):
keys = self._parse_key(key)
del self.config[keys[0]][keys[1]]
def save(self):
with open(self.path, 'w') as f:
self.config.write(f)
settings = Settings(CONFIG_PATH)

19
morgoth/utils.py Normal file
Просмотреть файл

@ -0,0 +1,19 @@
from urllib.error import HTTPError
from colorama import Style
def output(str, *styles):
print(Style.RESET_ALL, end='')
if styles:
print(*styles, end='')
print(str, end='')
print(Style.RESET_ALL)
def validate_environment(env, **kwargs):
try:
is_valid = env.validate(**kwargs)
except HTTPError:
return False, 'Could not authenticate.'
return is_valid, '' if is_valid else 'Invalid environment.'

Просмотреть файл

@ -11,6 +11,28 @@ docutils==0.14 \
--hash=sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6 \
--hash=sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6 \
--hash=sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274
psutil==5.3.0 \
--hash=sha256:6f8f858cdb79397509ee067ae9d25bee8f4b4902453ac8d155fa1629f03aa39d \
--hash=sha256:b31d6d19e445b56559abaa21703a6bc4b162aaf9ab99867b6f2bbbdb2c7fce66 \
--hash=sha256:7f1ba5011095e39b3f543e9c87008409dd8a57a3e48ea1022c348244b5af77bf \
--hash=sha256:853f68a85cec0137acf0504d8ca6d40d899e48ecbe931130f593a072a35b812e \
--hash=sha256:01d9cb9473eee0e7e88319f9a5205a69e6e160b3ab2bd430a05b93bfae1528c2 \
--hash=sha256:91d37262095c1a0f97a78f5034e10e0108e3fa326c85baa17f8cdd63fa5f81b9 \
--hash=sha256:bd1776dc14b197388d728db72c103c0ebec834690ef1ce138035abf0123e2268 \
--hash=sha256:7fadb1b1357ef58821b3f1fc2afb6e1601609b0daa3b55c2fabf765e0ea98901 \
--hash=sha256:d5f4634a19e7d4692f37d8d67f8418f85f2bc1e2129914ec0e4208bf7838bf63 \
--hash=sha256:31505ee459913ef63fa4c1c0d9a11a4da60b5c5ec6a92d6d7f5d12b9653fc61b \
--hash=sha256:a3940e06e92c84ab6e82b95dad056241beea93c3c9b1d07ddf96485079855185 \
--hash=sha256:ba94f021942d6cc27e18dcdccd2c1a0976f0596765ef412316ecb887d4fd3db2 \
--hash=sha256:0f2fccf98bc25e8d6d61e24b2cc6350b8dfe8fa7f5251c817e977d8c61146e5d \
--hash=sha256:d06f02c53260d16fb445e426410263b2d271cea19136b1bb715cf10b76960359 \
--hash=sha256:724439fb20d083c943a2c62db1aa240fa15fe23644c4d4a1e9f573ffaf0bbddd \
--hash=sha256:a58708f3f6f74897450babb012cd8067f8911e7c8a1f2991643ec9937a8f6c15 \
--hash=sha256:108dae5ecb68f6e6212bf0553be055a2a0eec210227d8e14c3a26368b118624a \
--hash=sha256:9832124af1e9ec0f298f17ab11c3bb91164f8068ec9429c39a7f7a0eae637a94 \
--hash=sha256:7b8d10e7d72862d1e97caba546b60ce263b3fcecd6176e4c94efebef87ee68d3 \
--hash=sha256:ed1f7cbbbf778a6ed98e25d48fdbdc098e66b360427661712610d72c1b4cf5f5 \
--hash=sha256:3d8d62f3da0b38dbfaf4756a32e18c866530b9066c298da3fc293cfefae22f0a
python-dateutil==2.6.1 \
--hash=sha256:891c38b2a02f5bb1be3e4793866c8df49c7d19baabf9c1bad62547e0b4866aca \
--hash=sha256:95511bae634d69bc7329ba55e646499a842bc4ec342ad54a8cdb65645a0aad3c

Просмотреть файл

@ -4,6 +4,13 @@ boto3==1.4.7 \
click==6.7 \
--hash=sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d \
--hash=sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b
colorama==0.3.9 \
--hash=sha256:463f8483208e921368c9f306094eb6f725c6ca42b0f97e313cb5d5512459feda \
--hash=sha256:48eb22f4f8461b1df5734a074b57042430fb06e1d61bd1e11b078c0fe6d7a1f1
gnupg==2.3.1 \
--hash=sha256:d2e16c486aaeecbb65633f8493ccab025e2487c0e1dc59a83c520b6f81dff9b8 \
--hash=sha256:be656a2f693dbac4362537c1b6cfd03131ac5ce7a4b2bb21bff7e3145f94f7ea \
--hash=sha256:8db5a05c369dbc231dab4c98515ce828f2dffdc14f1534441a6c59b71c6d2031
PyYAML==3.12 \
--hash=sha256:3262c96a1ca437e7e4763e2843746588a965426550f3797a79fca9c6199c431f \
--hash=sha256:16b20e970597e051997d90dc2cddc713a2876c47e3d92d59ee198700c5427736 \
@ -19,6 +26,3 @@ PyYAML==3.12 \
--hash=sha256:0c507b7f74b3d2dd4d1322ec8a94794927305ab4cebbe89cc47fe5e81541e6e8 \
--hash=sha256:b4c423ab23291d3945ac61346feeb9a0dc4184999ede5e7c43e1ffb975130ae6 \
--hash=sha256:5ac82e411044fb129bae5cfbeb3ba626acb2af31a8d17d175004b70862a741a7
colorama==0.3.9 \
--hash=sha256:463f8483208e921368c9f306094eb6f725c6ca42b0f97e313cb5d5512459feda \
--hash=sha256:48eb22f4f8461b1df5734a074b57042430fb06e1d61bd1e11b078c0fe6d7a1f1

21
setup.py Normal file
Просмотреть файл

@ -0,0 +1,21 @@
from setuptools import setup
setup(
name='morgoth',
version='0.1.0',
py_modules=[
'morgoth',
],
install_requires=[
'boto3',
'Click',
'colorama',
'gnupg',
],
entry_points={
'console_scripts': [
'morgoth = morgoth.cli:cli',
],
},
)