relengapi_clobberer: update on work on clobberer
This commit is contained in:
Rok Garbas
Родитель
c5d01ad150
Коммит
b34a58a8d5
|
|
@ -4,10 +4,21 @@
|
|||
|
||||
from __future__ import absolute_import
|
||||
|
||||
from relengapi_common import create_app, db
|
||||
from relengapi_clobberer import _app
|
||||
import os
|
||||
|
||||
from relengapi_common import create_app, db
|
||||
|
||||
|
||||
here = os.path.dirname(__file__)
|
||||
|
||||
def init_app(app):
|
||||
app.api.register(
|
||||
os.path.join(here, "swagger.yml"),
|
||||
base_url=app.config.get('CLOBBERER_BASE_URL'),
|
||||
)
|
||||
|
||||
app = create_app(__name__, [db, init_app])
|
||||
|
||||
app = create_app(__name__, [db, _app])
|
||||
|
||||
if __name__ == '__main__':
|
||||
app.run(debug=True)
|
||||
|
|
|
|||
|
|
@ -1,37 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
from flask_restplus import Resource
|
||||
|
||||
|
||||
class App:
|
||||
pass
|
||||
|
||||
|
||||
def init_app(app):
|
||||
|
||||
clobberer = App()
|
||||
|
||||
@app.api.route('/buildbot')
|
||||
class Buildbot(Resource):
|
||||
|
||||
def get(self):
|
||||
return {'hello': 'world'}
|
||||
|
||||
def post(self):
|
||||
return {'hello': 'world'}
|
||||
|
||||
@app.api.route('/taskcluster')
|
||||
class Taskcluster(Resource):
|
||||
|
||||
def get(self):
|
||||
return {'hello': 'world'}
|
||||
|
||||
def post(self):
|
||||
return {'hello': 'world'}
|
||||
|
||||
|
||||
return clobberer
|
||||
|
|
@ -1,134 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
import time
|
||||
import re
|
||||
import datetime
|
||||
import wsme.types
|
||||
|
||||
from flask import g
|
||||
from flask_login import current_user
|
||||
|
||||
from relengapi_common.api import apimethod
|
||||
from relengapi_clobberer import api
|
||||
from relengapi_clobberer.models import DB_DECLARATIVE_BASE, ClobbererTimes
|
||||
|
||||
|
||||
__name__ = 'clobberer'
|
||||
|
||||
|
||||
def _add_clobber(app, session, branch, builddir, slave=None):
|
||||
"""
|
||||
A common method for adding clobber times to a session. The session passed
|
||||
in is returned; but is only committed if the commit option is True.
|
||||
"""
|
||||
match = re.search('^' + api.BUILDBOT_BUILDDIR_REL_PREFIX + '.*', builddir)
|
||||
if match is None:
|
||||
try:
|
||||
who = current_user.authenticated_email
|
||||
except AttributeError:
|
||||
if current_user.anonymous:
|
||||
who = 'anonymous'
|
||||
else:
|
||||
# TokenUser doesn't show up as anonymous; but also has no
|
||||
# authenticated_email
|
||||
who = 'automation'
|
||||
|
||||
clobberer_times = ClobbererTimes.as_unique(
|
||||
session,
|
||||
branch=branch,
|
||||
builddir=builddir,
|
||||
slave=slave,
|
||||
)
|
||||
clobberer_times.lastclobber = int(time.time())
|
||||
clobberer_times.who = who
|
||||
session.add(clobberer_times)
|
||||
return None
|
||||
app.log.debug('Rejecting clobber of builddir with release '
|
||||
'prefix: {}'.format(builddir))
|
||||
return None
|
||||
|
||||
|
||||
class Branch(wsme.types.Base):
|
||||
"""Represents branches of buildbot
|
||||
"""
|
||||
name = wsme.types.wsattr(unicode, mandatory=True)
|
||||
data = wsme.types.wsattr(
|
||||
{unicode: [unicode]}, mandatory=False, default=list())
|
||||
|
||||
|
||||
def init_app(app):
|
||||
|
||||
caches_to_skip = app.config.get('TASKCLUSTER_CACHES_TO_SKIP', [])
|
||||
|
||||
@app.route('/')
|
||||
def root():
|
||||
# TODO: point to tools page for clobberer or documentation
|
||||
return 'Clobberer is running ...'
|
||||
|
||||
@app.route('/buildbot', methods=['GET'])
|
||||
@apimethod([Branch])
|
||||
def get_buildout():
|
||||
"""List of all buildbot branches.
|
||||
"""
|
||||
session = g.db.session(DB_DECLARATIVE_BASE)
|
||||
# TODO: only cache this in production
|
||||
# branches = app.cache.cached()(api.buildbot_branches)(session)
|
||||
branches = api.buildbot_branches(session)
|
||||
return [
|
||||
Branch(
|
||||
name=branch['name'],
|
||||
data={
|
||||
name: [
|
||||
datetime.datetime.fromtimestamp(
|
||||
builder.lastclobber).strftime("%Y-%m-%d %H:%M:%S")
|
||||
for builder in builders
|
||||
if builder.lastclobber
|
||||
]
|
||||
for name, builders in branch['builders'].items()
|
||||
}
|
||||
)
|
||||
for branch in branches
|
||||
]
|
||||
|
||||
@app.route('/buildbot', methods=['POST'])
|
||||
@apimethod(unicode, body=[unicode, unicode])
|
||||
def post_buildout(body):
|
||||
"""
|
||||
Request clobbers for particular branches and builddirs.
|
||||
"""
|
||||
session = g.db.session(DB_DECLARATIVE_BASE)
|
||||
for clobber in body:
|
||||
_add_clobber(
|
||||
app,
|
||||
session,
|
||||
branch=clobber.branch,
|
||||
builddir=clobber.builddir,
|
||||
slave=clobber.slave
|
||||
)
|
||||
session.commit()
|
||||
return None
|
||||
|
||||
@app.route('/taskcluster', methods=['GET'])
|
||||
@apimethod([Branch])
|
||||
def get_taskcluster():
|
||||
"""List of all the gecko branches with their worker types
|
||||
"""
|
||||
branches = app.cache.cached()(api.taskcluster_branches)()
|
||||
return [
|
||||
Branch(
|
||||
name=branchName,
|
||||
data={
|
||||
workerName: filter(lambda x: x not in caches_to_skip, worker['caches']) # noqa
|
||||
for workerName, worker in branch['workerTypes'].items()
|
||||
}
|
||||
)
|
||||
for branchName, branch in branches.items()
|
||||
]
|
||||
|
||||
# TODO post_taskcluster
|
||||
|
||||
return api
|
||||
|
|
@ -4,131 +4,61 @@
|
|||
|
||||
from __future__ import absolute_import
|
||||
|
||||
import taskcluster
|
||||
|
||||
from sqlalchemy import and_
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy import not_
|
||||
|
||||
from relengapi_clobberer.models import ClobbererBuilds
|
||||
from relengapi_clobberer.models import ClobbererTimes
|
||||
from flask import g, current_app
|
||||
from relengapi_clobberer import models
|
||||
|
||||
|
||||
BUILDBOT_BUILDDIR_REL_PREFIX = 'rel-'
|
||||
BUILDBOT_BUILDER_REL_PREFIX = 'release-'
|
||||
TASKCLUSTER_DECISION_NAMESPACE = 'gecko.v2.%s.latest.firefox.decision'
|
||||
def get_buildbot():
|
||||
return models.buildbot_branches(g.db.session)
|
||||
|
||||
|
||||
def buildbot_branches(session):
|
||||
"""List of all buildbot branches.
|
||||
"""
|
||||
def post_buildbot(body):
|
||||
result = []
|
||||
|
||||
branches = session.query(ClobbererBuilds.branch).distinct()
|
||||
|
||||
# Users shouldn't see any branch associated with a release builddir
|
||||
branches = branches.filter(not_(
|
||||
ClobbererBuilds.builddir.startswith(BUILDBOT_BUILDDIR_REL_PREFIX)))
|
||||
|
||||
branches = branches.order_by(ClobbererBuilds.branch)
|
||||
|
||||
return [dict(name=branch[0],
|
||||
builders=buildbot_branch_summary(session, branch[0]))
|
||||
for branch in branches]
|
||||
|
||||
|
||||
def buildbot_branch_summary(session, branch):
|
||||
"""Return a dictionary of most recent ClobbererTimess grouped by
|
||||
buildername.
|
||||
"""
|
||||
# Isolates the maximum lastclobber for each builddir on a branch
|
||||
max_ct_sub_query = session.query(
|
||||
func.max(ClobbererTimes.lastclobber).label('lastclobber'),
|
||||
ClobbererTimes.builddir,
|
||||
ClobbererTimes.branch
|
||||
).group_by(
|
||||
ClobbererTimes.builddir,
|
||||
ClobbererTimes.branch
|
||||
).filter(ClobbererTimes.branch == branch).subquery()
|
||||
|
||||
# Finds the "greatest n per group" by joining with the
|
||||
# max_ct_sub_query
|
||||
# This is necessary to get the correct "who" values
|
||||
sub_query = session.query(ClobbererTimes).join(max_ct_sub_query, and_(
|
||||
ClobbererTimes.builddir == max_ct_sub_query.c.builddir,
|
||||
ClobbererTimes.lastclobber == max_ct_sub_query.c.lastclobber,
|
||||
ClobbererTimes.branch == max_ct_sub_query.c.branch)).subquery()
|
||||
|
||||
# Attaches builddirs, along with their max lastclobber to a
|
||||
# buildername
|
||||
full_query = session.query(
|
||||
ClobbererBuilds.buildername,
|
||||
ClobbererBuilds.builddir,
|
||||
sub_query.c.lastclobber,
|
||||
sub_query.c.who
|
||||
).outerjoin(
|
||||
sub_query,
|
||||
ClobbererBuilds.builddir == sub_query.c.builddir,
|
||||
).filter(
|
||||
ClobbererBuilds.branch == branch,
|
||||
not_(ClobbererBuilds.buildername.startswith(BUILDBOT_BUILDER_REL_PREFIX)) # noqa
|
||||
).distinct().order_by(ClobbererBuilds.buildername)
|
||||
|
||||
summary = dict()
|
||||
for result in full_query:
|
||||
buildername, builddir, lastclobber, who = result
|
||||
summary.setdefault(buildername, [])
|
||||
summary[buildername].append(
|
||||
ClobbererTimes(
|
||||
branch=branch,
|
||||
builddir=builddir,
|
||||
lastclobber=lastclobber,
|
||||
who=who
|
||||
try:
|
||||
for clobber in body:
|
||||
result.append(
|
||||
models.clobber_buildbot(
|
||||
g.db.session,
|
||||
branch=clobber['branch'],
|
||||
builddir=clobber['builddir'],
|
||||
slave=clobber['slave']
|
||||
)
|
||||
)
|
||||
)
|
||||
return summary
|
||||
g.db.session.commit()
|
||||
|
||||
except Exception as e:
|
||||
g.db.session.rollback()
|
||||
return dict(error=str(e.message))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def taskcluster_branches():
|
||||
"""Dict of workerTypes per branch with their respected workerTypes
|
||||
"""
|
||||
index = taskcluster.Index()
|
||||
queue = taskcluster.Queue()
|
||||
def get_taskcluster():
|
||||
caches_to_skip = current_app.config.get('TASKCLUSTER_CACHES_TO_SKIP', [])
|
||||
return models.taskcluster_branches(caches_to_skip)
|
||||
|
||||
result = index.listNamespaces('gecko.v2', dict(limit=1000))
|
||||
|
||||
branches = {
|
||||
i['name']: dict(name=i['name'], workerTypes=dict())
|
||||
for i in result.get('namespaces', [])
|
||||
}
|
||||
def post_taskcluster():
|
||||
# TODO: need to make this route work
|
||||
credentials = []
|
||||
|
||||
for branchName, branch in branches.items():
|
||||
# XXX: it should get authenticated via Authenticated header
|
||||
client_id = current_app.config.get('TASKCLUSTER_CLIENT_ID')
|
||||
access_token = current_app.config.get('TASKCLUSTER_ACCESS_TOKEN')
|
||||
|
||||
# decision task might not exist
|
||||
try:
|
||||
decision_task = index.findTask(
|
||||
TASKCLUSTER_DECISION_NAMESPACE % branchName)
|
||||
decision_graph = queue.getLatestArtifact(
|
||||
decision_task['taskId'], 'public/graph.json')
|
||||
except taskcluster.exceptions.TaskclusterRestFailure:
|
||||
continue
|
||||
if client_id and access_token:
|
||||
credentials = [dict(
|
||||
credentials=dict(
|
||||
clientId=client_id,
|
||||
accessToken=access_token,
|
||||
))]
|
||||
|
||||
for task in decision_graph.get('tasks', []):
|
||||
task = task['task']
|
||||
task_cache = task.get('payload', dict()).get('cache', dict())
|
||||
purge_cache = taskcluster.PurgeCache(*credentials)
|
||||
|
||||
provisionerId = task.get('provisionerId')
|
||||
if provisionerId:
|
||||
branch['provisionerId'] = provisionerId
|
||||
for item in body:
|
||||
purge_cache.purgeCache(item.provisionerId,
|
||||
item.workerType,
|
||||
dict(cacheName=item.cacheName))
|
||||
|
||||
workerType = task.get('workerType')
|
||||
if workerType:
|
||||
branch['workerTypes'].setdefault(
|
||||
workerType, dict(name=workerType, caches=[]))
|
||||
|
||||
if len(task_cache) > 0:
|
||||
branch['workerTypes'][workerType]['caches'] = list(set(
|
||||
branch['workerTypes'][workerType]['caches'] +
|
||||
task_cache.keys()
|
||||
))
|
||||
|
||||
return branches
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -4,34 +4,34 @@
|
|||
|
||||
from __future__ import absolute_import
|
||||
|
||||
import time
|
||||
|
||||
import sqlalchemy as sa
|
||||
import taskcluster as tc
|
||||
import time
|
||||
|
||||
from relengapi_common import db
|
||||
|
||||
DB_DECLARATIVE_BASE = 'clobberer'
|
||||
from relengapi_common.db import db
|
||||
|
||||
|
||||
class ClobbererBase(db.declarative_base(DB_DECLARATIVE_BASE)):
|
||||
__abstract__ = True
|
||||
|
||||
id = sa.Column(sa.Integer, primary_key=True)
|
||||
branch = sa.Column(sa.String(50), index=True)
|
||||
builddir = sa.Column(sa.String(100), index=True)
|
||||
BUILDBOT_BUILDDIR_REL_PREFIX = 'rel-'
|
||||
BUILDBOT_BUILDER_REL_PREFIX = 'release-'
|
||||
TASKCLUSTER_DECISION_NAMESPACE = 'gecko.v2.%s.latest.firefox.decision'
|
||||
|
||||
|
||||
class ClobbererBuilds(ClobbererBase, db.UniqueMixin):
|
||||
class Build(db.Model):
|
||||
"""
|
||||
A clobberable builds.
|
||||
"""
|
||||
|
||||
__tablename__ = 'clobberer_builds'
|
||||
|
||||
id = sa.Column(sa.Integer, primary_key=True)
|
||||
branch = sa.Column(sa.String(50), index=True)
|
||||
builddir = sa.Column(sa.String(100), index=True)
|
||||
buildername = sa.Column(sa.String(100))
|
||||
last_build_time = sa.Column(
|
||||
sa.Integer,
|
||||
nullable=False,
|
||||
default=int(time.time())
|
||||
default=lambda: int(time.time())
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
|
@ -48,17 +48,18 @@ class ClobbererBuilds(ClobbererBase, db.UniqueMixin):
|
|||
)
|
||||
|
||||
|
||||
class ClobbererTimes(ClobbererBase, db.UniqueMixin):
|
||||
"""
|
||||
A clobber request.
|
||||
"""
|
||||
class ClobberTime(db.Model):
|
||||
|
||||
__tablename__ = 'clobberer_times'
|
||||
__table_args__ = (
|
||||
# Index to speed up lastclobber lookups
|
||||
sa.Index('ix_get_clobberer_times', 'slave', 'builddir', 'branch'),
|
||||
)
|
||||
|
||||
id = sa.Column(sa.Integer, primary_key=True)
|
||||
branch = sa.Column(sa.String(50), index=True)
|
||||
slave = sa.Column(sa.String(30), index=True)
|
||||
builddir = sa.Column(sa.String(100), index=True)
|
||||
lastclobber = sa.Column(
|
||||
sa.Integer,
|
||||
nullable=False,
|
||||
|
|
@ -78,3 +79,180 @@ class ClobbererTimes(ClobbererBase, db.UniqueMixin):
|
|||
cls.slave == slave,
|
||||
cls.builddir == builddir,
|
||||
)
|
||||
|
||||
|
||||
def buildbot_branches(db_session):
|
||||
"""List of all buildbot branches.
|
||||
"""
|
||||
|
||||
branches = db_session.query(
|
||||
Build.branch
|
||||
).filter(
|
||||
# Users shouldn't see any branch associated with a release builddir
|
||||
sa.not_(
|
||||
Build.builddir.startswith(BUILDBOT_BUILDDIR_REL_PREFIX),
|
||||
)
|
||||
).order_by(
|
||||
Build.branch
|
||||
).distinct()
|
||||
|
||||
|
||||
return [
|
||||
dict(
|
||||
name=branch[0] or "",
|
||||
builders=[
|
||||
dict(
|
||||
name=builder[0] or "",
|
||||
branch=builder[1] or "",
|
||||
slave=builder[2] or "",
|
||||
builddir=builder[3] or "",
|
||||
lastclobber=builder[4] or -1,
|
||||
who=builder[5] or "",
|
||||
)
|
||||
for builder in buildbot_branch_builders(db_session, branch[0])
|
||||
if all([builder[1], builder[2], builder[3]])
|
||||
],
|
||||
)
|
||||
for branch in branches
|
||||
]
|
||||
|
||||
|
||||
def buildbot_branch_builders(db_session, branch):
|
||||
"""Return a dictionary of most recent ClobberTime grouped by
|
||||
buildername.
|
||||
"""
|
||||
# Isolates the maximum lastclobber for each builddir on a branch
|
||||
max_ct_sub_query = db_session.query(
|
||||
sa.func.max(ClobberTime.lastclobber).label('lastclobber'),
|
||||
ClobberTime.branch,
|
||||
ClobberTime.builddir,
|
||||
).group_by(
|
||||
ClobberTime.branch,
|
||||
ClobberTime.builddir,
|
||||
).filter(
|
||||
ClobberTime.branch == branch
|
||||
).subquery()
|
||||
|
||||
# Finds the "greatest n per group" by joining with the
|
||||
# max_ct_sub_query
|
||||
# This is necessary to get the correct "who" values
|
||||
sub_query = db_session.query(
|
||||
ClobberTime
|
||||
).join(
|
||||
max_ct_sub_query,
|
||||
sa.and_(
|
||||
ClobberTime.builddir == max_ct_sub_query.c.builddir,
|
||||
ClobberTime.lastclobber == max_ct_sub_query.c.lastclobber,
|
||||
ClobberTime.branch == max_ct_sub_query.c.branch,
|
||||
),
|
||||
).subquery()
|
||||
|
||||
# Attaches builddir, along with their max lastclobber to a
|
||||
# buildername
|
||||
return db_session.query(
|
||||
Build.buildername,
|
||||
sub_query.c.branch,
|
||||
sub_query.c.slave,
|
||||
sub_query.c.builddir,
|
||||
sub_query.c.lastclobber,
|
||||
sub_query.c.who
|
||||
).outerjoin(
|
||||
sub_query,
|
||||
Build.builddir == sub_query.c.builddir,
|
||||
).filter(
|
||||
Build.branch == branch,
|
||||
sa.not_(Build.buildername.startswith(BUILDBOT_BUILDER_REL_PREFIX))
|
||||
).order_by(
|
||||
Build.buildername
|
||||
).distinct()
|
||||
|
||||
|
||||
## TODO: this will change with tc authentication, it should be passed
|
||||
#try:
|
||||
# who = current_user.authenticated_email
|
||||
#except AttributeError:
|
||||
# if current_user.anonymous:
|
||||
# who = 'anonymous'
|
||||
# else:
|
||||
# # TokenUser doesn't show up as anonymous; but also has no
|
||||
# # authenticated_email
|
||||
# who = 'automation'
|
||||
|
||||
def buildbot_clobber(db_session, branch, slave, builddir, who, log=None):
|
||||
""" TODO:
|
||||
"""
|
||||
|
||||
builder = ClobberTime.unique_hash(branch, slave, builddir)
|
||||
|
||||
match = re.search('^' + BUILDBOT_BUILDDIR_REL_PREFIX + '.*', builddir)
|
||||
if match is None:
|
||||
|
||||
if log:
|
||||
log.debug('Clobbering builder: {}'.format(builder))
|
||||
|
||||
clobberer_time = ClobberTime.as_unique(
|
||||
db_session,
|
||||
branch=branch,
|
||||
slave=slave,
|
||||
builddir=builddir,
|
||||
)
|
||||
clobberer_time.lastclobber = int(time.time())
|
||||
clobberer_time.who = who
|
||||
|
||||
db_session.add(clobberer_time)
|
||||
db_session.commit()
|
||||
|
||||
if log:
|
||||
log.debug('Clobbered builder: {}'.format(builder))
|
||||
|
||||
return clobberer_time
|
||||
|
||||
|
||||
if log:
|
||||
log.debug('Skipping clobbering of builder: {}'.format(builder))
|
||||
|
||||
|
||||
def taskcluster_branches():
|
||||
"""Dict of workerTypes per branch with their respected workerTypes
|
||||
"""
|
||||
index = tc.Index()
|
||||
queue = tc.Queue()
|
||||
|
||||
result = index.listNamespaces('gecko.v2', dict(limit=1000))
|
||||
|
||||
branches = {
|
||||
i['name']: dict(name=i['name'], workerTypes=dict())
|
||||
for i in result.get('namespaces', [])
|
||||
}
|
||||
|
||||
for branchName, branch in branches.items():
|
||||
|
||||
# decision task might not exist
|
||||
try:
|
||||
decision_task = index.findTask(
|
||||
TASKCLUSTER_DECISION_NAMESPACE % branchName)
|
||||
decision_graph = queue.getLatestArtifact(
|
||||
decision_task['taskId'], 'public/graph.json')
|
||||
except tc.exceptions.TaskclusterRestFailure:
|
||||
continue
|
||||
|
||||
for task in decision_graph.get('tasks', []):
|
||||
task = task['task']
|
||||
task_cache = task.get('payload', dict()).get('cache', dict())
|
||||
|
||||
provisionerId = task.get('provisionerId')
|
||||
if provisionerId:
|
||||
branch['provisionerId'] = provisionerId
|
||||
|
||||
workerType = task.get('workerType')
|
||||
if workerType:
|
||||
branch['workerTypes'].setdefault(
|
||||
workerType, dict(name=workerType, caches=[]))
|
||||
|
||||
if len(task_cache) > 0:
|
||||
branch['workerTypes'][workerType]['caches'] = list(set(
|
||||
branch['workerTypes'][workerType]['caches'] +
|
||||
task_cache.keys()
|
||||
))
|
||||
|
||||
return branches
|
||||
|
|
|
|||
|
|
@ -19,9 +19,28 @@ paths:
|
|||
|
||||
post:
|
||||
operationId: "relengapi_clobberer.api.post_buildbot"
|
||||
parameters:
|
||||
- name: body
|
||||
in: body
|
||||
description: List of Builds to clobber.
|
||||
required: true
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/definitions/Builder'
|
||||
responses:
|
||||
200:
|
||||
description: Branches clobbered
|
||||
description: Builders clobbered
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/definitions/Success'
|
||||
500:
|
||||
description: Something went wrong when clobbering builders
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/definitions/Error'
|
||||
|
||||
|
||||
/taskcluster:
|
||||
|
|
@ -31,10 +50,6 @@ paths:
|
|||
responses:
|
||||
200:
|
||||
description: An array of branches
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/definitions/Branch'
|
||||
|
||||
post:
|
||||
operationId: "relengapi_clobberer.api.post_taskcluster"
|
||||
|
|
@ -44,48 +59,52 @@ paths:
|
|||
|
||||
definitions:
|
||||
|
||||
BuildTime:
|
||||
type: object
|
||||
description: Definition of a BuildTime
|
||||
required:
|
||||
- lastclobber
|
||||
properties:
|
||||
branch:
|
||||
type: string
|
||||
builddir:
|
||||
type: string
|
||||
lastclobber:
|
||||
type: integer
|
||||
who:
|
||||
type: string
|
||||
Success:
|
||||
type: string
|
||||
|
||||
Builder:
|
||||
Error:
|
||||
type: object
|
||||
description: Definition of a WorkerType
|
||||
required:
|
||||
- name
|
||||
- build_times
|
||||
- error_title
|
||||
properties:
|
||||
name:
|
||||
error_title:
|
||||
type: string
|
||||
error_message:
|
||||
type: string
|
||||
description: Name of WorkerType
|
||||
build_times:
|
||||
type: array
|
||||
description: Build times
|
||||
items:
|
||||
$ref: '#/definitions/BuildTime'
|
||||
|
||||
Branch:
|
||||
type: object
|
||||
description: Definition of a Branch
|
||||
required:
|
||||
- name
|
||||
- builders
|
||||
properties:
|
||||
name:
|
||||
type: string
|
||||
description: Name of Branch
|
||||
builders:
|
||||
type: array
|
||||
description: Builders per branch
|
||||
items:
|
||||
$ref: '#/definitions/Builder'
|
||||
|
||||
Builder:
|
||||
type: object
|
||||
required:
|
||||
- name
|
||||
- branch
|
||||
- builddir
|
||||
- slave
|
||||
- lastclobber
|
||||
- who
|
||||
properties:
|
||||
name:
|
||||
type: string
|
||||
branch:
|
||||
type: string
|
||||
builddir:
|
||||
type: string
|
||||
slave:
|
||||
type: string
|
||||
lastclobber:
|
||||
type: integer
|
||||
who:
|
||||
type: string
|
||||
|
|
|
|||
|
|
@ -39,15 +39,22 @@ def create_app(name, extensions=[], config=None, **kw):
|
|||
os.path.join(os.path.dirname(__file__), 'templates'))
|
||||
|
||||
for extension in [log, auth, api, cache] + extensions:
|
||||
extension_name = extension.__name__.split('.')[-1]
|
||||
setattr(app, extension_name, extension.init_app(app))
|
||||
if type(extension) is tuple:
|
||||
extension_name, extension_init = extension
|
||||
elif not hasattr(extension, 'init_app'):
|
||||
extension_name = None
|
||||
extension_init = extension
|
||||
else:
|
||||
extension_name = extension.__name__.split('.')[-1]
|
||||
extension_init = extension.init_app
|
||||
|
||||
_app = extension_init(app)
|
||||
if _app and extension_name is not None:
|
||||
setattr(app, extension_name, _app)
|
||||
|
||||
if hasattr(app, 'log'):
|
||||
app.log.debug('extension `%s` configured.' % extension_name)
|
||||
|
||||
# configure/initialize specific app features
|
||||
# aws -> tooltool, archiver (only s3 needed)
|
||||
# memcached -> treestatus (via https://pythonhosted.org/Flask-Cache)
|
||||
|
||||
return app
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,130 @@
|
|||
|
||||
from __future__ import absolute_import
|
||||
|
||||
from flask_restplus import Api
|
||||
import connexion
|
||||
import logging
|
||||
import pathlib
|
||||
import werkzeug.exceptions
|
||||
|
||||
|
||||
logger = logging.getLogger('relengapi_common.api')
|
||||
|
||||
|
||||
def common_error_handler(exception):
|
||||
"""
|
||||
TODO: add description
|
||||
|
||||
:param extension: TODO
|
||||
:type exception: Exception
|
||||
|
||||
:rtype: TODO:
|
||||
"""
|
||||
|
||||
if not isinstance(exception, werkzeug.exceptions.HTTPException):
|
||||
exception = werkzeug.exceptions.InternalServerError()
|
||||
|
||||
return connexion.problem(
|
||||
title=exception.name,
|
||||
detail=exception.description,
|
||||
status=exception.code,
|
||||
)
|
||||
|
||||
|
||||
class Api:
|
||||
"""
|
||||
TODO: add description
|
||||
TODO: annotate class
|
||||
"""
|
||||
|
||||
def __init__(self, app):
|
||||
"""
|
||||
TODO: add description
|
||||
TODO: annotate function
|
||||
"""
|
||||
self.__app = app
|
||||
|
||||
logger.debug('Setting JSON encoder.')
|
||||
app.json_encoder = connexion.decorators.produces.JSONEncoder
|
||||
|
||||
logger.debug('Setting common error handler for all error codes.')
|
||||
for error_code in werkzeug.exceptions.default_exceptions:
|
||||
app.register_error_handler(error_code, common_error_handler)
|
||||
|
||||
def register(self,
|
||||
swagger_file,
|
||||
base_url=None,
|
||||
arguments=None,
|
||||
auth_all_paths=False,
|
||||
swagger_json=True,
|
||||
swagger_ui=True,
|
||||
swagger_path=None,
|
||||
swagger_url="swagger",
|
||||
validate_responses=True,
|
||||
strict_validation=True,
|
||||
resolver=connexion.resolver.Resolver(),
|
||||
):
|
||||
"""
|
||||
Adds an API to the application based on a swagger file
|
||||
|
||||
:param swagger_file: swagger file with the specification
|
||||
:type swagger_file: str
|
||||
|
||||
:param base_url: base path where to add this api
|
||||
:type base_url: str | None
|
||||
|
||||
:param arguments: api version specific arguments to replace on the
|
||||
specification
|
||||
:type arguments: dict | None
|
||||
|
||||
:param auth_all_paths: whether to authenticate not defined paths
|
||||
:type auth_all_paths: bool
|
||||
|
||||
:param swagger_json: whether to include swagger json or not
|
||||
:type swagger_json: bool
|
||||
|
||||
:param swagger_ui: whether to include swagger ui or not
|
||||
:type swagger_ui: bool
|
||||
|
||||
:param swagger_path: path to swagger-ui directory
|
||||
:type swagger_path: string | None
|
||||
|
||||
:param swagger_url: URL to access swagger-ui documentation
|
||||
:type swagger_url: string | None
|
||||
|
||||
:param validate_responses: True enables validation. Validation errors
|
||||
generate HTTP 500 responses.
|
||||
:type validate_responses: bool
|
||||
|
||||
:param strict_validation: True enables validation on invalid request
|
||||
parameters
|
||||
:type strict_validation: bool
|
||||
|
||||
:param resolver: Operation resolver.
|
||||
:type resolver: connexion.resolver.Resolver | types.FunctionType
|
||||
|
||||
:rtype: None
|
||||
"""
|
||||
|
||||
if hasattr(resolver, '__call__'):
|
||||
resolver = connexion.resolver.Resolver(resolver)
|
||||
|
||||
logger.debug('Adding API: %s', swagger_file)
|
||||
|
||||
self.__api = connexion.api.Api(
|
||||
swagger_yaml_path=pathlib.Path(swagger_file),
|
||||
base_url=base_url,
|
||||
arguments=arguments,
|
||||
swagger_json=swagger_json,
|
||||
swagger_ui=swagger_ui,
|
||||
swagger_path=swagger_path,
|
||||
swagger_url=swagger_url,
|
||||
resolver=resolver,
|
||||
validate_responses=validate_responses,
|
||||
strict_validation=strict_validation,
|
||||
auth_all_paths=auth_all_paths,
|
||||
debug=self.__app.debug,
|
||||
)
|
||||
self.__app.register_blueprint(self.__api.blueprint)
|
||||
|
||||
|
||||
def init_app(app):
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
from flask_cache import Cache
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -8,22 +8,14 @@ from flask import g
|
|||
from flask_sqlalchemy import SQLAlchemy
|
||||
|
||||
|
||||
db = SQLAlchemy()
|
||||
|
||||
def init_app(app):
|
||||
db = SQLAlchemy()
|
||||
db.init_app(app)
|
||||
|
||||
# ensure tables get created
|
||||
# TODO: for dbname in db.database_names:
|
||||
# TODO: app.log.info("creating tables for database %s", dbname)
|
||||
|
||||
# TODO: meta = db.metadata[dbname]
|
||||
# TODO: engine = db.engine(dbname)
|
||||
# TODO: meta.create_all(bind=engine, checkfirst=True)
|
||||
db.create_all(app=app)
|
||||
|
||||
@app.before_request
|
||||
def setup_request():
|
||||
g.db = app.db
|
||||
|
||||
return db
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,6 @@ Flask-Cors
|
|||
Flask-Login
|
||||
Flask-SQLAlchemy
|
||||
Logbook
|
||||
flask-restplus
|
||||
connexion
|
||||
structlog
|
||||
taskcluster
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче