inital shipit code

docs/shipit.rst

@@ -0,0 +1,35 @@
+ShipIt
+======
+
+
+current implementation
+----------------------
+
+- 2 parts webfrontend and releaserunner
+- webfrontend is used to
+  - accepts release data and stores it in the database
+  - start a release
+  - be the source of truth for other services
+  - does all the sanity checks before the release
+- releaserunner
+  - talks to webfrontend to get releases ready to go
+  - creates the tc graph (tasks are signed)
+  - starts a release process
+  - marks the release in shipit to "started"
+- readonly endpoints (used by other services)
+  - TODO: list them
+- there is a cronjob somewhere, generating https://product-details.mozilla.org/
+
+
+
+- multiple release products
+  - Firefox (7-8days)
+  - Firefox Beta (1-2day)
+  - Fennex
+  - Fennex Beta
+  - Thunderbird??
+    
+
+Technical stuff:
+ - relengapi stack
+ - relengapi repo

src/shipit/setup.py

@@ -0,0 +1,37 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+import os
+
+from setuptools import find_packages
+from setuptools import setup
+
+here = os.path.dirname(__file__)
+
+setup(
+    name='shipit',
+    version=open(os.path.join(here, 'VERSION')).read().strip(),
+    description='The code behind https://ship-it.mozilla-releng.net',
+    author='Mozilla RelEng',
+    author_email='release@mozilla.com',
+    url='https://ship-it.mozilla-releng.net',
+    install_requires=[
+        "relengapi_common",
+        "taskcluster",
+    ],
+    extras_require={
+    },
+    packages=find_packages(),
+    include_package_data=True,
+    zip_safe=False,
+    entry_points={
+        "console_scripts": [
+            'relengapi = relengapi.cmd:main',
+        ],
+    },
+    license='MPL2',
+)
+

src/shipit/shipit/__init__.py

@@ -0,0 +1,14 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+from relengapi_common import create_app, db
+from relengapi_clobberer import _flask
+
+app = create_app(__name__, [db, _flask])
+
+if __name__ == '__main__':
+    app.run(debug=True)
+

src/shipit/shipit/_flask.py

@@ -0,0 +1,16 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+
+class ShipIt:
+
+    def __init__(self, app):
+        self.app = app
+
+
+def init_app(app):
+    return ShipIt(app)
+

src/shipit/shipit/examples.py

@@ -0,0 +1,227 @@
+import json
+import datetime
+from functools import wraps
+
+
+# workflow.json
+# {
+#     "state1": {
+#         transitions: [
+#             "https://.../check-rev"
+#         ]
+#     }
+# }
+
+release = Process.create('schemas/workflow_firefox_beta.json')
+release.config
+# {
+#     "ticks": [
+#       {"state": "state1", "timestamp": 1234, "counter": 1}
+#     ],
+#     "states": [
+#        {
+#         "name": "state1",
+#         "transition": {
+#           "url": "http://...",
+#           "callback_url": "https://...."
+#           "config": {}
+#         },
+#         "status": "ready/error/running/success",
+#        },
+#        {
+#          "name": state2",
+#        }
+#     ]
+# }
+
+assert release == Process(release.config)
+
+
+class State:
+
+    def status(self):
+        # running
+        # success
+        # error
+
+input_data = State("input_data")
+
+
+release.pipeline = [
+    input_data
+    "check_rev"
+    ...
+]
+
+release.current_state = input_data
+release.tick(input)
+
+
+release.current_state = "check_rev"
+release.tick({}) # /release/<uid>/tick
+
+@transition(....)
+def input_firefox_release_data(input_):
+    signing_releng = Process.create(input_.something)
+    signing_qa = Process.create(input_.somethin2)
+
+    if signing_releng.done and signing_qa.done:
+        return Success(
+        )
+    return Error(
+        signing_releng.error
+        signing_qa.error
+    )
+
+
+
+
+
+class Success:
+    pass
+
+
+class Error:
+    pass
+
+
+class ValidationError(RuntimeError):
+    pass
+
+
+def validate(*args):
+    pass
+
+
+def check(*args):
+    pass
+
+
+def transition(input_schema, success_schema, error_schema):
+    def wrapper(func):
+        @wraps(func)
+        def inner(input_):
+            try:
+                validate(input_schema, input_)
+            except ValidationError as e:
+                pass
+            result = func(input_)
+            if isinstance(result, Success):
+                return validate(success_schema, result)
+            elif isinstance(result, Error):
+                return validate(error_schema, result)
+            else:
+                raise RuntimeError('ouch!')
+        return inner
+    return wrapper
+
+
+# /check-rev/<hash>
+# /check-rev/latest
+@app.route("/check-revs")
+@transition('schemas/input.json',
+            success='schemas/output.json',
+            error='schemas/error.json')
+def check_for_existing_revision(input_):
+    success = check(input_['rev'], input_['repo'])
+
+    if success
+        return Success(
+            rev=input_['rev'],
+            repo=input_['repo'],
+            exists=datetime.datetime.now(),
+        )
+
+    return Error(
+        input=input_,
+        message="Bakjshd ajkhd "
+    )
+
+# Expected output
+"""
+{
+    "product": "firefox",
+    "version": "48.0.1",
+    "build_number": 5,
+    "revision": "abcdeds",
+    "repo": "https://hg.mozilla.org/releases/mozilla-release",
+    "locales": {
+        "af": "abcdef",
+        "de": "axxfasd",
+    },
+    "gpg_pub_key": "data",
+}
+"""
+
+
+# after stage0
+"""
+{
+   stage0: {
+    "product": "firefox",
+    "repository_url": "https://hg.mozilla.org/releases/mozilla-release",
+    "timestamps": [
+        ( "start", "2016-07-29...", "a message" )
+    ]
+   }
+}
+"""
+
+# after stage1
+"""
+{
+   stage0: {
+     "product": "firefox",
+     "product_repository_url": "https://hg.mozilla.org/releases/mozilla-release",
+     "timestamps": [
+       ( "start", "2016-07-29...", "a message" ),
+       ( "end", "2016-07-29...", "a message" )
+     ]
+   },
+   stage1: {
+     "product_revision": "135457abd",
+     "locales": {
+       "af": "abcdef",
+       "de": "axxfasd",
+     }
+   }
+}
+"""
+
+# after stage2
+"""
+{
+   stage0: {
+     "product": "firefox",
+     "product_repository_url": "https://hg.mozilla.org/releases/mozilla-release",
+     "timestamps": [
+       ( "start", "2016-07-29...", "a message" ),
+       ( "end", "2016-07-29...", "a message" )
+     ]
+   },
+   stage1: {
+     "product_revision": "135457abd",
+     "locales": {
+       "af": "abcdef",
+       "de": "axxfasd",
+     }
+   },
+   stage2: {
+     "product_revision_status": 200,
+     "locales_status": {
+       "af": 200,
+       "de": 404,
+     }
+   },
+   # use TC Hooks? the task should call the corresponding URL when done with the status. Kill the hook after.
+   stage3: {
+     "en-US_bianry_status":
+        {"linux": 200,
+         "linux64": 404,
+         "win32": 404,
+         ...
+        }
+   }
+}
+"""
+