word (#192)

new HC ytdl solution

plans/youtube-dl-api-server/habitat/hooks/run

@@ -2,5 +2,34 @@
 
 exec 2>&1
 
-mkdir -p "{{ pkg.svc_config_path }}/env"
-envdir {{ pkg.svc_config_path }}/env gunicorn -w {{ cfg.general.num_workers }} -b {{ cfg.general.bind_ip }}:{{ cfg.general.port }} youtube_dl_server.app:app
+#mkdir -p "{{ pkg.svc_config_path }}/env"
+#envdir {{ pkg.svc_config_path }}/env gunicorn -w {{ cfg.general.num_workers }} -b {{ cfg.general.bind_ip }}:{{ cfg.general.port }} app:app
+cd {{ pkg.path }}
+ls -lha cd {{ pkg.path }}
+gunicorn -w {{ cfg.general.num_workers }} -b {{ cfg.general.bind_ip }}:{{ cfg.general.port }} app:app
+
+
+
+# #!/bin/bash
+
+# # exec 2>&1
+
+# # mkdir -p "{{ pkg.svc_config_path }}/env"
+# # envdir {{ pkg.svc_config_path }}/env gunicorn -w {{ cfg.general.num_workers }} -b {{ cfg.general.bind_ip }}:{{ cfg.general.port }} youtube_dl_server.app:app
+# # cd {{ pkg.path }}
+# pkgDir={{ pkg.path }}
+# echo "----------------------- ytdl ----------------------"
+# echo "pkgDir: $pkgDir"
+# ls -lha {{ pkg.path }}
+# echo "----------------------- ytdl ----------------------"
+
+# {{ pkg.path }}/gunicorn app:app -w 2 --threads 2 -b 0.0.0.0:8080
+
+
+
+# docker run -i --privileged -v $PWD/harts:/hab/cache/artifacts 
+# -v $PWD:/repo -e BLDR_HAB_TOKEN=${{ secrets.BLDR_HAB_TOKEN}} 
+# -e BLDR_RET_TOKEN=${{ secrets.BLDR_RET_TOKEN}} 
+# -e BLDR_RET_PUB_B64=${{ secrets.BLDR_RET_PUB_B64}} 
+# -e BLDR_HAB_PVT_B64=${{ secrets.BLDR_HAB_PVT_B64}} 
+# mozillareality/${{ github.workflow }}:${{ github.run_number }} bash /repo/${{ inputs.bio_script_path }} | tee stdouts

plans/youtube-dl-api-server/habitat/plan.sh

@@ -1,12 +1,12 @@
 pkg_name=youtube-dl-api-server
-pkg_version=0.4
+pkg_version=0.5
 pkg_origin=mozillareality
 pkg_maintainer="Mozilla Mixed Reality <mixreality@mozilla.com>"
 pkg_license=('unlicense')
 pkg_description="A youtube-dl REST API server"
 pkg_upstream_url="https://github.com/mozillareality/youtube-dl-api-server"
-pkg_source="https://github.com/mozillareality/youtube-dl-api-server/archive/${pkg_version}.tar.gz"
-pkg_shasum="6107513539ac18ef14377a0ecea55e54248e0113d7bef479da98a3dc19dad8d1"
+# pkg_source="https://github.com/mozillareality/youtube-dl-api-server/archive/${pkg_version}.tar.gz"
+# pkg_shasum="6107513539ac18ef14377a0ecea55e54248e0113d7bef479da98a3dc19dad8d1"
 pkg_deps=(
   core/envdir/1.0.1/20190305224317
   core/lzop/1.04/20190116190143
@@ -21,14 +21,20 @@ do_prepare() {
 }
 
 do_build() {
-  python setup.py sdist
+  # python setup.py sdist
+  echo "ffffffffffffffff"
 }
 
 do_install() {
-  pip install "dist/youtube_dl_server-${pkg_version}.tar.gz"
+  # pip install "dist/youtube_dl_server-${pkg_version}.tar.gz"
+  cp -R ./ytdl-api/*.py ${pkg_prefix}
+  cp -R ./ytdl-api-deps/* ${pkg_prefix}/
   pip install gunicorn
   rm -rf dist build
 
   # Write out versions of all pip packages to package
   pip freeze > "$pkg_prefix/requirements.txt"
+
+  ls -lha ${pkg_prefix}
+
 }

plans/youtube-dl-api-server/scripts/bio-build.sh

@@ -0,0 +1,55 @@
+set -e
+
+# build with docker:
+#   docker run -i --privileged -v $PWD:/repo mozillareality/hubs bash /repo/scripts/bio-build.sh
+#
+# any alpine image with bash will work
+
+
+# BLDR_RET_PUB_B64='U0lHLVBVQi0xC
+# BLDR_HAB_PVT_B64='U0lHLVNFQy0xC
+# BLDR_HAB_TOKEN='_Qk9YLTEKYmxkci
+# BLDR_RET_TOKEN='_Qk9YLTEKYmxkci
+
+apk add git curl py3-pip
+
+## preps
+org="biome-sh";repo="biome"
+ver=$(curl -s https://api.github.com/repos/$org/$repo/releases/latest | grep "tag_name" | awk '{print substr($2, 2, length($2)-3)}')
+dl="https://github.com/$org/$repo/releases/download/$ver/bio-${ver#"v"}-x86_64-linux.tar.gz"
+echo "[info] getting bio from: $dl" && curl -L -o bio.gz $dl && tar -xf bio.gz
+cp ./bio /usr/bin/bio && bio --version
+
+export HAB_ORIGIN=mozillareality
+
+mkdir -p /hab/cache/keys/
+mkdir -p ./hab/cache/keys/
+echo $BLDR_RET_PUB_B64 | base64 -d > /hab/cache/keys/mozillareality-20190117233449.pub
+echo $BLDR_RET_PUB_B64 | base64 -d > ./hab/cache/keys/mozillareality-20190117233449.pub
+echo $BLDR_HAB_PVT_B64 | base64 -d > /hab/cache/keys/mozillareality-20190117233449.sig.key
+echo $BLDR_HAB_PVT_B64 | base64 -d > /hab/cache/keys/mozillareality-20190117233449.sig.key
+
+
+echo "### build hab pkg"
+export HAB_AUTH_TOKEN=$BLDR_HAB_TOKEN
+
+
+mkdir /repo/ytdl-api-deps && cd /repo/ytdl-api-deps
+git clone https://github.com/ytdl-org/youtube-dl.git
+mv ./youtube-dl/youtube_dl/ ./youtube_dl/
+pip install flask -t ./repo/ytdl-api-deps/
+
+cd /repo
+ls -lha
+bio pkg build -k mozillareality .
+# exit 1
+
+### upload
+echo "### upload hab pkg"
+export HAB_BLDR_URL="https://bldr.reticulum.io"
+export HAB_AUTH_TOKEN=$BLDR_RET_TOKEN
+export HAB_ORIGIN_KEYS=mozillareality_ret
+echo $BLDR_RET_PUB_B64 | base64 -d > /hab/cache/keys/mozillareality-20190117233449.pub
+hart="/hab/cache/artifacts/$HAB_ORIGIN-youtube-dl-api-server*.hart"
+ls -lha $hart
+bio pkg upload $hart

plans/youtube-dl-api-server/ytdl-api/__init__.py

@@ -0,0 +1 @@
+from .version import __version__  # noqa: used externally

plans/youtube-dl-api-server/ytdl-api/__main__.py

@@ -0,0 +1,6 @@
+import sys
+
+from .server import main
+
+if __name__ == '__main__':
+    sys.exit(main())

plans/youtube-dl-api-server/ytdl-api/app.py

@@ -0,0 +1,196 @@
+import functools
+import logging
+import traceback
+import sys
+
+from flask import Flask, Blueprint, current_app, jsonify, request, redirect, abort
+import youtube_dl
+from youtube_dl.version import __version__ as youtube_dl_version
+from youtube_dl.utils import std_headers, random_user_agent
+
+# from .version import __version__
+
+if not hasattr(sys.stderr, 'isatty'):
+    # In GAE it's not defined and we must monkeypatch
+    sys.stderr.isatty = lambda: False
+
+
+class SimpleYDL(youtube_dl.YoutubeDL):
+    def __init__(self, *args, **kargs):
+        super(SimpleYDL, self).__init__(*args, **kargs)
+        self.add_default_info_extractors()
+
+
+def get_videos(url, extra_params):
+    '''
+    Get a list with a dict for every video founded
+    '''
+    ydl_params = {
+        'format': 'best',
+        'cachedir': False,
+        'logger': current_app.logger.getChild('youtube-dl'),
+        # 'proxy': current_app.config['proxy'],
+    }
+    ydl_params.update(extra_params)
+    ydl = SimpleYDL(ydl_params)
+    res = ydl.extract_info(url, download=False)
+    return res
+
+
+def flatten_result(result):
+    r_type = result.get('_type', 'video')
+    if r_type == 'video':
+        videos = [result]
+    elif r_type == 'playlist':
+        videos = []
+        for entry in result['entries']:
+            videos.extend(flatten_result(entry))
+    elif r_type == 'compat_list':
+        videos = []
+        for r in result['entries']:
+            videos.extend(flatten_result(r))
+    return videos
+
+
+api = Blueprint('api', __name__)
+
+
+def route_api(subpath, *args, **kargs):
+    return api.route('/api/' + subpath, *args, **kargs)
+
+
+def set_access_control(f):
+    @functools.wraps(f)
+    def wrapper(*args, **kargs):
+        response = f(*args, **kargs)
+        response.headers['Access-Control-Allow-Origin'] = '*'
+        return response
+    return wrapper
+
+
+@api.errorhandler(youtube_dl.utils.DownloadError)
+@api.errorhandler(youtube_dl.utils.ExtractorError)
+def handle_youtube_dl_error(error):
+    logging.error(traceback.format_exc())
+    result = jsonify({'error': str(error)})
+    result.status_code = 500
+    return result
+
+
+class WrongParameterTypeError(ValueError):
+    def __init__(self, value, type, parameter):
+        message = '"{}" expects a {}, got "{}"'.format(parameter, type, value)
+        super(WrongParameterTypeError, self).__init__(message)
+
+
+@api.errorhandler(WrongParameterTypeError)
+def handle_wrong_parameter(error):
+    logging.error(traceback.format_exc())
+    result = jsonify({'error': str(error)})
+    result.status_code = 400
+    return result
+
+
+@api.before_request
+def block_on_user_agent():
+    user_agent = request.user_agent.string
+    forbidden_uas = current_app.config.get('FORBIDDEN_USER_AGENTS', [])
+    if user_agent in forbidden_uas:
+        abort(429)
+
+
+def query_bool(value, name, default=None):
+    if value is None:
+        return default
+    value = value.lower()
+    if value == 'true':
+        return True
+    elif value == 'false':
+        return False
+    else:
+        raise WrongParameterTypeError(value, 'bool', name)
+
+
+ALLOWED_EXTRA_PARAMS = {
+    'format': str,
+    'playliststart': int,
+    'playlistend': int,
+    'playlist_items': str,
+    'playlistreverse': bool,
+    'matchtitle': str,
+    'rejecttitle': str,
+    'writesubtitles': bool,
+    'writeautomaticsub': bool,
+    'allsubtitles': bool,
+    'subtitlesformat': str,
+    'subtitleslangs': list,
+}
+
+
+def get_result():
+    url = request.args['url']
+    extra_params = {}
+
+    std_headers['User-Agent'] = random_user_agent()
+
+    for k, v in request.args.items():
+        if k == "user_agent":
+            std_headers['User-Agent'] = v
+        else:
+            if k in ALLOWED_EXTRA_PARAMS:
+                convertf = ALLOWED_EXTRA_PARAMS[k]
+                if convertf == bool:
+                    convertf = lambda x: query_bool(x, k)
+                elif convertf == list:
+                    convertf = lambda x: x.split(',')
+                extra_params[k] = convertf(v)
+
+    res = get_videos(url, extra_params)
+
+    return res
+
+
+@route_api('info')
+@set_access_control
+def info():
+    url = request.args['url']
+    result = get_result()
+    key = 'info'
+    if query_bool(request.args.get('flatten'), 'flatten', False):
+        result = flatten_result(result)
+        key = 'videos'
+    result = {
+        'url': url,
+        key: result,
+    }
+    return jsonify(result)
+
+
+@route_api('play')
+def play():
+    result = flatten_result(get_result())
+    return redirect(result[0]['url'])
+
+
+@route_api('extractors')
+@set_access_control
+def list_extractors():
+    ie_list = [{
+        'name': ie.IE_NAME,
+        'working': ie.working(),
+    } for ie in youtube_dl.gen_extractors()]
+    return jsonify(extractors=ie_list)
+
+
+@route_api('version')
+@set_access_control
+def version():
+    result = {
+        'youtube-dl': youtube_dl_version,
+        'youtube-dl-api-server': '0.4',
+    }
+    return jsonify(result)
+
+app = Flask(__name__)
+app.register_blueprint(api)
+app.config.from_pyfile('../application.cfg', silent=True)

plans/youtube-dl-api-server/ytdl-api/server.py

@@ -0,0 +1,56 @@
+import argparse
+
+from .app import app
+from .version import __version__
+
+"""
+    A server for providing the app anywhere, no need for GAE
+"""
+
+
+def main():
+    desc = """
+           The youtube-dl API server.
+           """
+
+    parser = argparse.ArgumentParser(description=desc)
+
+    parser.add_argument(
+        '-p', '--port',
+        default=9191,
+        type=int,
+        help='The port the server will use. The default is: %(default)s',
+    )
+
+    parser.add_argument(
+        '--host',
+        default='localhost',
+        type=str,
+        help='The host the server will use. The default is: %(default)s',
+    )
+
+    parser.add_argument(
+        '--number-processes',
+        default=5,
+        type=int,
+        help=('The number of processes the server will use. The default is: '
+              '%(default)s'),
+    )
+
+    parser.add_argument(
+        '--proxy',
+        default=None,
+        type=str,
+        help='Proxy server to use, if any. The default is None.',
+    )
+
+    parser.add_argument('--version', action='store_true',
+                        help='Print the version of the server')
+
+    args = parser.parse_args()
+    if args.version:
+        print(__version__)
+        exit(0)
+
+    app.config['proxy'] = args.proxy
+    app.run(args.host, args.port, processes=args.number_processes)

plans/youtube-dl-api-server/ytdl-api/version.py

@@ -0,0 +1 @@
+__version__ = '0.4'