From 8bdf48d46cf393c35d6ef4ebff8acac11043ecec Mon Sep 17 00:00:00 2001 From: Geng Tan Date: Fri, 3 Mar 2023 00:29:37 -0500 Subject: [PATCH] word (#192) new HC ytdl solution --- plans/youtube-dl-api-server/habitat/hooks/run | 33 ++- plans/youtube-dl-api-server/habitat/plan.sh | 16 +- .../scripts/bio-build.sh | 55 +++++ .../ytdl-api/__init__.py | 1 + .../ytdl-api/__main__.py | 6 + plans/youtube-dl-api-server/ytdl-api/app.py | 196 ++++++++++++++++++ .../youtube-dl-api-server/ytdl-api/server.py | 56 +++++ .../youtube-dl-api-server/ytdl-api/version.py | 1 + 8 files changed, 357 insertions(+), 7 deletions(-) create mode 100755 plans/youtube-dl-api-server/scripts/bio-build.sh create mode 100644 plans/youtube-dl-api-server/ytdl-api/__init__.py create mode 100644 plans/youtube-dl-api-server/ytdl-api/__main__.py create mode 100644 plans/youtube-dl-api-server/ytdl-api/app.py create mode 100644 plans/youtube-dl-api-server/ytdl-api/server.py create mode 100644 plans/youtube-dl-api-server/ytdl-api/version.py diff --git a/plans/youtube-dl-api-server/habitat/hooks/run b/plans/youtube-dl-api-server/habitat/hooks/run index d82b1af..3bf508e 100644 --- a/plans/youtube-dl-api-server/habitat/hooks/run +++ b/plans/youtube-dl-api-server/habitat/hooks/run @@ -2,5 +2,34 @@ exec 2>&1 -mkdir -p "{{ pkg.svc_config_path }}/env" -envdir {{ pkg.svc_config_path }}/env gunicorn -w {{ cfg.general.num_workers }} -b {{ cfg.general.bind_ip }}:{{ cfg.general.port }} youtube_dl_server.app:app +#mkdir -p "{{ pkg.svc_config_path }}/env" +#envdir {{ pkg.svc_config_path }}/env gunicorn -w {{ cfg.general.num_workers }} -b {{ cfg.general.bind_ip }}:{{ cfg.general.port }} app:app +cd {{ pkg.path }} +ls -lha cd {{ pkg.path }} +gunicorn -w {{ cfg.general.num_workers }} -b {{ cfg.general.bind_ip }}:{{ cfg.general.port }} app:app + + + +# #!/bin/bash + +# # exec 2>&1 + +# # mkdir -p "{{ pkg.svc_config_path }}/env" +# # envdir {{ pkg.svc_config_path }}/env gunicorn -w {{ cfg.general.num_workers }} -b {{ cfg.general.bind_ip }}:{{ cfg.general.port }} youtube_dl_server.app:app +# # cd {{ pkg.path }} +# pkgDir={{ pkg.path }} +# echo "----------------------- ytdl ----------------------" +# echo "pkgDir: $pkgDir" +# ls -lha {{ pkg.path }} +# echo "----------------------- ytdl ----------------------" + +# {{ pkg.path }}/gunicorn app:app -w 2 --threads 2 -b 0.0.0.0:8080 + + + +# docker run -i --privileged -v $PWD/harts:/hab/cache/artifacts +# -v $PWD:/repo -e BLDR_HAB_TOKEN=${{ secrets.BLDR_HAB_TOKEN}} +# -e BLDR_RET_TOKEN=${{ secrets.BLDR_RET_TOKEN}} +# -e BLDR_RET_PUB_B64=${{ secrets.BLDR_RET_PUB_B64}} +# -e BLDR_HAB_PVT_B64=${{ secrets.BLDR_HAB_PVT_B64}} +# mozillareality/${{ github.workflow }}:${{ github.run_number }} bash /repo/${{ inputs.bio_script_path }} | tee stdouts \ No newline at end of file diff --git a/plans/youtube-dl-api-server/habitat/plan.sh b/plans/youtube-dl-api-server/habitat/plan.sh index 6d4f59d..14e35cd 100644 --- a/plans/youtube-dl-api-server/habitat/plan.sh +++ b/plans/youtube-dl-api-server/habitat/plan.sh @@ -1,12 +1,12 @@ pkg_name=youtube-dl-api-server -pkg_version=0.4 +pkg_version=0.5 pkg_origin=mozillareality pkg_maintainer="Mozilla Mixed Reality " pkg_license=('unlicense') pkg_description="A youtube-dl REST API server" pkg_upstream_url="https://github.com/mozillareality/youtube-dl-api-server" -pkg_source="https://github.com/mozillareality/youtube-dl-api-server/archive/${pkg_version}.tar.gz" -pkg_shasum="6107513539ac18ef14377a0ecea55e54248e0113d7bef479da98a3dc19dad8d1" +# pkg_source="https://github.com/mozillareality/youtube-dl-api-server/archive/${pkg_version}.tar.gz" +# pkg_shasum="6107513539ac18ef14377a0ecea55e54248e0113d7bef479da98a3dc19dad8d1" pkg_deps=( core/envdir/1.0.1/20190305224317 core/lzop/1.04/20190116190143 @@ -21,14 +21,20 @@ do_prepare() { } do_build() { - python setup.py sdist + # python setup.py sdist + echo "ffffffffffffffff" } do_install() { - pip install "dist/youtube_dl_server-${pkg_version}.tar.gz" + # pip install "dist/youtube_dl_server-${pkg_version}.tar.gz" + cp -R ./ytdl-api/*.py ${pkg_prefix} + cp -R ./ytdl-api-deps/* ${pkg_prefix}/ pip install gunicorn rm -rf dist build # Write out versions of all pip packages to package pip freeze > "$pkg_prefix/requirements.txt" + + ls -lha ${pkg_prefix} + } diff --git a/plans/youtube-dl-api-server/scripts/bio-build.sh b/plans/youtube-dl-api-server/scripts/bio-build.sh new file mode 100755 index 0000000..1c1128a --- /dev/null +++ b/plans/youtube-dl-api-server/scripts/bio-build.sh @@ -0,0 +1,55 @@ +set -e + +# build with docker: +# docker run -i --privileged -v $PWD:/repo mozillareality/hubs bash /repo/scripts/bio-build.sh +# +# any alpine image with bash will work + + +# BLDR_RET_PUB_B64='U0lHLVBVQi0xC +# BLDR_HAB_PVT_B64='U0lHLVNFQy0xC +# BLDR_HAB_TOKEN='_Qk9YLTEKYmxkci +# BLDR_RET_TOKEN='_Qk9YLTEKYmxkci + +apk add git curl py3-pip + +## preps +org="biome-sh";repo="biome" +ver=$(curl -s https://api.github.com/repos/$org/$repo/releases/latest | grep "tag_name" | awk '{print substr($2, 2, length($2)-3)}') +dl="https://github.com/$org/$repo/releases/download/$ver/bio-${ver#"v"}-x86_64-linux.tar.gz" +echo "[info] getting bio from: $dl" && curl -L -o bio.gz $dl && tar -xf bio.gz +cp ./bio /usr/bin/bio && bio --version + +export HAB_ORIGIN=mozillareality + +mkdir -p /hab/cache/keys/ +mkdir -p ./hab/cache/keys/ +echo $BLDR_RET_PUB_B64 | base64 -d > /hab/cache/keys/mozillareality-20190117233449.pub +echo $BLDR_RET_PUB_B64 | base64 -d > ./hab/cache/keys/mozillareality-20190117233449.pub +echo $BLDR_HAB_PVT_B64 | base64 -d > /hab/cache/keys/mozillareality-20190117233449.sig.key +echo $BLDR_HAB_PVT_B64 | base64 -d > /hab/cache/keys/mozillareality-20190117233449.sig.key + + +echo "### build hab pkg" +export HAB_AUTH_TOKEN=$BLDR_HAB_TOKEN + + +mkdir /repo/ytdl-api-deps && cd /repo/ytdl-api-deps +git clone https://github.com/ytdl-org/youtube-dl.git +mv ./youtube-dl/youtube_dl/ ./youtube_dl/ +pip install flask -t ./repo/ytdl-api-deps/ + +cd /repo +ls -lha +bio pkg build -k mozillareality . +# exit 1 + +### upload +echo "### upload hab pkg" +export HAB_BLDR_URL="https://bldr.reticulum.io" +export HAB_AUTH_TOKEN=$BLDR_RET_TOKEN +export HAB_ORIGIN_KEYS=mozillareality_ret +echo $BLDR_RET_PUB_B64 | base64 -d > /hab/cache/keys/mozillareality-20190117233449.pub +hart="/hab/cache/artifacts/$HAB_ORIGIN-youtube-dl-api-server*.hart" +ls -lha $hart +bio pkg upload $hart \ No newline at end of file diff --git a/plans/youtube-dl-api-server/ytdl-api/__init__.py b/plans/youtube-dl-api-server/ytdl-api/__init__.py new file mode 100644 index 0000000..a1473ae --- /dev/null +++ b/plans/youtube-dl-api-server/ytdl-api/__init__.py @@ -0,0 +1 @@ +from .version import __version__ # noqa: used externally diff --git a/plans/youtube-dl-api-server/ytdl-api/__main__.py b/plans/youtube-dl-api-server/ytdl-api/__main__.py new file mode 100644 index 0000000..c7e26bc --- /dev/null +++ b/plans/youtube-dl-api-server/ytdl-api/__main__.py @@ -0,0 +1,6 @@ +import sys + +from .server import main + +if __name__ == '__main__': + sys.exit(main()) diff --git a/plans/youtube-dl-api-server/ytdl-api/app.py b/plans/youtube-dl-api-server/ytdl-api/app.py new file mode 100644 index 0000000..7bc93d2 --- /dev/null +++ b/plans/youtube-dl-api-server/ytdl-api/app.py @@ -0,0 +1,196 @@ +import functools +import logging +import traceback +import sys + +from flask import Flask, Blueprint, current_app, jsonify, request, redirect, abort +import youtube_dl +from youtube_dl.version import __version__ as youtube_dl_version +from youtube_dl.utils import std_headers, random_user_agent + +# from .version import __version__ + +if not hasattr(sys.stderr, 'isatty'): + # In GAE it's not defined and we must monkeypatch + sys.stderr.isatty = lambda: False + + +class SimpleYDL(youtube_dl.YoutubeDL): + def __init__(self, *args, **kargs): + super(SimpleYDL, self).__init__(*args, **kargs) + self.add_default_info_extractors() + + +def get_videos(url, extra_params): + ''' + Get a list with a dict for every video founded + ''' + ydl_params = { + 'format': 'best', + 'cachedir': False, + 'logger': current_app.logger.getChild('youtube-dl'), + # 'proxy': current_app.config['proxy'], + } + ydl_params.update(extra_params) + ydl = SimpleYDL(ydl_params) + res = ydl.extract_info(url, download=False) + return res + + +def flatten_result(result): + r_type = result.get('_type', 'video') + if r_type == 'video': + videos = [result] + elif r_type == 'playlist': + videos = [] + for entry in result['entries']: + videos.extend(flatten_result(entry)) + elif r_type == 'compat_list': + videos = [] + for r in result['entries']: + videos.extend(flatten_result(r)) + return videos + + +api = Blueprint('api', __name__) + + +def route_api(subpath, *args, **kargs): + return api.route('/api/' + subpath, *args, **kargs) + + +def set_access_control(f): + @functools.wraps(f) + def wrapper(*args, **kargs): + response = f(*args, **kargs) + response.headers['Access-Control-Allow-Origin'] = '*' + return response + return wrapper + + +@api.errorhandler(youtube_dl.utils.DownloadError) +@api.errorhandler(youtube_dl.utils.ExtractorError) +def handle_youtube_dl_error(error): + logging.error(traceback.format_exc()) + result = jsonify({'error': str(error)}) + result.status_code = 500 + return result + + +class WrongParameterTypeError(ValueError): + def __init__(self, value, type, parameter): + message = '"{}" expects a {}, got "{}"'.format(parameter, type, value) + super(WrongParameterTypeError, self).__init__(message) + + +@api.errorhandler(WrongParameterTypeError) +def handle_wrong_parameter(error): + logging.error(traceback.format_exc()) + result = jsonify({'error': str(error)}) + result.status_code = 400 + return result + + +@api.before_request +def block_on_user_agent(): + user_agent = request.user_agent.string + forbidden_uas = current_app.config.get('FORBIDDEN_USER_AGENTS', []) + if user_agent in forbidden_uas: + abort(429) + + +def query_bool(value, name, default=None): + if value is None: + return default + value = value.lower() + if value == 'true': + return True + elif value == 'false': + return False + else: + raise WrongParameterTypeError(value, 'bool', name) + + +ALLOWED_EXTRA_PARAMS = { + 'format': str, + 'playliststart': int, + 'playlistend': int, + 'playlist_items': str, + 'playlistreverse': bool, + 'matchtitle': str, + 'rejecttitle': str, + 'writesubtitles': bool, + 'writeautomaticsub': bool, + 'allsubtitles': bool, + 'subtitlesformat': str, + 'subtitleslangs': list, +} + + +def get_result(): + url = request.args['url'] + extra_params = {} + + std_headers['User-Agent'] = random_user_agent() + + for k, v in request.args.items(): + if k == "user_agent": + std_headers['User-Agent'] = v + else: + if k in ALLOWED_EXTRA_PARAMS: + convertf = ALLOWED_EXTRA_PARAMS[k] + if convertf == bool: + convertf = lambda x: query_bool(x, k) + elif convertf == list: + convertf = lambda x: x.split(',') + extra_params[k] = convertf(v) + + res = get_videos(url, extra_params) + + return res + + +@route_api('info') +@set_access_control +def info(): + url = request.args['url'] + result = get_result() + key = 'info' + if query_bool(request.args.get('flatten'), 'flatten', False): + result = flatten_result(result) + key = 'videos' + result = { + 'url': url, + key: result, + } + return jsonify(result) + + +@route_api('play') +def play(): + result = flatten_result(get_result()) + return redirect(result[0]['url']) + + +@route_api('extractors') +@set_access_control +def list_extractors(): + ie_list = [{ + 'name': ie.IE_NAME, + 'working': ie.working(), + } for ie in youtube_dl.gen_extractors()] + return jsonify(extractors=ie_list) + + +@route_api('version') +@set_access_control +def version(): + result = { + 'youtube-dl': youtube_dl_version, + 'youtube-dl-api-server': '0.4', + } + return jsonify(result) + +app = Flask(__name__) +app.register_blueprint(api) +app.config.from_pyfile('../application.cfg', silent=True) diff --git a/plans/youtube-dl-api-server/ytdl-api/server.py b/plans/youtube-dl-api-server/ytdl-api/server.py new file mode 100644 index 0000000..81603bf --- /dev/null +++ b/plans/youtube-dl-api-server/ytdl-api/server.py @@ -0,0 +1,56 @@ +import argparse + +from .app import app +from .version import __version__ + +""" + A server for providing the app anywhere, no need for GAE +""" + + +def main(): + desc = """ + The youtube-dl API server. + """ + + parser = argparse.ArgumentParser(description=desc) + + parser.add_argument( + '-p', '--port', + default=9191, + type=int, + help='The port the server will use. The default is: %(default)s', + ) + + parser.add_argument( + '--host', + default='localhost', + type=str, + help='The host the server will use. The default is: %(default)s', + ) + + parser.add_argument( + '--number-processes', + default=5, + type=int, + help=('The number of processes the server will use. The default is: ' + '%(default)s'), + ) + + parser.add_argument( + '--proxy', + default=None, + type=str, + help='Proxy server to use, if any. The default is None.', + ) + + parser.add_argument('--version', action='store_true', + help='Print the version of the server') + + args = parser.parse_args() + if args.version: + print(__version__) + exit(0) + + app.config['proxy'] = args.proxy + app.run(args.host, args.port, processes=args.number_processes) diff --git a/plans/youtube-dl-api-server/ytdl-api/version.py b/plans/youtube-dl-api-server/ytdl-api/version.py new file mode 100644 index 0000000..58d168b --- /dev/null +++ b/plans/youtube-dl-api-server/ytdl-api/version.py @@ -0,0 +1 @@ +__version__ = '0.4'