From ba1961349c9d7ee57d97a86f11222bab678af9dc Mon Sep 17 00:00:00 2001 From: Ethan Dalool Date: Sun, 8 Oct 2017 21:39:07 -0700 Subject: [PATCH] It's been a while --- .../ycdl_flask/static}/common.css | 0 .../ycdl_flask/static}/common.js | 0 .../ycdl_flask/static}/favicon.png | Bin .../ycdl_flask/static}/favicon.svg | 0 .../ycdl_flask/templates}/channel.html | 16 ++--- .../ycdl_flask/templates}/channels.html | 0 .../ycdl_flask/templates}/header.html | 0 .../ycdl_flask/templates}/root.html | 0 frontends/ycdl_flask/ycdl_flask/__init__.py | 3 + .../ycdl_flask/ycdl_flask/ycdl_flask.py | 68 +++++++++++------- .../ycdl_flask/ycdl_flask_launch.py | 11 +-- ycdl_repl.py => frontends/ycdl_repl.py | 3 +- .../download_thumbnails.py | 0 .../refresh_channels.py | 0 utilities/ytqueue.py | 45 ++++++++++++ ycdl.py => ycdl/__init__.py | 11 ++- helpers.py => ycdl/helpers.py | 0 ytapi.py => ycdl/ytapi.py | 2 +- ytqueue.py | 24 ------- 19 files changed, 117 insertions(+), 66 deletions(-) rename {static => frontends/ycdl_flask/static}/common.css (100%) rename {static => frontends/ycdl_flask/static}/common.js (100%) rename {static => frontends/ycdl_flask/static}/favicon.png (100%) rename {static => frontends/ycdl_flask/static}/favicon.svg (100%) rename {templates => frontends/ycdl_flask/templates}/channel.html (93%) rename {templates => frontends/ycdl_flask/templates}/channels.html (100%) rename {templates => frontends/ycdl_flask/templates}/header.html (100%) rename {templates => frontends/ycdl_flask/templates}/root.html (100%) create mode 100644 frontends/ycdl_flask/ycdl_flask/__init__.py rename ycdl_site.py => frontends/ycdl_flask/ycdl_flask/ycdl_flask.py (80%) rename ycdl_launch.py => frontends/ycdl_flask/ycdl_flask_launch.py (67%) rename ycdl_repl.py => frontends/ycdl_repl.py (72%) rename download_thumbnails.py => utilities/download_thumbnails.py (100%) rename refresh_channels.py => utilities/refresh_channels.py (100%) create mode 100644 utilities/ytqueue.py rename ycdl.py => ycdl/__init__.py (96%) rename helpers.py => ycdl/helpers.py (100%) rename ytapi.py => ycdl/ytapi.py (99%) delete mode 100644 ytqueue.py diff --git a/static/common.css b/frontends/ycdl_flask/static/common.css similarity index 100% rename from static/common.css rename to frontends/ycdl_flask/static/common.css diff --git a/static/common.js b/frontends/ycdl_flask/static/common.js similarity index 100% rename from static/common.js rename to frontends/ycdl_flask/static/common.js diff --git a/static/favicon.png b/frontends/ycdl_flask/static/favicon.png similarity index 100% rename from static/favicon.png rename to frontends/ycdl_flask/static/favicon.png diff --git a/static/favicon.svg b/frontends/ycdl_flask/static/favicon.svg similarity index 100% rename from static/favicon.svg rename to frontends/ycdl_flask/static/favicon.svg diff --git a/templates/channel.html b/frontends/ycdl_flask/templates/channel.html similarity index 93% rename from templates/channel.html rename to frontends/ycdl_flask/templates/channel.html index 5292a4b..e53f299 100644 --- a/templates/channel.html +++ b/frontends/ycdl_flask/templates/channel.html @@ -77,15 +77,15 @@ onclick="refresh_channel('{{channel['id']}}', false, function(){location.reload()})">Refresh new videos - All - Pending - Ignored - Downloaded + All + Pending + Ignored + Downloaded {% else %} - All - Pending - Ignored - Downloaded + All + Pending + Ignored + Downloaded {% endif %} {{videos|length}} items diff --git a/templates/channels.html b/frontends/ycdl_flask/templates/channels.html similarity index 100% rename from templates/channels.html rename to frontends/ycdl_flask/templates/channels.html diff --git a/templates/header.html b/frontends/ycdl_flask/templates/header.html similarity index 100% rename from templates/header.html rename to frontends/ycdl_flask/templates/header.html diff --git a/templates/root.html b/frontends/ycdl_flask/templates/root.html similarity index 100% rename from templates/root.html rename to frontends/ycdl_flask/templates/root.html diff --git a/frontends/ycdl_flask/ycdl_flask/__init__.py b/frontends/ycdl_flask/ycdl_flask/__init__.py new file mode 100644 index 0000000..e7fe26c --- /dev/null +++ b/frontends/ycdl_flask/ycdl_flask/__init__.py @@ -0,0 +1,3 @@ +from . import ycdl_flask + +site = ycdl_flask.site diff --git a/ycdl_site.py b/frontends/ycdl_flask/ycdl_flask/ycdl_flask.py similarity index 80% rename from ycdl_site.py rename to frontends/ycdl_flask/ycdl_flask/ycdl_flask.py index 05bdf38..c345fcc 100644 --- a/ycdl_site.py +++ b/frontends/ycdl_flask/ycdl_flask/ycdl_flask.py @@ -2,22 +2,36 @@ Do not execute this file directly. Use ycdl_launch.py to start the server with gevent. ''' +import logging +logging.getLogger('googleapicliet.discovery_cache').setLevel(logging.ERROR) + import datetime import flask from flask import request import json import mimetypes import os +import traceback import bot -import helpers import ycdl -import ytapi -youtube_core = ytapi.Youtube(bot.YOUTUBE_KEY) +from voussoirkit import pathclass + +root_dir = pathclass.Path(__file__).parent.parent + +TEMPLATE_DIR = root_dir.with_child('templates') +STATIC_DIR = root_dir.with_child('static') +FAVICON_PATH = STATIC_DIR.with_child('favicon.png') + +youtube_core = ycdl.ytapi.Youtube(bot.YOUTUBE_KEY) youtube = ycdl.YCDL(youtube_core) -site = flask.Flask(__name__) +site = flask.Flask( + __name__, + template_folder=TEMPLATE_DIR.absolute_path, + static_folder=STATIC_DIR.absolute_path, +) site.config.update( SEND_FILE_MAX_AGE_DEFAULT=180, TEMPLATES_AUTO_RELOAD=True, @@ -91,7 +105,7 @@ def send_file(filepath): if request.method == 'HEAD': outgoing_data = bytes() else: - outgoing_data = helpers.read_filebytes(filepath, range_min=range_min, range_max=range_max) + outgoing_data = ycdl.helpers.read_filebytes(filepath, range_min=range_min, range_max=range_max) response = flask.Response( outgoing_data, @@ -112,8 +126,8 @@ def root(): @site.route('/favicon.ico') @site.route('/favicon.png') def favicon(): - filename = os.path.join('static', 'favicon.png') - return flask.send_file(filename) + return flask.send_file(FAVICON_PATH.absolute_path) + @site.route('/channels') def get_channels(): @@ -123,24 +137,30 @@ def get_channels(): return flask.render_template('channels.html', channels=channels) @site.route('/videos') +@site.route('/watch') @site.route('/videos/') @site.route('/channel/') @site.route('/channel//') def get_channel(channel_id=None, download_filter=None): if channel_id is not None: - youtube.add_channel(channel_id) + try: + youtube.add_channel(channel_id) + except Exception: + traceback.print_exc() channel = youtube.get_channel(channel_id) - if channel is None: - flask.abort(404) else: channel = None videos = youtube.get_videos(channel_id=channel_id, download_filter=download_filter) - search_term = request.args.get('q', None) - if search_term is not None: - search_term = search_term.lower() - videos = [v for v in videos if search_term in v['title'].lower()] + search_terms = request.args.get('q', '').lower().strip().replace('+', ' ').split() + if search_terms: + videos = [v for v in videos if all(term in v['title'].lower() for term in search_terms)] + + video_id = request.args.get('v', '') + if video_id: + youtube.insert_video(video_id) + videos = [youtube.get_video(video_id)] limit = request.args.get('limit', None) if limit is not None: @@ -155,14 +175,12 @@ def get_channel(channel_id=None, download_filter=None): published = datetime.datetime.utcfromtimestamp(published) published = published.strftime('%Y %m %d') video['_published_str'] = published - return flask.render_template('channel.html', channel=channel, videos=videos) - -@site.route('/static/') -def get_static(filename): - filename = filename.replace('\\', os.sep) - filename = filename.replace('/', os.sep) - filename = os.path.join('static', filename) - return flask.send_file(filename) + return flask.render_template( + 'channel.html', + channel=channel, + videos=videos, + query_string='?' + request.query_string.decode('utf-8'), + ) @site.route('/mark_video_state', methods=['POST']) def post_mark_video_state(): @@ -184,7 +202,7 @@ def post_mark_video_state(): @site.route('/refresh_all_channels', methods=['POST']) def post_refresh_all_channels(): force = request.form.get('force', False) - force = helpers.truthystring(force) + force = ycdl.helpers.truthystring(force) youtube.refresh_all_channels(force=force) return make_json_response({}) @@ -204,7 +222,7 @@ def post_refresh_channel(): flask.abort(404) force = request.form.get('force', False) - force = helpers.truthystring(force) + force = ycdl.helpers.truthystring(force) youtube.refresh_channel(channel_id, force=force) return make_json_response({}) @@ -215,7 +233,7 @@ def post_start_download(): video_id = request.form['video_id'] try: youtube.download_video(video_id) - except ytapi.VideoNotFound: + except ycdl.ytapi.VideoNotFound: flask.abort(404) return make_json_response({'video_id': video_id, 'state': 'downloaded'}) diff --git a/ycdl_launch.py b/frontends/ycdl_flask/ycdl_flask_launch.py similarity index 67% rename from ycdl_launch.py rename to frontends/ycdl_flask/ycdl_flask_launch.py index 1b21718..05fac91 100644 --- a/ycdl_launch.py +++ b/frontends/ycdl_flask/ycdl_flask_launch.py @@ -1,3 +1,6 @@ +import logging +logging.getLogger('googleapicliet.discovery_cache').setLevel(logging.ERROR) + import gevent.monkey gevent.monkey.patch_all() @@ -5,7 +8,7 @@ import gevent.pywsgi import gevent.wsgi import sys -import ycdl_site +import ycdl_flask if len(sys.argv) == 2: port = int(sys.argv[1]) @@ -15,14 +18,14 @@ else: if port == 443: http = gevent.pywsgi.WSGIServer( listener=('', port), - application=ycdl_site.site, + application=ycdl_flask.site, keyfile='https\\flasksite.key', certfile='https\\flasksite.crt', ) else: http = gevent.pywsgi.WSGIServer( - listener=('', port), - application=ycdl_site.site, + listener=('0.0.0.0', port), + application=ycdl_flask.site, ) diff --git a/ycdl_repl.py b/frontends/ycdl_repl.py similarity index 72% rename from ycdl_repl.py rename to frontends/ycdl_repl.py index 660359c..ff6e021 100644 --- a/ycdl_repl.py +++ b/frontends/ycdl_repl.py @@ -5,7 +5,6 @@ session with these variables preloaded. import bot import ycdl -import ytapi -youtube_core = ytapi.Youtube(bot.YOUTUBE_KEY) +youtube_core = ycdl.ytapi.Youtube(bot.YOUTUBE_KEY) youtube = ycdl.YCDL(youtube_core) diff --git a/download_thumbnails.py b/utilities/download_thumbnails.py similarity index 100% rename from download_thumbnails.py rename to utilities/download_thumbnails.py diff --git a/refresh_channels.py b/utilities/refresh_channels.py similarity index 100% rename from refresh_channels.py rename to utilities/refresh_channels.py diff --git a/utilities/ytqueue.py b/utilities/ytqueue.py new file mode 100644 index 0000000..7ae3e1b --- /dev/null +++ b/utilities/ytqueue.py @@ -0,0 +1,45 @@ +''' +I was having trouble making my Flask server perform the youtube-dl without +clogging up the other site activities. So instead I'll just have the server +export ytqueue files, which this script will download as a separate process. + +Rather than maintaining a text file or database of IDs to be downloaded, +I'm fine with creating each ID as a file and letting the filesystem act +as the to-do list. +''' + +import argparse +import os +import sys +import time + +YOUTUBE_DL = 'youtube-dlw https://www.youtube.com/watch?v={id}' + +def ytqueue(only_once=False): + while True: + print(time.strftime('%H:%M:%S'), 'Looking for files.') + queue = [f for f in os.listdir() if f.endswith('.ytqueue')] + for filename in queue: + yt_id = filename.split('.')[0] + command = YOUTUBE_DL.format(id=yt_id) + exit_code = os.system(command) + if exit_code == 0: + os.remove(filename) + if only_once: + break + time.sleep(10) + +def ytqueue_argparse(args): + return ytqueue(only_once=args.once) + +def main(argv): + parser = argparse.ArgumentParser() + + parser.add_argument('--once', dest='once', action='store_true') + parser.set_defaults(func=ytqueue_argparse) + + args = parser.parse_args(argv) + args.func(args) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/ycdl.py b/ycdl/__init__.py similarity index 96% rename from ycdl.py rename to ycdl/__init__.py index 4c52785..b4ac022 100644 --- a/ycdl.py +++ b/ycdl/__init__.py @@ -2,10 +2,11 @@ import logging import os import sqlite3 -import ytapi +from . import helpers +from . import ytapi def YOUTUBE_DL_COMMAND(video_id): - path = 'C:\\Incoming\\ytqueue\\{id}.ytqueue'.format(id=video_id) + path = 'D:\\Incoming\\ytqueue\\{id}.ytqueue'.format(id=video_id) open(path, 'w') logging.basicConfig(level=logging.DEBUG) @@ -207,6 +208,12 @@ class YCDL: channels.sort(key=lambda x: x['name'].lower()) return channels + def get_video(self, video_id): + self.cur.execute('SELECT * FROM videos WHERE id == ?', [video_id]) + video = self.cur.fetchone() + video = {key: video[SQL_VIDEO[key]] for key in SQL_VIDEO} + return video + def get_videos(self, channel_id=None, download_filter=None): wheres = [] bindings = [] diff --git a/helpers.py b/ycdl/helpers.py similarity index 100% rename from helpers.py rename to ycdl/helpers.py diff --git a/ytapi.py b/ycdl/ytapi.py similarity index 99% rename from ytapi.py rename to ycdl/ytapi.py index b205f5b..442d0f6 100644 --- a/ytapi.py +++ b/ycdl/ytapi.py @@ -1,7 +1,7 @@ import apiclient.discovery import datetime -import helpers +from . import helpers class VideoNotFound(Exception): pass diff --git a/ytqueue.py b/ytqueue.py deleted file mode 100644 index f4f0861..0000000 --- a/ytqueue.py +++ /dev/null @@ -1,24 +0,0 @@ -''' -I was having trouble making my Flask server perform the youtube-dl without -clogging up the other site activities. So instead I'll just have the server -export ytqueue files, which this script will download as a separate process. - -Rather than maintaining a text file or database of IDs to be downloaded, -I'm fine with creating each ID as a file and letting the filesystem act -as the to-do list. -''' -import os -import time - -YOUTUBE_DL = 'youtube-dlw https://www.youtube.com/watch?v={id}' - -while True: - print(time.strftime('%H:%M:%S'), 'Looking for files.') - queue = [f for f in os.listdir() if f.endswith('.ytqueue')] - for filename in queue: - yt_id = filename.split('.')[0] - command = YOUTUBE_DL.format(id=yt_id) - exit_code = os.system(command) - if exit_code == 0: - os.remove(filename) - time.sleep(10)