From 4f6080859adb5c37bbf72118a09e51911d096665 Mon Sep 17 00:00:00 2001 From: Ethan Dalool Date: Tue, 22 Sep 2020 02:50:24 -0700 Subject: [PATCH] Synchronize Etiquette and YCDL. --- frontends/ycdl_flask/backend/common.py | 87 +++---------------- frontends/ycdl_flask/backend/jinja_filters.py | 23 +++++ frontends/ycdl_flask/ycdl_flask_entrypoint.py | 13 +++ frontends/ycdl_flask/ycdl_flask_launch.py | 53 +++++++---- utilities/database_upgrader.py | 1 - utilities/download_thumbnails.py | 1 - ycdl/exceptions.py | 21 +++-- ycdl/ycdldb.py | 10 ++- ycdl/ytapi.py | 1 - 9 files changed, 111 insertions(+), 99 deletions(-) create mode 100644 frontends/ycdl_flask/ycdl_flask_entrypoint.py diff --git a/frontends/ycdl_flask/backend/common.py b/frontends/ycdl_flask/backend/common.py index c346c10..a9cc291 100644 --- a/frontends/ycdl_flask/backend/common.py +++ b/frontends/ycdl_flask/backend/common.py @@ -1,6 +1,6 @@ ''' Do not execute this file directly. -Use ycdl_launch.py to start the server with gevent. +Use ycdl_flask_launch.py to start the server with gevent. ''' import flask; from flask import request import gzip @@ -12,20 +12,18 @@ import time from voussoirkit import pathclass -import bot import ycdl from . import jinja_filters +# Flask init ####################################################################################### + root_dir = pathclass.Path(__file__).parent.parent TEMPLATE_DIR = root_dir.with_child('templates') STATIC_DIR = root_dir.with_child('static') FAVICON_PATH = STATIC_DIR.with_child('favicon.png') -youtube_core = ycdl.ytapi.Youtube(bot.get_youtube_key()) -ycdldb = ycdl.ycdldb.YCDLDB(youtube_core) - site = flask.Flask( __name__, template_folder=TEMPLATE_DIR.absolute_path, @@ -36,10 +34,15 @@ site.config.update( TEMPLATES_AUTO_RELOAD=True, ) site.jinja_env.add_extension('jinja2.ext.do') -site.jinja_env.filters['seconds_to_hms'] = jinja_filters.seconds_to_hms +site.jinja_env.trim_blocks = True +site.jinja_env.lstrip_blocks = True +jinja_filters.register_all(site) site.debug = True +#################################################################################################### + gzip_minimum_size = 500 +gzip_maximum_size = 5 * 2**20 gzip_level = 3 @site.after_request def after_request(response): @@ -53,6 +56,7 @@ def after_request(response): bail = bail or response.status_code < 200 bail = bail or response.status_code >= 300 bail = bail or response.direct_passthrough + bail = bail or int(response.headers.get('Content-Length', 0)) > gzip_maximum_size bail = bail or len(response.get_data()) < gzip_minimum_size bail = bail or 'gzip' not in accept_encoding.lower() bail = bail or 'Content-Encoding' in response.headers @@ -75,74 +79,9 @@ def after_request(response): #################################################################################################### #################################################################################################### -def send_file(filepath): - ''' - Range-enabled file sending. - ''' - try: - file_size = os.path.getsize(filepath) - except FileNotFoundError: - flask.abort(404) - - outgoing_headers = {} - mimetype = mimetypes.guess_type(filepath)[0] - if mimetype is not None: - if 'text/' in mimetype: - mimetype += '; charset=utf-8' - outgoing_headers['Content-Type'] = mimetype - - if 'range' in request.headers: - desired_range = request.headers['range'].lower() - desired_range = desired_range.split('bytes=')[-1] - - int_helper = lambda x: int(x) if x.isdigit() else None - if '-' in desired_range: - (desired_min, desired_max) = desired_range.split('-') - range_min = int_helper(desired_min) - range_max = int_helper(desired_max) - else: - range_min = int_helper(desired_range) - - if range_min is None: - range_min = 0 - if range_max is None: - range_max = file_size - - # because ranges are 0-indexed - range_max = min(range_max, file_size - 1) - range_min = max(range_min, 0) - - range_header = 'bytes {min}-{max}/{outof}'.format( - min=range_min, - max=range_max, - outof=file_size, - ) - outgoing_headers['Content-Range'] = range_header - status = 206 - else: - range_max = file_size - 1 - range_min = 0 - status = 200 - - outgoing_headers['Accept-Ranges'] = 'bytes' - outgoing_headers['Content-Length'] = (range_max - range_min) + 1 - - if request.method == 'HEAD': - outgoing_data = bytes() - else: - outgoing_data = ycdl.helpers.read_filebytes(filepath, range_min=range_min, range_max=range_max) - - response = flask.Response( - outgoing_data, - status=status, - headers=outgoing_headers, - ) - return response - -#################################################################################################### -#################################################################################################### -#################################################################################################### -#################################################################################################### +def init_ycdldb(*args, **kwargs): + global ycdldb + ycdldb = ycdl.ycdldb.YCDLDB(*args, **kwargs) def refresher_thread(rate): while True: diff --git a/frontends/ycdl_flask/backend/jinja_filters.py b/frontends/ycdl_flask/backend/jinja_filters.py index 04c9ca5..96c5dc8 100644 --- a/frontends/ycdl_flask/backend/jinja_filters.py +++ b/frontends/ycdl_flask/backend/jinja_filters.py @@ -1,5 +1,28 @@ import math +#################################################################################################### + +filter_functions = [] +global_functions = [] + +def filter_function(function): + filter_functions.append(function) + return function + +def global_function(function): + global_functions.append(function) + return function + +def register_all(site): + for function in filter_functions: + site.jinja_env.filters[function.__name__] = function + + for function in global_functions: + site.jinja_env.globals[function.__name__] = function + +#################################################################################################### + +@filter_function def seconds_to_hms(seconds): ''' Convert integer number of seconds to an hh:mm:ss string. diff --git a/frontends/ycdl_flask/ycdl_flask_entrypoint.py b/frontends/ycdl_flask/ycdl_flask_entrypoint.py new file mode 100644 index 0000000..4a634e0 --- /dev/null +++ b/frontends/ycdl_flask/ycdl_flask_entrypoint.py @@ -0,0 +1,13 @@ +''' +This file is the WSGI entrypoint for remote / production use. + +If you are using Gunicorn, for example: +gunicorn ycdl_flask_entrypoint:site --bind "0.0.0.0:PORT" --access-logfile "-" +''' +import werkzeug.middleware.proxy_fix + +import backend + +backend.site.wsgi_app = werkzeug.middleware.proxy_fix.ProxyFix(backend.site.wsgi_app) + +site = backend.site diff --git a/frontends/ycdl_flask/ycdl_flask_launch.py b/frontends/ycdl_flask/ycdl_flask_launch.py index e598975..499b55a 100644 --- a/frontends/ycdl_flask/ycdl_flask_launch.py +++ b/frontends/ycdl_flask/ycdl_flask_launch.py @@ -1,35 +1,54 @@ +import gevent.monkey; gevent.monkey.patch_all() + import logging logging.basicConfig() logging.getLogger('ycdl').setLevel(logging.DEBUG) -import gevent.monkey -gevent.monkey.patch_all() - import argparse import gevent.pywsgi import sys -import backend +from voussoirkit import pathclass -def ycdl_flask_launch(port, refresh_rate): - if port == 443: +import bot +import ycdl + +import ycdl_flask_entrypoint + +HTTPS_DIR = pathclass.Path(__file__).parent.with_child('https') + +def ycdl_flask_launch(create, port, refresh_rate, use_https): + if use_https is None: + use_https = port == 443 + + if use_https: http = gevent.pywsgi.WSGIServer( - listener=('', port), - application=backend.site, - keyfile='https\\flasksite.key', - certfile='https\\flasksite.crt', + listener=('0.0.0.0', port), + application=ycdl_flask_entrypoint.site, + keyfile=HTTPS_DIR.with_child('ycdl.key').absolute_path, + certfile=HTTPS_DIR.with_child('ycdl.crt').absolute_path, ) else: http = gevent.pywsgi.WSGIServer( listener=('0.0.0.0', port), - application=backend.site, + application=ycdl_flask_entrypoint.site, ) - if refresh_rate is not None: - backend.common.start_refresher_thread(refresh_rate) + youtube_core = ycdl.ytapi.Youtube(bot.get_youtube_key()) + ycdl_flask_entrypoint.backend.common.init_ycdldb(youtube_core, create=create) - print(f'Starting server on port {port}') - http.serve_forever() + if refresh_rate is not None: + ycdl_flask_entrypoint.backend.common.start_refresher_thread(refresh_rate) + + message = f'Starting server on port {port}' + if use_https: + message += ' (https)' + print(message) + + try: + http.serve_forever() + except KeyboardInterrupt: + pass def ycdl_flask_launch_argparse(args): if args.do_refresh: @@ -38,16 +57,20 @@ def ycdl_flask_launch_argparse(args): refresh_rate = None return ycdl_flask_launch( + create=args.create, port=args.port, refresh_rate=refresh_rate, + use_https=args.use_https, ) def main(argv): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('port', nargs='?', type=int, default=5000) + parser.add_argument('--dont_create', '--dont-create', '--no-create', dest='create', action='store_false', default=True) parser.add_argument('--no_refresh', '--no-refresh', dest='do_refresh', action='store_false', default=True) parser.add_argument('--refresh_rate', '--refresh-rate', dest='refresh_rate', type=int, default=60 * 60 * 6) + parser.add_argument('--https', dest='use_https', action='store_true', default=None) parser.set_defaults(func=ycdl_flask_launch_argparse) args = parser.parse_args(argv) diff --git a/utilities/database_upgrader.py b/utilities/database_upgrader.py index 4d6e788..6b79ae1 100644 --- a/utilities/database_upgrader.py +++ b/utilities/database_upgrader.py @@ -194,7 +194,6 @@ def upgrade_all(data_directory): current_version = version_number print('Upgrades finished.') - def upgrade_all_argparse(args): return upgrade_all(data_directory=args.data_directory) diff --git a/utilities/download_thumbnails.py b/utilities/download_thumbnails.py index af41746..5cecced 100644 --- a/utilities/download_thumbnails.py +++ b/utilities/download_thumbnails.py @@ -4,7 +4,6 @@ import traceback import ycdl from voussoirkit import downloady - youtube_core = ycdl.ytapi.Youtube(bot.get_youtube_key()) ycdldb = ycdl.ycdldb.YCDLDB(youtube_core) diff --git a/ycdl/exceptions.py b/ycdl/exceptions.py index b25dba9..5665a42 100644 --- a/ycdl/exceptions.py +++ b/ycdl/exceptions.py @@ -27,6 +27,7 @@ class ErrorTypeAdder(type): class YCDLException(Exception, metaclass=ErrorTypeAdder): ''' + Base type for all of the YCDL exceptions. Subtypes should have a class attribute `error_message`. The error message may contain {format} strings which will be formatted using the Exception's constructor arguments. @@ -43,27 +44,35 @@ class YCDLException(Exception, metaclass=ErrorTypeAdder): def __str__(self): return self.error_type + '\n' + self.error_message -class InvalidVideoState(YCDLException): - error_message = '{} is not a valid state.' +# NO SUCH ########################################################################################## - -# NO SUCH class NoSuchChannel(YCDLException): error_message = 'Channel {} does not exist.' class NoSuchVideo(YCDLException): error_message = 'Video {} does not exist.' +# VIDEO ERRORS ##################################################################################### + +class InvalidVideoState(YCDLException): + error_message = '{} is not a valid state.' + +# SQL ERRORS ####################################################################################### -# SQL ERRORS class BadSQL(YCDLException): pass class BadTable(BadSQL): error_message = 'Table "{}" does not exist.' +# GENERAL ERRORS ################################################################################### + +class BadDataDirectory(YCDLException): + ''' + Raised by YCDLDB __init__ if the requested data_directory is invalid. + ''' + error_message = 'Bad data directory "{}"' -# GENERAL ERRORS OUTOFDATE = ''' Database is out of date. {existing} should be {new}. Please run utilities\\database_upgrader.py "{filepath.absolute_path}" diff --git a/ycdl/ycdldb.py b/ycdl/ycdldb.py index 2869f30..906be4a 100644 --- a/ycdl/ycdldb.py +++ b/ycdl/ycdldb.py @@ -15,7 +15,6 @@ from voussoirkit import configlayers from voussoirkit import pathclass from voussoirkit import sqlhelpers - class YCDLDBCacheManagerMixin: _THING_CLASSES = { 'channel': @@ -466,6 +465,7 @@ class YCDLDB( def __init__( self, youtube, + create=True, data_directory=None, skip_version_check=False, ): @@ -478,12 +478,20 @@ class YCDLDB( self.data_directory = pathclass.Path(data_directory) + if self.data_directory.exists and not self.data_directory.is_dir: + raise exceptions.BadDataDirectory(self.data_directory.absolute_path) + # LOGGING self.log = logging.getLogger(__name__) # DATABASE self.database_filepath = self.data_directory.with_child(constants.DEFAULT_DBNAME) existing_database = self.database_filepath.exists + if not existing_database and not create: + msg = f'"{self.database_filepath.absolute_path}" does not exist and create is off.' + raise FileNotFoundError(msg) + + os.makedirs(self.data_directory.absolute_path, exist_ok=True) self.sql = sqlite3.connect(self.database_filepath.absolute_path) if existing_database: diff --git a/ycdl/ytapi.py b/ycdl/ytapi.py index ec449d0..a2eff53 100644 --- a/ycdl/ytapi.py +++ b/ycdl/ytapi.py @@ -4,7 +4,6 @@ import logging from . import helpers - def int_none(x): if x is None: return None