Synchronize Etiquette and YCDL.

frontends/ycdl_flask/backend/common.py

@@ -1,6 +1,6 @@
 '''
 Do not execute this file directly.
-Use ycdl_launch.py to start the server with gevent.
+Use ycdl_flask_launch.py to start the server with gevent.
 '''
 import flask; from flask import request
 import gzip
@@ -12,20 +12,18 @@ import time
 
 from voussoirkit import pathclass
 
-import bot
 import ycdl
 
 from . import jinja_filters
 
+# Flask init #######################################################################################
+
 root_dir = pathclass.Path(__file__).parent.parent
 
 TEMPLATE_DIR = root_dir.with_child('templates')
 STATIC_DIR = root_dir.with_child('static')
 FAVICON_PATH = STATIC_DIR.with_child('favicon.png')
 
-youtube_core = ycdl.ytapi.Youtube(bot.get_youtube_key())
-ycdldb = ycdl.ycdldb.YCDLDB(youtube_core)
-
 site = flask.Flask(
     __name__,
     template_folder=TEMPLATE_DIR.absolute_path,
@@ -36,10 +34,15 @@ site.config.update(
     TEMPLATES_AUTO_RELOAD=True,
 )
 site.jinja_env.add_extension('jinja2.ext.do')
-site.jinja_env.filters['seconds_to_hms'] = jinja_filters.seconds_to_hms
+site.jinja_env.trim_blocks = True
+site.jinja_env.lstrip_blocks = True
+jinja_filters.register_all(site)
 site.debug = True
 
+####################################################################################################
+
 gzip_minimum_size = 500
+gzip_maximum_size = 5 * 2**20
 gzip_level = 3
 @site.after_request
 def after_request(response):
@@ -53,6 +56,7 @@ def after_request(response):
     bail = bail or response.status_code < 200
     bail = bail or response.status_code >= 300
     bail = bail or response.direct_passthrough
+    bail = bail or int(response.headers.get('Content-Length', 0)) > gzip_maximum_size
     bail = bail or len(response.get_data()) < gzip_minimum_size
     bail = bail or 'gzip' not in accept_encoding.lower()
     bail = bail or 'Content-Encoding' in response.headers
@@ -75,74 +79,9 @@ def after_request(response):
 ####################################################################################################
 ####################################################################################################
 
-def send_file(filepath):
+def init_ycdldb(*args, **kwargs):
-    '''
+    global ycdldb
-    Range-enabled file sending.
+    ycdldb = ycdl.ycdldb.YCDLDB(*args, **kwargs)
-    '''
-    try:
-        file_size = os.path.getsize(filepath)
-    except FileNotFoundError:
-        flask.abort(404)
-
-    outgoing_headers = {}
-    mimetype = mimetypes.guess_type(filepath)[0]
-    if mimetype is not None:
-        if 'text/' in mimetype:
-            mimetype += '; charset=utf-8'
-        outgoing_headers['Content-Type'] = mimetype
-
-    if 'range' in request.headers:
-        desired_range = request.headers['range'].lower()
-        desired_range = desired_range.split('bytes=')[-1]
-
-        int_helper = lambda x: int(x) if x.isdigit() else None
-        if '-' in desired_range:
-            (desired_min, desired_max) = desired_range.split('-')
-            range_min = int_helper(desired_min)
-            range_max = int_helper(desired_max)
-        else:
-            range_min = int_helper(desired_range)
-
-        if range_min is None:
-            range_min = 0
-        if range_max is None:
-            range_max = file_size
-
-        # because ranges are 0-indexed
-        range_max = min(range_max, file_size - 1)
-        range_min = max(range_min, 0)
-
-        range_header = 'bytes {min}-{max}/{outof}'.format(
-            min=range_min,
-            max=range_max,
-            outof=file_size,
-        )
-        outgoing_headers['Content-Range'] = range_header
-        status = 206
-    else:
-        range_max = file_size - 1
-        range_min = 0
-        status = 200
-
-    outgoing_headers['Accept-Ranges'] = 'bytes'
-    outgoing_headers['Content-Length'] = (range_max - range_min) + 1
-
-    if request.method == 'HEAD':
-        outgoing_data = bytes()
-    else:
-        outgoing_data = ycdl.helpers.read_filebytes(filepath, range_min=range_min, range_max=range_max)
-
-    response = flask.Response(
-        outgoing_data,
-        status=status,
-        headers=outgoing_headers,
-    )
-    return response
-
-####################################################################################################
-####################################################################################################
-####################################################################################################
-####################################################################################################
 
 def refresher_thread(rate):
     while True:

frontends/ycdl_flask/backend/jinja_filters.py

@@ -1,5 +1,28 @@
 import math
 
+####################################################################################################
+
+filter_functions = []
+global_functions = []
+
+def filter_function(function):
+    filter_functions.append(function)
+    return function
+
+def global_function(function):
+    global_functions.append(function)
+    return function
+
+def register_all(site):
+    for function in filter_functions:
+        site.jinja_env.filters[function.__name__] = function
+
+    for function in global_functions:
+        site.jinja_env.globals[function.__name__] = function
+
+####################################################################################################
+
+@filter_function
 def seconds_to_hms(seconds):
     '''
     Convert integer number of seconds to an hh:mm:ss string.

frontends/ycdl_flask/ycdl_flask_entrypoint.py

@@ -0,0 +1,13 @@
+'''
+This file is the WSGI entrypoint for remote / production use.
+
+If you are using Gunicorn, for example:
+gunicorn ycdl_flask_entrypoint:site --bind "0.0.0.0:PORT" --access-logfile "-"
+'''
+import werkzeug.middleware.proxy_fix
+
+import backend
+
+backend.site.wsgi_app = werkzeug.middleware.proxy_fix.ProxyFix(backend.site.wsgi_app)
+
+site = backend.site

frontends/ycdl_flask/ycdl_flask_launch.py

@@ -1,35 +1,54 @@
+import gevent.monkey; gevent.monkey.patch_all()
+
 import logging
 logging.basicConfig()
 logging.getLogger('ycdl').setLevel(logging.DEBUG)
 
-import gevent.monkey
-gevent.monkey.patch_all()
-
 import argparse
 import gevent.pywsgi
 import sys
 
-import backend
+from voussoirkit import pathclass
 
-def ycdl_flask_launch(port, refresh_rate):
+import bot
-    if port == 443:
+import ycdl
+
+import ycdl_flask_entrypoint
+
+HTTPS_DIR = pathclass.Path(__file__).parent.with_child('https')
+
+def ycdl_flask_launch(create, port, refresh_rate, use_https):
+    if use_https is None:
+        use_https = port == 443
+
+    if use_https:
         http = gevent.pywsgi.WSGIServer(
-            listener=('', port),
+            listener=('0.0.0.0', port),
-            application=backend.site,
+            application=ycdl_flask_entrypoint.site,
-            keyfile='https\\flasksite.key',
+            keyfile=HTTPS_DIR.with_child('ycdl.key').absolute_path,
-            certfile='https\\flasksite.crt',
+            certfile=HTTPS_DIR.with_child('ycdl.crt').absolute_path,
         )
     else:
         http = gevent.pywsgi.WSGIServer(
             listener=('0.0.0.0', port),
-            application=backend.site,
+            application=ycdl_flask_entrypoint.site,
         )
 
-    if refresh_rate is not None:
+    youtube_core = ycdl.ytapi.Youtube(bot.get_youtube_key())
-        backend.common.start_refresher_thread(refresh_rate)
+    ycdl_flask_entrypoint.backend.common.init_ycdldb(youtube_core, create=create)
 
-    print(f'Starting server on port {port}')
+    if refresh_rate is not None:
+        ycdl_flask_entrypoint.backend.common.start_refresher_thread(refresh_rate)
+
+    message = f'Starting server on port {port}'
+    if use_https:
+        message += ' (https)'
+    print(message)
+
+    try:
         http.serve_forever()
+    except KeyboardInterrupt:
+        pass
 
 def ycdl_flask_launch_argparse(args):
     if args.do_refresh:
@@ -38,16 +57,20 @@ def ycdl_flask_launch_argparse(args):
         refresh_rate = None
 
     return ycdl_flask_launch(
+        create=args.create,
         port=args.port,
         refresh_rate=refresh_rate,
+        use_https=args.use_https,
     )
 
 def main(argv):
     parser = argparse.ArgumentParser(description=__doc__)
 
     parser.add_argument('port', nargs='?', type=int, default=5000)
+    parser.add_argument('--dont_create', '--dont-create', '--no-create', dest='create', action='store_false', default=True)
     parser.add_argument('--no_refresh', '--no-refresh', dest='do_refresh', action='store_false', default=True)
     parser.add_argument('--refresh_rate', '--refresh-rate', dest='refresh_rate', type=int, default=60 * 60 * 6)
+    parser.add_argument('--https', dest='use_https', action='store_true', default=None)
     parser.set_defaults(func=ycdl_flask_launch_argparse)
 
     args = parser.parse_args(argv)

utilities/database_upgrader.py

@@ -194,7 +194,6 @@ def upgrade_all(data_directory):
         current_version = version_number
     print('Upgrades finished.')
 
-
 def upgrade_all_argparse(args):
     return upgrade_all(data_directory=args.data_directory)
 

utilities/download_thumbnails.py

@@ -4,7 +4,6 @@ import traceback
 import ycdl
 from voussoirkit import downloady
 
-
 youtube_core = ycdl.ytapi.Youtube(bot.get_youtube_key())
 ycdldb = ycdl.ycdldb.YCDLDB(youtube_core)
 

ycdl/exceptions.py

@@ -27,6 +27,7 @@ class ErrorTypeAdder(type):
 
 class YCDLException(Exception, metaclass=ErrorTypeAdder):
     '''
+    Base type for all of the YCDL exceptions.
     Subtypes should have a class attribute `error_message`. The error message
     may contain {format} strings which will be formatted using the
     Exception's constructor arguments.
@@ -43,27 +44,35 @@ class YCDLException(Exception, metaclass=ErrorTypeAdder):
     def __str__(self):
         return self.error_type + '\n' + self.error_message
 
-class InvalidVideoState(YCDLException):
+# NO SUCH ##########################################################################################
-    error_message = '{} is not a valid state.'
 
-
-# NO SUCH
 class NoSuchChannel(YCDLException):
     error_message = 'Channel {} does not exist.'
 
 class NoSuchVideo(YCDLException):
     error_message = 'Video {} does not exist.'
 
+# VIDEO ERRORS #####################################################################################
+
+class InvalidVideoState(YCDLException):
+    error_message = '{} is not a valid state.'
+
+# SQL ERRORS #######################################################################################
 
-# SQL ERRORS
 class BadSQL(YCDLException):
     pass
 
 class BadTable(BadSQL):
     error_message = 'Table "{}" does not exist.'
 
+# GENERAL ERRORS ###################################################################################
+
+class BadDataDirectory(YCDLException):
+    '''
+    Raised by YCDLDB __init__ if the requested data_directory is invalid.
+    '''
+    error_message = 'Bad data directory "{}"'
 
-# GENERAL ERRORS
 OUTOFDATE = '''
 Database is out of date. {existing} should be {new}.
 Please run utilities\\database_upgrader.py "{filepath.absolute_path}"

ycdl/ycdldb.py

@@ -15,7 +15,6 @@ from voussoirkit import configlayers
 from voussoirkit import pathclass
 from voussoirkit import sqlhelpers
 
-
 class YCDLDBCacheManagerMixin:
     _THING_CLASSES = {
         'channel':
@@ -466,6 +465,7 @@ class YCDLDB(
     def __init__(
             self,
             youtube,
+            create=True,
             data_directory=None,
             skip_version_check=False,
         ):
@@ -478,12 +478,20 @@ class YCDLDB(
 
         self.data_directory = pathclass.Path(data_directory)
 
+        if self.data_directory.exists and not self.data_directory.is_dir:
+            raise exceptions.BadDataDirectory(self.data_directory.absolute_path)
+
         # LOGGING
         self.log = logging.getLogger(__name__)
 
         # DATABASE
         self.database_filepath = self.data_directory.with_child(constants.DEFAULT_DBNAME)
         existing_database = self.database_filepath.exists
+        if not existing_database and not create:
+            msg = f'"{self.database_filepath.absolute_path}" does not exist and create is off.'
+            raise FileNotFoundError(msg)
+
+        os.makedirs(self.data_directory.absolute_path, exist_ok=True)
         self.sql = sqlite3.connect(self.database_filepath.absolute_path)
 
         if existing_database:

ycdl/ytapi.py

@@ -4,7 +4,6 @@ import logging
 
 from . import helpers
 
-
 def int_none(x):
     if x is None:
         return None