diff --git a/frontends/ycdl_cli.py b/frontends/ycdl_cli.py new file mode 100644 index 0000000..45e6e71 --- /dev/null +++ b/frontends/ycdl_cli.py @@ -0,0 +1,242 @@ +import argparse +import sys +import traceback + +from voussoirkit import betterhelp +from voussoirkit import interactive +from voussoirkit import pipeable +from voussoirkit import vlogging +from voussoirkit import operatornotify + +import ycdl + +log = vlogging.getLogger(__name__, 'ycdl_cli') + +# HELPERS ########################################################################################## + +def closest_db(): + return ycdl.ycdldb.YCDLDB.closest_ycdldb() + +# ARGPARSE ######################################################################################### + +def add_channel_argparse(args): + ycdldb = closest_db() + ycdldb.add_channel( + channel_id=args.channel_id, + automark=args.automark, + download_directory=args.download_directory, + get_videos=args.get_videos, + name=args.name, + queuefile_extension=args.queuefile_extension, + ) + + if args.autoyes or interactive.getpermission('Commit?'): + ycdldb.commit() + + return 0 + +def channel_list_argparse(args): + ycdldb = closest_db() + channels = sorted(ycdldb.get_channels(), key=lambda c: c.name.lower()) + for channel in channels: + line = args.format.format( + automark=channel.automark, + autorefresh=channel.autorefresh, + id=channel.id, + name=channel.name, + queuefile_extension=channel.queuefile_extension, + uploads_playlist=channel.uploads_playlist, + ) + pipeable.stdout(line) + + return 0 + +def delete_channel_argparse(args): + ycdldb = closest_db() + for channel_id in args.channel_id: + channel = ycdldb.get_channel(channel_id) + channel.delete() + + if args.autoyes or interactive.getpermission('Commit?'): + ycdldb.commit() + + return 0 + +def init_argparse(args): + ycdldb = ycdl.ycdldb.YCDLDB(create=True) + ycdldb.commit() + pipeable.stdout(ycdldb.data_directory.absolute_path) + return 0 + +def refresh_channels_argparse(args): + needs_commit = False + status = 0 + + ycdldb = closest_db() + if args.channels: + channels = [ycdldb.get_channel(c) for c in args.channels] + for channel in channels: + try: + channel.refresh(force=args.force) + needs_commit = True + except Exception as exc: + log.warning(traceback.format_exc()) + status = 1 + else: + excs = ycdldb.refresh_all_channels(force=args.force, skip_failures=True) + needs_commit = True + + if not needs_commit: + return status + + if args.autoyes or interactive.getpermission('Commit?'): + ycdldb.commit() + + return status + +DOCSTRING = ''' +YCDL CLI +======== + +{add_channel} + +{channel_list} + +{delete_channel} + +{init} + +{refresh_channels} + +TO SEE DETAILS ON EACH COMMAND, RUN +> ycdl_cli.py --help +''' + +SUB_DOCSTRINGS = dict( +add_channel=''' +add_channel: + Add a channel to the database. + + > ycdl_cli.py add_channel channel_id + + flags: + --automark X: + Set the channel's automark to this value, which should be 'pending', + 'downloaded', or 'ignored'. + + --download_directory X: + Set the channel's download directory to this path, which must + be a directory. + + --name X: + Override the channel's own name with a name of your choosing. + + --no_videos: + By default, the channel's videos will be fetched right away. + Add this argument if you don't want to do that yet. + + --queuefile_extension X: + Set the queuefile extension for all videos downloaded from this channel. + + Examples: + > ycdl_cli.py add_channel UCFhXFikryT4aFcLkLw2LBLA +'''.strip(), + +channel_list=''' +channel_list: + Print all channels in the database. + + > ycdl_cli.py channel_list + + --format X: + A string like "{id}: {name}" to format the attributes of the channel. + The available attributes are id, name, automark, autorefresh, + uploads_playlist queuefile_extension. + + > ycdl_cli.py list_channels +'''.strip(), + +delete_channel=''' +delete_channel: + Delete a channel and all its videos from the database. + + You can pass multiple channel IDs. + + > ycdl_cli.py delete_channel channel_id [channel_id channel_id] + + Examples: + > ycdl_cli.py delete_channel UCOYBuFGi8T3NM5fNAptCLCw + > ycdl_cli.py delete_channel UCOYBuFGi8T3NM5fNAptCLCw UCmu9PVIZBk-ZCi-Sk2F2utA UCEKJKJ3FO-9SFv5x5BzyxhQ +'''.strip(), + +init=''' +init: + Create a new YCDL database in the current directory. + + > ycdl_cli.py init +'''.strip(), + +refresh_channels=''' +refresh_channels: + Refresh some or all channels in the database. + + > ycdl_cli.py refresh_channels + + flags: + --channels X Y Z: + Any number of channel IDs. + + --force: + If omitted, only new videos are downloaded. + If included, channels are refreshed completely. + Examples: + > ycdl_cli.py refresh_channels --force + > ycdl_cli.py refresh_channels --channels UC1_uAIS3r8Vu6JjXWvastJg +''' +) + +DOCSTRING = betterhelp.add_previews(DOCSTRING, SUB_DOCSTRINGS) + +@operatornotify.main_decorator(subject='ycdl_cli') +@vlogging.main_decorator +def main(argv): + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers() + + p_add_channel = subparsers.add_parser('add_channel', aliases=['add-channel']) + p_add_channel.add_argument('channel_id') + p_add_channel.add_argument('--automark', default='pending') + p_add_channel.add_argument('--download_directory', '--download-directory', default=None) + p_add_channel.add_argument('--name', default=None) + p_add_channel.add_argument('--no_videos', '--no-videos', dest='get_videos', action='store_false') + p_add_channel.add_argument('--queuefile_extension', '--queuefile-extension', default=None) + p_add_channel.add_argument('--yes', dest='autoyes', action='store_true') + p_add_channel.set_defaults(func=add_channel_argparse) + + p_channel_list = subparsers.add_parser('channel_list', aliases=['channel-list']) + p_channel_list.add_argument('--format', default='{id}:{name}') + p_channel_list.set_defaults(func=channel_list_argparse) + + p_delete_channel = subparsers.add_parser('delete_channel', aliases=['delete-channel']) + p_delete_channel.add_argument('channel_id', nargs='+') + p_delete_channel.add_argument('--yes', dest='autoyes', action='store_true') + p_delete_channel.set_defaults(func=delete_channel_argparse) + + p_init = subparsers.add_parser('init') + p_init.set_defaults(func=init_argparse) + + p_refresh_channels = subparsers.add_parser('refresh_channels', aliases=['refresh-channels']) + p_refresh_channels.add_argument('--channels', nargs='*') + p_refresh_channels.add_argument('--force', action='store_true') + p_refresh_channels.add_argument('--yes', dest='autoyes', action='store_true') + p_refresh_channels.set_defaults(func=refresh_channels_argparse) + + try: + return betterhelp.subparser_main(argv, parser, DOCSTRING, SUB_DOCSTRINGS) + except ycdl.exceptions.NoClosestYCDLDB as exc: + pipeable.stderr(exc.error_message) + pipeable.stderr('Try `ycdl_cli.py init` to create the database.') + return 1 + +if __name__ == '__main__': + raise SystemExit(main(sys.argv[1:])) diff --git a/frontends/ycdl_flask/backend/common.py b/frontends/ycdl_flask/backend/common.py index 9fe9d0d..2a57a05 100644 --- a/frontends/ycdl_flask/backend/common.py +++ b/frontends/ycdl_flask/backend/common.py @@ -59,7 +59,7 @@ def after_request(response): def init_ycdldb(*args, **kwargs): global ycdldb - ycdldb = ycdl.ycdldb.YCDLDB(*args, **kwargs) + ycdldb = ycdl.ycdldb.YCDLDB.closest_ycdldb(*args, **kwargs) def refresher_thread(rate): while True: diff --git a/frontends/ycdl_flask/backend/endpoints/channel_endpoints.py b/frontends/ycdl_flask/backend/endpoints/channel_endpoints.py index 98a72a7..7be1515 100644 --- a/frontends/ycdl_flask/backend/endpoints/channel_endpoints.py +++ b/frontends/ycdl_flask/backend/endpoints/channel_endpoints.py @@ -115,7 +115,7 @@ def post_add_channel(): except ycdl.ytapi.ChannelNotFound: return flasktools.json_response({}, status=404) - channel = common.ycdldb.add_channel(channel_id, get_videos=True) + channel = common.ycdldb.add_channel(channel_id, get_videos=True, commit=True) return flasktools.json_response(channel.jsonify()) @site.route('/channel//delete', methods=['POST']) @@ -125,7 +125,7 @@ def post_delete_channel(channel_id): except ycdl.exceptions.NoSuchChannel as exc: return flasktools.json_response(exc.jsonify(), status=404) - channel.delete() + channel.delete(commit=True) return flasktools.json_response({}) @site.route('/channel//refresh', methods=['POST']) @@ -137,14 +137,14 @@ def post_refresh_channel(channel_id): except ycdl.exceptions.NoSuchChannel as exc: return flasktools.json_response(exc.jsonify(), status=404) - channel.refresh(force=force) + channel.refresh(force=force, commit=True) return flasktools.json_response(channel.jsonify()) @site.route('/refresh_all_channels', methods=['POST']) def post_refresh_all_channels(): force = request.form.get('force', False) force = stringtools.truthystring(force, False) - common.ycdldb.refresh_all_channels(force=force, skip_failures=True) + common.ycdldb.refresh_all_channels(force=force, skip_failures=True, commit=True) return flasktools.json_response({}) @site.route('/channel//set_automark', methods=['POST']) @@ -153,7 +153,7 @@ def post_set_automark(channel_id): channel = common.ycdldb.get_channel(channel_id) try: - channel.set_automark(state) + channel.set_automark(state, commit=True) except ycdl.exceptions.InvalidVideoState: flask.abort(400) @@ -167,7 +167,7 @@ def post_set_autorefresh(channel_id): try: autorefresh = stringtools.truthystring(autorefresh) - channel.set_autorefresh(autorefresh) + channel.set_autorefresh(autorefresh, commit=True) except (ValueError, TypeError): flask.abort(400) @@ -179,7 +179,7 @@ def post_set_download_directory(channel_id): channel = common.ycdldb.get_channel(channel_id) try: - channel.set_download_directory(download_directory) + channel.set_download_directory(download_directory, commit=True) except pathclass.NotDirectory: exc = { 'error_type': 'NOT_DIRECTORY', @@ -196,7 +196,7 @@ def post_set_queuefile_extension(channel_id): extension = request.form['extension'] channel = common.ycdldb.get_channel(channel_id) - channel.set_queuefile_extension(extension) + channel.set_queuefile_extension(extension, commit=True) response = {'queuefile_extension': channel.queuefile_extension} return flasktools.json_response(response) diff --git a/frontends/ycdl_flask/backend/endpoints/video_endpoints.py b/frontends/ycdl_flask/backend/endpoints/video_endpoints.py index ac7ae13..64fdf33 100644 --- a/frontends/ycdl_flask/backend/endpoints/video_endpoints.py +++ b/frontends/ycdl_flask/backend/endpoints/video_endpoints.py @@ -19,7 +19,7 @@ def post_mark_video_state(): for video_id in video_ids: video = common.ycdldb.get_video(video_id) video.mark_state(state, commit=False) - common.ycdldb.sql.commit() + common.ycdldb.commit() except ycdl.exceptions.NoSuchVideo: common.ycdldb.rollback() @@ -40,7 +40,7 @@ def post_start_download(): video_ids = video_ids.split(',') for video_id in video_ids: common.ycdldb.download_video(video_id, commit=False) - common.ycdldb.sql.commit() + common.ycdldb.commit() except ycdl.ytapi.VideoNotFound: common.ycdldb.rollback() diff --git a/frontends/ycdl_flask/templates/channel.html b/frontends/ycdl_flask/templates/channel.html index 5f6ff5b..1b903b2 100644 --- a/frontends/ycdl_flask/templates/channel.html +++ b/frontends/ycdl_flask/templates/channel.html @@ -259,6 +259,8 @@ https://stackoverflow.com/a/35153397
+
Channel ID: {{channel.id}}
+
diff --git a/frontends/ycdl_flask/ycdl_flask_dev.py b/frontends/ycdl_flask/ycdl_flask_dev.py index ca04f6f..6446c5b 100644 --- a/frontends/ycdl_flask/ycdl_flask_dev.py +++ b/frontends/ycdl_flask/ycdl_flask_dev.py @@ -40,7 +40,6 @@ from voussoirkit import vlogging log = vlogging.getLogger(__name__, 'ycdl_flask_dev') import ycdl -import youtube_credentials import backend site = backend.site @@ -54,7 +53,6 @@ HTTPS_DIR = pathclass.Path(__file__).parent.with_child('https') def ycdl_flask_launch( *, - create, localhost_only, port, refresh_rate, @@ -79,8 +77,12 @@ def ycdl_flask_launch( if localhost_only: site.localhost_only = True - youtube_core = ycdl.ytapi.Youtube(youtube_credentials.get_youtube_key()) - backend.common.init_ycdldb(youtube_core, create=create) + try: + backend.common.init_ycdldb() + except ycdl.exceptions.NoClosestYCDLDB as exc: + log.error(exc.error_message) + log.error('Try `ycdl_cli.py init` to create the database.') + return 1 message = f'Starting server on port {port}, pid={os.getpid()}.' if use_https: @@ -100,7 +102,6 @@ def ycdl_flask_launch( def ycdl_flask_launch_argparse(args): return ycdl_flask_launch( - create=args.create, localhost_only=args.localhost_only, port=args.port, refresh_rate=args.refresh_rate, @@ -113,7 +114,6 @@ def main(argv): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('port', nargs='?', type=int, default=5000) - parser.add_argument('--dont_create', '--dont-create', '--no-create', dest='create', action='store_false', default=True) parser.add_argument('--https', dest='use_https', action='store_true', default=None) parser.add_argument('--localhost_only', '--localhost-only', dest='localhost_only', action='store_true') parser.add_argument('--refresh_rate', '--refresh-rate', dest='refresh_rate', type=int, default=None) diff --git a/frontends/ycdl_flask/ycdl_flask_prod.py b/frontends/ycdl_flask/ycdl_flask_prod.py index c9e18f4..db67dad 100644 --- a/frontends/ycdl_flask/ycdl_flask_prod.py +++ b/frontends/ycdl_flask/ycdl_flask_prod.py @@ -9,7 +9,6 @@ gunicorn ycdl_flask_prod:site --bind "0.0.0.0:PORT" --access-logfile "-" import werkzeug.middleware.proxy_fix import ycdl -import youtube_credentials from ycdl_flask import backend @@ -19,6 +18,5 @@ site = backend.site site.debug = False # NOTE: Consider adding a local .json config file. -youtube_core = ycdl.ytapi.Youtube(youtube_credentials.get_youtube_key()) -backend.common.init_ycdldb(youtube_core, create=False) +backend.common.init_ycdldb() backend.common.start_refresher_thread(86400) diff --git a/frontends/ycdl_repl.py b/frontends/ycdl_repl.py index 52ca449..ad428ea 100644 --- a/frontends/ycdl_repl.py +++ b/frontends/ycdl_repl.py @@ -1,13 +1,47 @@ -''' -Run `python -i ycdl_repl.py to get an interpreter -session with these variables preloaded. -''' -import logging -logging.basicConfig() -logging.getLogger('ycdl').setLevel(logging.DEBUG) +import argparse +import code +import sys + +from voussoirkit import interactive +from voussoirkit import pipeable +from voussoirkit import vlogging import ycdl -import youtube_credentials -youtube = ycdl.ytapi.Youtube(youtube_credentials.get_youtube_key()) -Y = ycdl.ycdldb.YCDLDB(youtube) +def yrepl_argparse(args): + global Y + + try: + Y = ycdl.ycdldb.YCDLDB.closest_ycdldb() + except ycdl.exceptions.NoClosestYCDLDB as exc: + pipeable.stderr(exc.error_message) + pipeable.stderr('Try `ycdl_cli.py init` to create the database.') + return 1 + + if args.exec_statement: + exec(args.exec_statement) + Y.commit() + else: + while True: + try: + code.interact(banner='', local=dict(globals(), **locals())) + except SystemExit: + pass + if len(Y.savepoints) == 0: + break + print('You have uncommited changes, are you sure you want to quit?') + if interactive.getpermission(): + break + +@vlogging.main_decorator +def main(argv): + parser = argparse.ArgumentParser() + + parser.add_argument('--exec', dest='exec_statement', default=None) + parser.set_defaults(func=yrepl_argparse) + + args = parser.parse_args(argv) + return args.func(args) + +if __name__ == '__main__': + raise SystemExit(main(sys.argv[1:])) diff --git a/utilities/database_upgrader.py b/utilities/database_upgrader.py index 59f7d2a..35bc489 100644 --- a/utilities/database_upgrader.py +++ b/utilities/database_upgrader.py @@ -2,7 +2,6 @@ import argparse import sys import ycdl -import youtube_credentials class Migrator: ''' @@ -320,8 +319,7 @@ def upgrade_all(data_directory): Given the directory containing a ycdl database, apply all of the needed upgrade_x_to_y functions in order. ''' - youtube = ycdl.ytapi.Youtube(youtube_credentials.get_youtube_key()) - ycdldb = ycdl.ycdldb.YCDLDB(youtube, data_directory, skip_version_check=True) + ycdldb = ycdl.ycdldb.YCDLDB(data_directory, skip_version_check=True) cur = ycdldb.sql.cursor() diff --git a/utilities/download_thumbnails.py b/utilities/download_thumbnails.py index 00001d7..0fc7e3f 100644 --- a/utilities/download_thumbnails.py +++ b/utilities/download_thumbnails.py @@ -4,10 +4,8 @@ import traceback from voussoirkit import downloady import ycdl -import youtube_credentials -youtube_core = ycdl.ytapi.Youtube(youtube_credentials.get_youtube_key()) -ycdldb = ycdl.ycdldb.YCDLDB(youtube_core) +ycdldb = ycdl.ycdldb.YCDLDB() DIRECTORY = '.\\youtube thumbnails' diff --git a/ycdl/constants.py b/ycdl/constants.py index 430bf6b..7162be0 100644 --- a/ycdl/constants.py +++ b/ycdl/constants.py @@ -50,7 +50,7 @@ COMMIT; SQL_COLUMNS = sqlhelpers.extract_table_column_map(DB_INIT) SQL_INDEX = sqlhelpers.reverse_table_column_map(SQL_COLUMNS) -DEFAULT_DATADIR = '.' +DEFAULT_DATADIR = '_ycdl' DEFAULT_DBNAME = 'ycdl.db' DEFAULT_CONFIGNAME = 'ycdl.json' diff --git a/ycdl/exceptions.py b/ycdl/exceptions.py index 1d64670..2fdbfc0 100644 --- a/ycdl/exceptions.py +++ b/ycdl/exceptions.py @@ -54,6 +54,11 @@ class NoSuchVideo(YCDLException): class NoVideos(YCDLException): error_message = 'Channel {} has no videos.' +# CHANNEL ERRORS ################################################################################### + +class ChannelRefreshFailed(YCDLException): + error_message = 'failed to refresh {channel} ({exc}).' + # VIDEO ERRORS ##################################################################################### class InvalidVideoState(YCDLException): @@ -64,14 +69,6 @@ class InvalidVideoState(YCDLException): class RSSAssistFailed(YCDLException): error_message = '{}' -# SQL ERRORS ####################################################################################### - -class BadSQL(YCDLException): - pass - -class BadTable(BadSQL): - error_message = 'Table "{}" does not exist.' - # GENERAL ERRORS ################################################################################### class BadDataDirectory(YCDLException): @@ -86,3 +83,10 @@ Please run utilities\\database_upgrader.py "{filepath.absolute_path}" '''.strip() class DatabaseOutOfDate(YCDLException): error_message = OUTOFDATE + +class NoClosestYCDLDB(YCDLException): + ''' + For calls to YCDLDB.closest_ycdldb where none exists between cwd and + drive root. + ''' + error_message = 'There is no YCDLDB in "{}" or its parents.' diff --git a/ycdl/objects.py b/ycdl/objects.py index be2a99d..035b021 100644 --- a/ycdl/objects.py +++ b/ycdl/objects.py @@ -1,41 +1,33 @@ import datetime +import googleapiclient.errors import typing from voussoirkit import pathclass from voussoirkit import stringtools +from voussoirkit import vlogging +from voussoirkit import worms + +log = vlogging.getLogger(__name__) from . import constants from . import exceptions from . import ytrss -def normalize_db_row(db_row, table) -> dict: - ''' - Raises KeyError if table is not one of the recognized tables. - - Raises TypeError if db_row is not the right type. - ''' - if isinstance(db_row, dict): - return db_row - - if isinstance(db_row, (list, tuple)): - return dict(zip(constants.SQL_COLUMNS[table], db_row)) - - raise TypeError(f'db_row should be {dict}, {list}, or {tuple}, not {type(db_row)}.') - -class Base: +class ObjectBase(worms.Object): def __init__(self, ycdldb): - super().__init__() + super().__init__(ycdldb) self.ycdldb = ycdldb -class Channel(Base): +class Channel(ObjectBase): table = 'channels' + no_such_exception = exceptions.NoSuchChannel def __init__(self, ycdldb, db_row): super().__init__(ycdldb) - db_row = normalize_db_row(db_row, self.table) + db_row = self.ycdldb.normalize_db_row(db_row, self.table) self.id = db_row['id'] - self.name = db_row['name'] + self.name = db_row['name'] or self.id self.uploads_playlist = db_row['uploads_playlist'] self.download_directory = self.normalize_download_directory( db_row['download_directory'], @@ -48,6 +40,9 @@ class Channel(Base): def __repr__(self): return f'Channel:{self.id}' + def __str__(self): + return f'Channel:{self.id}:{self.name}' + @staticmethod def normalize_autorefresh(autorefresh): if isinstance(autorefresh, (str, int)): @@ -82,6 +77,20 @@ class Channel(Base): return download_directory + @staticmethod + def normalize_name(name): + if name is None: + return None + + if not isinstance(name, str): + raise TypeError(f'name should be {str}, not {type(name)}.') + + name = name.strip() + if not name: + return None + + return name + @staticmethod def normalize_queuefile_extension(queuefile_extension) -> typing.Optional[str]: if queuefile_extension is None: @@ -112,19 +121,17 @@ class Channel(Base): videos = self.ycdldb.youtube.get_videos(new_ids) return videos - def delete(self, commit=True): - self.ycdldb.log.info('Deleting %s.', self) + @worms.transaction + def delete(self): + log.info('Deleting %s.', self) - self.ycdldb.sql_delete(table='videos', pairs={'author_id': self.id}) - self.ycdldb.sql_delete(table='channels', pairs={'id': self.id}) - - if commit: - self.ycdldb.commit() + self.ycdldb.delete(table='videos', pairs={'author_id': self.id}) + self.ycdldb.delete(table='channels', pairs={'id': self.id}) def get_most_recent_video_id(self): query = 'SELECT id FROM videos WHERE author_id == ? ORDER BY published DESC LIMIT 1' bindings = [self.id] - row = self.ycdldb.sql_select_one(query, bindings) + row = self.ycdldb.select_one(query, bindings) if row is None: raise exceptions.NoVideos(self) return row[0] @@ -132,7 +139,7 @@ class Channel(Base): def has_pending(self): query = 'SELECT 1 FROM videos WHERE author_id == ? AND state == "pending" LIMIT 1' bindings = [self.id] - return self.ycdldb.sql_select_one(query, bindings) is not None + return self.ycdldb.select_one(query, bindings) is not None def jsonify(self): j = { @@ -142,8 +149,9 @@ class Channel(Base): } return j - def refresh(self, *, force=False, rss_assisted=True, commit=True): - self.ycdldb.log.info('Refreshing %s.', self.id) + @worms.transaction + def refresh(self, *, force=False, rss_assisted=True): + log.info('Refreshing %s.', self.id) if force or (not self.uploads_playlist): self.reset_uploads_playlist_id() @@ -154,16 +162,20 @@ class Channel(Base): try: video_generator = self._rss_assisted_videos() except exceptions.RSSAssistFailed as exc: - self.ycdldb.log.debug('Caught %s.', exc) + log.debug('Caught %s.', exc) video_generator = self.ycdldb.youtube.get_playlist_videos(self.uploads_playlist) seen_ids = set() - for video in video_generator: - seen_ids.add(video.id) - status = self.ycdldb.ingest_video(video, commit=False) - if (not status['new']) and (not force): - break + try: + for video in video_generator: + seen_ids.add(video.id) + status = self.ycdldb.ingest_video(video) + + if (not status['new']) and (not force): + break + except googleapiclient.errors.HttpError as exc: + raise exceptions.ChannelRefreshFailed(channel=self.id, exc=exc) # Now we will refresh some other IDs that may not have been refreshed # by the previous loop. @@ -186,15 +198,12 @@ class Channel(Base): refresh_ids.update(v.id for v in videos) if refresh_ids: - self.ycdldb.log.debug('Refreshing %d ids separately.', len(refresh_ids)) + log.debug('Refreshing %d ids separately.', len(refresh_ids)) # We call ingest_video instead of insert_video so that # premieres / livestreams which have finished can be automarked. for video_id in self.ycdldb.youtube.get_videos(refresh_ids): - self.ycdldb.ingest_video(video_id, commit=False) - - if commit: - self.ycdldb.commit() + self.ycdldb.ingest_video(video_id) def reset_uploads_playlist_id(self): ''' @@ -204,7 +213,8 @@ class Channel(Base): self.set_uploads_playlist_id(self.uploads_playlist) return self.uploads_playlist - def set_automark(self, state, commit=True): + @worms.transaction + def set_automark(self, state): if state not in constants.VIDEO_STATES: raise exceptions.InvalidVideoState(state) @@ -212,53 +222,56 @@ class Channel(Base): 'id': self.id, 'automark': state, } - self.ycdldb.sql_update(table='channels', pairs=pairs, where_key='id') + self.ycdldb.update(table='channels', pairs=pairs, where_key='id') self.automark = state - if commit: - self.ycdldb.commit() - - def set_autorefresh(self, autorefresh, commit=True): + @worms.transaction + def set_autorefresh(self, autorefresh): autorefresh = self.normalize_autorefresh(autorefresh) pairs = { 'id': self.id, 'autorefresh': autorefresh, } - self.ycdldb.sql_update(table='channels', pairs=pairs, where_key='id') + self.ycdldb.update(table='channels', pairs=pairs, where_key='id') self.autorefresh = autorefresh - if commit: - self.ycdldb.commit() - - def set_download_directory(self, download_directory, commit=True): + @worms.transaction + def set_download_directory(self, download_directory): download_directory = self.normalize_download_directory(download_directory) pairs = { 'id': self.id, 'download_directory': download_directory.absolute_path if download_directory else None, } - self.ycdldb.sql_update(table='channels', pairs=pairs, where_key='id') + self.ycdldb.update(table='channels', pairs=pairs, where_key='id') self.download_directory = download_directory - if commit: - self.ycdldb.commit() + @worms.transaction + def set_name(self, name): + name = self.normalize_name(name) - def set_queuefile_extension(self, queuefile_extension, commit=True): + pairs = { + 'id': self.id, + 'name': name, + } + self.ycdldb.update(table='channels', pairs=pairs, where_key='id') + self.name = name + + @worms.transaction + def set_queuefile_extension(self, queuefile_extension): queuefile_extension = self.normalize_queuefile_extension(queuefile_extension) pairs = { 'id': self.id, 'queuefile_extension': queuefile_extension, } - self.ycdldb.sql_update(table='channels', pairs=pairs, where_key='id') + self.ycdldb.update(table='channels', pairs=pairs, where_key='id') self.queuefile_extension = queuefile_extension - if commit: - self.ycdldb.commit() - - def set_uploads_playlist_id(self, playlist_id, commit=True): - self.ycdldb.log.debug('Setting %s upload playlist to %s.', self.id, playlist_id) + @worms.transaction + def set_uploads_playlist_id(self, playlist_id): + log.debug('Setting %s upload playlist to %s.', self.id, playlist_id) if not isinstance(playlist_id, str): raise TypeError(f'Playlist id must be a string, not {type(playlist_id)}.') @@ -266,18 +279,16 @@ class Channel(Base): 'id': self.id, 'uploads_playlist': playlist_id, } - self.ycdldb.sql_update(table='channels', pairs=pairs, where_key='id') + self.ycdldb.update(table='channels', pairs=pairs, where_key='id') self.uploads_playlist = playlist_id - if commit: - self.ycdldb.commit() - -class Video(Base): +class Video(ObjectBase): table = 'videos' + no_such_exception = exceptions.NoSuchVideo def __init__(self, ycdldb, db_row): super().__init__(ycdldb) - db_row = normalize_db_row(db_row, self.table) + db_row = self.ycdldb.normalize_db_row(db_row, self.table) self.id = db_row['id'] self.published = db_row['published'] @@ -300,13 +311,11 @@ class Video(Base): except exceptions.NoSuchChannel: return None - def delete(self, commit=True): - self.ycdldb.log.info('Deleting %s.', self) + @worms.transaction + def delete(self): + log.info('Deleting %s.', self) - self.ycdldb.sql_delete(table='videos', pairs={'id': self.id}) - - if commit: - self.ycdldb.commit() + self.ycdldb.delete(table='videos', pairs={'id': self.id}) def jsonify(self): j = { @@ -322,24 +331,25 @@ class Video(Base): } return j - def mark_state(self, state, commit=True): + @worms.transaction + def mark_state(self, state): ''' Mark the video as ignored, pending, or downloaded. + + Note: Marking as downloaded will not create the queue file, this only + updates the database. See yclddb.download_video. ''' if state not in constants.VIDEO_STATES: raise exceptions.InvalidVideoState(state) - self.ycdldb.log.info('Marking %s as %s.', self, state) + log.info('Marking %s as %s.', self, state) pairs = { 'id': self.id, 'state': state, } self.state = state - self.ycdldb.sql_update(table='videos', pairs=pairs, where_key='id') - - if commit: - self.ycdldb.commit() + self.ycdldb.update(table='videos', pairs=pairs, where_key='id') @property def published_string(self): diff --git a/ycdl/ycdldb.py b/ycdl/ycdldb.py index ee473cb..d186d8f 100644 --- a/ycdl/ycdldb.py +++ b/ycdl/ycdldb.py @@ -1,176 +1,92 @@ import json import sqlite3 +from voussoirkit import cacheclass +from voussoirkit import configlayers +from voussoirkit import pathclass +from voussoirkit import vlogging +from voussoirkit import worms + +log = vlogging.getLogger(__name__) + from . import constants from . import exceptions from . import objects from . import ytapi from . import ytrss -from voussoirkit import cacheclass -from voussoirkit import configlayers -from voussoirkit import pathclass -from voussoirkit import sqlhelpers -from voussoirkit import vlogging - -class YCDLDBCacheManagerMixin: - _THING_CLASSES = { - 'channel': - { - 'class': objects.Channel, - 'exception': exceptions.NoSuchChannel, - }, - 'video': - { - 'class': objects.Video, - 'exception': exceptions.NoSuchVideo, - }, - } - - def __init__(self): - super().__init__() - - def get_cached_instance(self, thing_type, db_row): - ''' - Check if there is already an instance in the cache and return that. - Otherwise, a new instance is created, cached, and returned. - - Note that in order to call this method you have to already have a - db_row which means performing some select. If you only have the ID, - use get_thing_by_id, as there may already be a cached instance to save - you the select. - ''' - thing_map = self._THING_CLASSES[thing_type] - - thing_class = thing_map['class'] - thing_table = thing_class.table - thing_cache = self.caches[thing_type] - - if isinstance(db_row, dict): - thing_id = db_row['id'] - else: - thing_index = constants.SQL_INDEX[thing_table] - thing_id = db_row[thing_index['id']] - - try: - thing = thing_cache[thing_id] - except KeyError: - thing = thing_class(self, db_row) - thing_cache[thing_id] = thing - return thing - - def get_thing_by_id(self, thing_type, thing_id): - ''' - This method will first check the cache to see if there is already an - instance with that ID, in which case we don't need to perform any SQL - select. If it is not in the cache, then a new instance is created, - cached, and returned. - ''' - thing_map = self._THING_CLASSES[thing_type] - - thing_class = thing_map['class'] - if isinstance(thing_id, thing_class): - # This could be used to check if your old reference to an object is - # still in the cache, or re-select it from the db to make sure it - # still exists and re-cache. - # Probably an uncommon need but... no harm I think. - thing_id = thing_id.id - - thing_cache = self.caches[thing_type] - try: - return thing_cache[thing_id] - except KeyError: - pass - - query = f'SELECT * FROM {thing_class.table} WHERE id == ?' - bindings = [thing_id] - thing_row = self.sql_select_one(query, bindings) - if thing_row is None: - raise thing_map['exception'](thing_id) - thing = thing_class(self, thing_row) - thing_cache[thing_id] = thing - return thing - - def get_things(self, thing_type): - ''' - Yield things, unfiltered, in whatever order they appear in the database. - ''' - thing_map = self._THING_CLASSES[thing_type] - table = thing_map['class'].table - query = f'SELECT * FROM {table}' - - things = self.sql_select(query) - for thing_row in things: - thing = self.get_cached_instance(thing_type, thing_row) - yield thing - - def get_things_by_sql(self, thing_type, query, bindings=None): - ''' - Use an arbitrary SQL query to select things from the database. - Your query select *, all the columns of the thing's table. - ''' - thing_rows = self.sql_select(query, bindings) - for thing_row in thing_rows: - yield self.get_cached_instance(thing_type, thing_row) +import youtube_credentials class YCDLDBChannelMixin: def __init__(self): super().__init__() + @worms.transaction def add_channel( self, channel_id, *, - commit=True, + automark='pending', download_directory=None, queuefile_extension=None, get_videos=False, name=None, ): + ''' + Raises exceptions.InvalidVideoState if automark is not + one of constants.VIDEO_STATES. + Raises TypeError if name is not a string. + Raises TypeError if queuefile_extension is not a string. + Raises pathclass.NotDirectory is download_directory is not an existing + directory (via objects.Channel.normalize_download_directory). + ''' try: return self.get_channel(channel_id) except exceptions.NoSuchChannel: pass + if automark not in constants.VIDEO_STATES: + raise exceptions.InvalidVideoState(automark) + + name = objects.Channel.normalize_name(name) if name is None: name = self.youtube.get_user_name(channel_id) download_directory = objects.Channel.normalize_download_directory(download_directory) + download_directory = download_directory.absolute_path if download_directory else None queuefile_extension = objects.Channel.normalize_queuefile_extension(queuefile_extension) - self.log.info('Adding channel %s %s', channel_id, name) + log.info('Adding channel %s %s', channel_id, name) data = { 'id': channel_id, 'name': name, 'uploads_playlist': self.youtube.get_user_uploads_playlist_id(channel_id), - 'download_directory': download_directory.absolute_path if download_directory else None, + 'download_directory': download_directory, 'queuefile_extension': queuefile_extension, - 'automark': 'pending', + 'automark': automark, 'autorefresh': True, } - self.sql_insert(table='channels', data=data) + self.insert(table='channels', data=data) channel = objects.Channel(self, data) - self.caches['channel'][channel_id] = channel if get_videos: - channel.refresh(commit=False) + channel.refresh() - if commit: - self.commit() return channel def get_channel(self, channel_id): - return self.get_thing_by_id('channel', channel_id) + return self.get_object_by_id(objects.Channel, channel_id) def get_channels(self): - return self.get_things(thing_type='channel') + return self.get_objects(objects.Channel) def get_channels_by_sql(self, query, bindings=None): - return self.get_things_by_sql('channel', query, bindings) + return self.get_objects_by_sql(objects.Channel, query, bindings) - def _rss_assisted_refresh(self, channels, skip_failures=False, commit=True): + @worms.transaction + def _rss_assisted_refresh(self, channels, skip_failures=False): ''' Youtube provides RSS feeds for every channel. These feeds do not require the API token and seem to have generous ratelimits, or @@ -200,6 +116,7 @@ class YCDLDBChannelMixin: channel.refresh(rss_assisted=False) except Exception as exc: if skip_failures: + log.warning(exc) excs.append(exc) else: raise @@ -210,7 +127,7 @@ class YCDLDBChannelMixin: new_ids = ytrss.get_user_videos_since(channel.id, most_recent_video) yield from new_ids except (exceptions.NoVideos, exceptions.RSSAssistFailed) as exc: - self.log.debug( + log.debug( 'RSS assist for %s failed "%s", using traditional refresh.', channel.id, exc.error_message @@ -219,115 +136,43 @@ class YCDLDBChannelMixin: new_ids = (id for channel in channels for id in assisted(channel)) for video in self.youtube.get_videos(new_ids): - self.ingest_video(video, commit=False) - - if commit: - self.commit() + self.ingest_video(video) return excs + @worms.transaction def refresh_all_channels( self, *, force=False, rss_assisted=True, skip_failures=False, - commit=True, ): - self.log.info('Refreshing all channels.') + log.info('Refreshing all channels.') channels = self.get_channels_by_sql('SELECT * FROM channels WHERE autorefresh == 1') if rss_assisted and not force: - return self._rss_assisted_refresh(channels, skip_failures=skip_failures, commit=commit) + return self._rss_assisted_refresh(channels, skip_failures=skip_failures) excs = [] for channel in channels: try: - channel.refresh(force=force, commit=commit) + channel.refresh(force=force) except Exception as exc: if skip_failures: - self.log.warning(exc) + log.warning(exc) excs.append(exc) else: raise - if commit: - self.commit() - return excs -class YCDLSQLMixin: - def __init__(self): - super().__init__() - self._cached_sql_tables = None - - def assert_table_exists(self, table): - if not self._cached_sql_tables: - self._cached_sql_tables = self.get_sql_tables() - if table not in self._cached_sql_tables: - raise exceptions.BadTable(table) - - def commit(self, message=None): - if message is not None: - self.log.debug('Committing - %s.', message) - - self.sql.commit() - - def get_sql_tables(self): - query = 'SELECT name FROM sqlite_master WHERE type = "table"' - cur = self.sql_execute(query) - tables = set(row[0] for row in cur.fetchall()) - return tables - - def rollback(self): - self.log.debug('Rolling back.') - self.sql_execute('ROLLBACK') - - def sql_delete(self, table, pairs): - self.assert_table_exists(table) - (qmarks, bindings) = sqlhelpers.delete_filler(pairs) - query = f'DELETE FROM {table} {qmarks}' - self.sql_execute(query, bindings) - - def sql_execute(self, query, bindings=[]): - if bindings is None: - bindings = [] - cur = self.sql.cursor() - self.log.loud('%s %s', query, bindings) - cur.execute(query, bindings) - return cur - - def sql_insert(self, table, data): - self.assert_table_exists(table) - column_names = constants.SQL_COLUMNS[table] - (qmarks, bindings) = sqlhelpers.insert_filler(column_names, data) - - query = f'INSERT INTO {table} VALUES({qmarks})' - self.sql_execute(query, bindings) - - def sql_select(self, query, bindings=None): - cur = self.sql_execute(query, bindings) - while True: - fetch = cur.fetchone() - if fetch is None: - break - yield fetch - - def sql_select_one(self, query, bindings=None): - cur = self.sql_execute(query, bindings) - return cur.fetchone() - - def sql_update(self, table, pairs, where_key): - self.assert_table_exists(table) - (qmarks, bindings) = sqlhelpers.update_filler(pairs, where_key=where_key) - query = f'UPDATE {table} {qmarks}' - self.sql_execute(query, bindings) - class YCDLDBVideoMixin: def __init__(self): super().__init__() - def download_video(self, video, commit=True, force=False): + @worms.transaction + def download_video(self, video, force=False): ''' Create the queuefile within the channel's associated directory, or the default directory from the config file. @@ -342,7 +187,7 @@ class YCDLDBVideoMixin: raise TypeError(video) if video.state != 'pending' and not force: - self.log.debug('%s does not need to be downloaded.', video.id) + log.debug('%s does not need to be downloaded.', video.id) return try: @@ -356,18 +201,17 @@ class YCDLDBVideoMixin: download_directory = pathclass.Path(download_directory) queuefile = download_directory.with_child(video.id).replace_extension(extension) - self.log.info('Creating %s.', queuefile.absolute_path) + def create_queuefile(): + log.info('Creating %s.', queuefile.absolute_path) - download_directory.makedirs(exist_ok=True) - queuefile.touch() + download_directory.makedirs(exist_ok=True) + queuefile.touch() - video.mark_state('downloaded', commit=False) - - if commit: - self.commit() + self.on_commit_queue.append({'action': create_queuefile}) + video.mark_state('downloaded') def get_video(self, video_id): - return self.get_thing_by_id('video', video_id) + return self.get_object_by_id(objects.Video, video_id) def get_videos(self, channel_id=None, *, state=None, orderby=None): wheres = [] @@ -403,32 +247,31 @@ class YCDLDBVideoMixin: query = 'SELECT * FROM videos' + wheres + orderbys - self.log.debug('%s %s', query, bindings) - explain = self.sql_execute('EXPLAIN QUERY PLAN ' + query, bindings) - self.log.debug('\n'.join(str(x) for x in explain.fetchall())) + log.debug('%s %s', query, bindings) + explain = self.execute('EXPLAIN QUERY PLAN ' + query, bindings) + log.debug('\n'.join(str(x) for x in explain.fetchall())) - rows = self.sql_select(query, bindings) + rows = self.select(query, bindings) for row in rows: - yield self.get_cached_instance('video', row) + yield self.get_cached_instance(objects.Video, row) def get_videos_by_sql(self, query, bindings=None): - return self.get_things_by_sql('video', query, bindings) + return self.get_objects_by_sql(objects.Video, query, bindings) - def insert_playlist(self, playlist_id, commit=True): + @worms.transaction + def insert_playlist(self, playlist_id): video_generator = self.youtube.get_playlist_videos(playlist_id) - results = [self.insert_video(video, commit=False) for video in video_generator] - - if commit: - self.commit() + results = [self.insert_video(video) for video in video_generator] return results - def ingest_video(self, video, commit=True): + @worms.transaction + def ingest_video(self, video): ''' Call `insert_video`, and additionally use the channel's automark to mark this video's state. ''' - status = self.insert_video(video, commit=False) + status = self.insert_video(video) if not status['new']: return status @@ -444,28 +287,26 @@ class YCDLDBVideoMixin: if author.automark == 'downloaded': if video.live_broadcast is not None: - self.log.debug( + log.debug( 'Not downloading %s because live_broadcast=%s.', video.id, video.live_broadcast, ) return status # download_video contains a call to mark_state. - self.download_video(video.id, commit=False) + self.download_video(video.id) else: - video.mark_state(author.automark, commit=False) - - if commit: - self.commit() + video.mark_state(author.automark) return status - def insert_video(self, video, *, add_channel=True, commit=True): + @worms.transaction + def insert_video(self, video, *, add_channel=True): if not isinstance(video, ytapi.Video): video = self.youtube.get_video(video) if add_channel: - self.add_channel(video.author_id, get_videos=False, commit=False) + self.add_channel(video.author_id, get_videos=False) try: existing = self.get_video(video.id) @@ -490,19 +331,16 @@ class YCDLDBVideoMixin: } if existing: - self.log.loud('Updating Video %s.', video.id) - self.sql_update(table='videos', pairs=data, where_key='id') + log.loud('Updating Video %s.', video.id) + self.update(table='videos', pairs=data, where_key='id') else: - self.log.loud('Inserting Video %s.', video.id) - self.sql_insert(table='videos', data=data) + log.loud('Inserting Video %s.', video.id) + self.insert(table='videos', data=data) # Override the cached copy with the new copy so that the cache contains # updated information (view counts etc.). video = objects.Video(self, data) - self.caches['video'][video.id] = video - - if commit: - self.commit() + self.caches[objects.Video][video.id] = video # For the benefit of ingest_video, which will only apply the channel's # automark to newly released videos, let's consider the video to be @@ -516,20 +354,21 @@ class YCDLDBVideoMixin: return {'new': is_new, 'video': video} class YCDLDB( - YCDLDBCacheManagerMixin, YCDLDBChannelMixin, YCDLDBVideoMixin, - YCDLSQLMixin, + worms.DatabaseWithCaching, ): def __init__( self, - youtube, + youtube=None, *, - create=True, + create=False, data_directory=None, skip_version_check=False, ): super().__init__() + if youtube is None: + youtube = ytapi.Youtube(youtube_credentials.get_youtube_key()) self.youtube = youtube # DATA DIR PREP @@ -541,10 +380,41 @@ class YCDLDB( if self.data_directory.exists and not self.data_directory.is_dir: raise exceptions.BadDataDirectory(self.data_directory.absolute_path) - # LOGGING - self.log = vlogging.getLogger(f'{__name__}:{self.data_directory.absolute_path}') - # DATABASE + self._init_sql(create=create, skip_version_check=skip_version_check) + + # CONFIG + self.config_filepath = self.data_directory.with_child(constants.DEFAULT_CONFIGNAME) + self.load_config() + + # WORMS + self._init_column_index() + self._init_caches() + + def _check_version(self): + ''' + Compare database's user_version against constants.DATABASE_VERSION, + raising exceptions.DatabaseOutOfDate if not correct. + ''' + existing = self.execute('PRAGMA user_version').fetchone()[0] + if existing != constants.DATABASE_VERSION: + raise exceptions.DatabaseOutOfDate( + existing=existing, + new=constants.DATABASE_VERSION, + filepath=self.data_directory, + ) + + def _init_caches(self): + self.caches = { + objects.Channel: cacheclass.Cache(maxlen=20_000), + objects.Video: cacheclass.Cache(maxlen=50_000), + } + + def _init_column_index(self): + self.COLUMNS = constants.SQL_COLUMNS + self.COLUMN_INDEX = constants.SQL_INDEX + + def _init_sql(self, create, skip_version_check): self.database_filepath = self.data_directory.with_child(constants.DEFAULT_DBNAME) existing_database = self.database_filepath.exists if not existing_database and not create: @@ -561,38 +431,46 @@ class YCDLDB( else: self._first_time_setup() - # CONFIG - self.config_filepath = self.data_directory.with_child(constants.DEFAULT_CONFIGNAME) - self.load_config() - - self.caches = { - 'channel': cacheclass.Cache(maxlen=20_000), - 'video': cacheclass.Cache(maxlen=50_000), - } - - def _check_version(self): - ''' - Compare database's user_version against constants.DATABASE_VERSION, - raising exceptions.DatabaseOutOfDate if not correct. - ''' - existing = self.sql.execute('PRAGMA user_version').fetchone()[0] - if existing != constants.DATABASE_VERSION: - raise exceptions.DatabaseOutOfDate( - existing=existing, - new=constants.DATABASE_VERSION, - filepath=self.data_directory, - ) - def _first_time_setup(self): - self.log.info('Running first-time database setup.') - self.sql.executescript(constants.DB_INIT) + log.info('Running first-time database setup.') + self.executescript(constants.DB_INIT) self.commit() def _load_pragmas(self): - self.log.debug('Reloading pragmas.') - self.sql.executescript(constants.DB_PRAGMAS) + log.debug('Reloading pragmas.') + self.executescript(constants.DB_PRAGMAS) self.commit() + @classmethod + def closest_ycdldb(cls, youtube=None, path='.', *args, **kwargs): + ''' + Starting from the given path and climbing upwards towards the filesystem + root, look for an existing YCDL data directory and return the + YCDLDB object. If none exists, raise exceptions.NoClosestYCDLDB. + ''' + path = pathclass.Path(path) + starting = path + + while True: + possible = path.with_child(constants.DEFAULT_DATADIR) + if possible.is_dir: + break + parent = path.parent + if path == parent: + raise exceptions.NoClosestYCDLDB(starting.absolute_path) + path = parent + + path = possible + ycdldb = cls( + youtube=youtube, + data_directory=path, + create=False, + *args, + **kwargs, + ) + log.debug('Found closest YCDLDB at %s.', path) + return ycdldb + def get_all_states(self): ''' Get a list of all the different states that are currently in use in @@ -602,7 +480,7 @@ class YCDLDB( # arbitrarily many states for user-defined purposes, but I kind of went # back on that so I'm not sure if it will be useful. query = 'SELECT DISTINCT state FROM videos' - states = self.sql_select(query) + states = self.select(query) states = [row[0] for row in states] return sorted(states) diff --git a/ycdl/ytapi.py b/ycdl/ytapi.py index 43175c2..e82c4dc 100644 --- a/ycdl/ytapi.py +++ b/ycdl/ytapi.py @@ -1,4 +1,4 @@ -import apiclient.discovery +import googleapiclient.discovery import isodate from voussoirkit import gentools @@ -53,7 +53,7 @@ class Video: class Youtube: def __init__(self, key): - self.youtube = apiclient.discovery.build( + self.youtube = googleapiclient.discovery.build( cache_discovery=False, developerKey=key, serviceName='youtube',