From 90eedca0d7f53faeae989591681c1504c3d55b93 Mon Sep 17 00:00:00 2001 From: Ethan Dalool Date: Tue, 6 Dec 2016 22:11:09 -0800 Subject: [PATCH] checkpoint --- download_thumbnails.py | 13 ++++++ templates/channel.html | 26 +++++------- templates/channels.html | 7 ++++ ycdl.py | 90 +++++++++++++++++++++++++++++++---------- ycdl_launch.py | 3 +- ycdl_refresh.py | 2 + ycdl_site.py | 27 ++++++++----- ytapi.py | 20 ++++++--- ytqueue.py | 5 ++- 9 files changed, 138 insertions(+), 55 deletions(-) create mode 100644 download_thumbnails.py create mode 100644 ycdl_refresh.py diff --git a/download_thumbnails.py b/download_thumbnails.py new file mode 100644 index 0000000..c19604b --- /dev/null +++ b/download_thumbnails.py @@ -0,0 +1,13 @@ +import os +import ycdl_easy +from voussoirkit import downloady + +DIRECTORY = 'C:\\users\\owner\\youtube thumbnails' + +videos = ycdl_easy.youtube.get_videos() +for video in videos: + thumbnail_path = os.path.join(DIRECTORY, video['id']) + '.jpg' + if os.path.exists(thumbnail_path): + continue + result = downloady.download_file(video['thumbnail'], thumbnail_path) + print(result) diff --git a/templates/channel.html b/templates/channel.html index 52783fa..b6a1ef0 100644 --- a/templates/channel.html +++ b/templates/channel.html @@ -11,9 +11,14 @@ #content_body { display: flex; + flex-grow: 1; + flex-shrink: 0; + flex-basis: auto; flex-direction: column; + width: 1440px; + margin: auto; + max-width: 100%; } - .video_card_downloaded, .video_card_ignored, .video_card_pending @@ -67,8 +72,10 @@ {{header.make_header()}}
- - + + All Pending Ignored @@ -107,7 +114,6 @@ class="video_action_ignore" onclick="mark_video_state('{{video['id']}}', 'ignored', receive_action_response);" >Ignore - {% endif %}
@@ -208,17 +214,5 @@ function start_download(video_id, callback) data.append("video_id", video_id); return post(url, data, callback); } - -function toggle_dropdown(dropdown) -{ - if (dropdown.style.display != "inline-flex") - { - dropdown.style.display = "inline-flex"; - } - else - { - dropdown.style.display = "none"; - } -} diff --git a/templates/channels.html b/templates/channels.html index db3050f..9d4c0a4 100644 --- a/templates/channels.html +++ b/templates/channels.html @@ -11,7 +11,13 @@ #content_body { display: flex; + flex-grow: 1; + flex-shrink: 0; + flex-basis: auto; flex-direction: column; + width: 1440px; + margin: auto; + max-width: 100%; } #new_channel_textbox, #new_channel_button @@ -85,6 +91,7 @@ function refresh_channel(channel_id, force, callback) data.append("force", force) return post(url, data, callback); } + function refresh_all_channels(force, callback) { var url = "/refresh_all_channels"; diff --git a/ycdl.py b/ycdl.py index cb5a8c2..e5bf286 100644 --- a/ycdl.py +++ b/ycdl.py @@ -4,16 +4,15 @@ import sqlite3 import ytapi -# AVAILABLE FORMATTERS: -# url, id -# Note that if the channel has a value in the `directory` column, the bot will -# chdir there before executing. -YOUTUBE_DL_COMMAND = 'touch C:\\Incoming\\ytqueue\\{id}.ytqueue' +def YOUTUBE_DL_COMMAND(video_id): + path = 'C:\\Incoming\\ytqueue\\{id}.ytqueue'.format(id=video_id) + open(path, 'w') logging.basicConfig(level=logging.DEBUG) log = logging.getLogger(__name__) logging.getLogger('googleapiclient.discovery').setLevel(logging.WARNING) logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.WARNING) +logging.getLogger('requests.packages.urllib3.util.retry').setLevel(logging.WARNING) SQL_CHANNEL_COLUMNS = [ 'id', @@ -64,6 +63,7 @@ DEFAULT_DBNAME = 'ycdl.db' ERROR_DATABASE_OUTOFDATE = 'Database is out-of-date. {current} should be {new}' + def verify_is_abspath(path): ''' TO DO: Determine whether this is actually correct. @@ -71,8 +71,16 @@ def verify_is_abspath(path): if os.path.abspath(path) != path: raise ValueError('Not an abspath') + +class InvalidVideoState(Exception): + pass + +class NoSuchVideo(Exception): + pass + + class YCDL: - def __init__(self, youtube, database_filename=None): + def __init__(self, youtube, database_filename=None, youtube_dl_function=None): self.youtube = youtube if database_filename is None: database_filename = DEFAULT_DBNAME @@ -90,6 +98,11 @@ class YCDL: print(message) raise SystemExit + if youtube_dl_function: + self.youtube_dl_function = youtube_dl_function + else: + self.youtube_dl_function = YOUTUBE_DL_COMMAND + statements = DB_INIT.split(';') for statement in statements: self.cur.execute(statement) @@ -134,28 +147,46 @@ class YCDL: return fetch[SQL_CHANNEL['directory']] def download_video(self, video, force=False): - if not isinstance(video, ytapi.Video): - video = self.youtube.get_video(video) + ''' + Execute the `YOUTUBE_DL_COMMAND`, within the channel's associated + directory if applicable. + ''' + # This logic is a little hazier than I would like, but it's all in the + # interest of minimizing unnecessary API calls. + if isinstance(video, ytapi.Video): + video_id = video.id + else: + video_id = video + self.cur.execute('SELECT * FROM videos WHERE id == ?', [video_id]) + video_row = self.cur.fetchone() + if video_row is None: + # Since the video was not in the db, we may not know about the channel either. + if not isinstance(video, ytapi.Video): + print('get video') + video = self.youtube.get_video(video) + channel_id = video.author_id + self.cur.execute('SELECT * FROM channels WHERE id == ?', [channel_id]) + if self.cur.fetchone() is None: + print('add channel') + self.add_channel(channel_id, get_videos=False, commit=False) + video_row = self.insert_video(video, commit=False)['row'] + else: + channel_id = video_row[SQL_VIDEO['author_id']] - self.add_channel(video.author_id, get_videos=False, commit=False) - status = self.insert_video(video, commit=True) - - if status['row'][SQL_VIDEO['download']] != 'pending' and not force: + if video_row[SQL_VIDEO['download']] != 'pending' and not force: print('That video does not need to be downloaded.') return - download_directory = self.channel_directory(video.author_id) + download_directory = self.channel_directory(channel_id) download_directory = download_directory or os.getcwd() current_directory = os.getcwd() os.makedirs(download_directory, exist_ok=True) os.chdir(download_directory) - url = 'https://www.youtube.com/watch?v={id}'.format(id=video.id) - command = YOUTUBE_DL_COMMAND.format(url=url, id=video.id) - os.system(command) + self.youtube_dl_function(video_id) os.chdir(current_directory) - self.cur.execute('UPDATE videos SET download = "downloaded" WHERE id == ?', [video.id]) + self.cur.execute('UPDATE videos SET download = "downloaded" WHERE id == ?', [video_id]) self.sql.commit() def get_channel(self, channel_id): @@ -173,14 +204,29 @@ class YCDL: channels.sort(key=lambda x: x['name'].lower()) return channels - def get_videos(self, channel_id=None): + def get_videos(self, channel_id=None, download_filter=None): + wheres = [] + bindings = [] if channel_id is not None: - self.cur.execute('SELECT * FROM videos WHERE author_id == ?', [channel_id]) + wheres.append('author_id') + bindings.append(channel_id) + + if download_filter is not None: + wheres.append('download') + bindings.append(download_filter) + + if wheres: + wheres = [x + ' == ?' for x in wheres] + wheres = ' WHERE ' + ' AND '.join(wheres) else: - self.cur.execute('SELECT * FROM videos ') + wheres = '' + + query = 'SELECT * FROM videos' + wheres + self.cur.execute(query, bindings) videos = self.cur.fetchall() if not videos: return [] + videos = [{key: video[SQL_VIDEO[key]] for key in SQL_VIDEO} for video in videos] videos.sort(key=lambda x: x['published'], reverse=True) return videos @@ -214,10 +260,10 @@ class YCDL: Mark the video as ignored, pending, or downloaded. ''' if state not in ['ignored', 'pending', 'downloaded']: - raise ValueError(state) + raise InvalidVideoState(state) self.cur.execute('SELECT * FROM videos WHERE id == ?', [video_id]) if self.cur.fetchone() is None: - raise KeyError(video_id) + raise NoSuchVideo(video_id) self.cur.execute('UPDATE videos SET download = ? WHERE id == ?', [state, video_id]) if commit: self.sql.commit() diff --git a/ycdl_launch.py b/ycdl_launch.py index 6ce64f6..1b21718 100644 --- a/ycdl_launch.py +++ b/ycdl_launch.py @@ -4,6 +4,7 @@ gevent.monkey.patch_all() import gevent.pywsgi import gevent.wsgi import sys + import ycdl_site if len(sys.argv) == 2: @@ -25,5 +26,5 @@ else: ) -print('Starting server') +print('Starting server on port %d' % port) http.serve_forever() diff --git a/ycdl_refresh.py b/ycdl_refresh.py new file mode 100644 index 0000000..7574774 --- /dev/null +++ b/ycdl_refresh.py @@ -0,0 +1,2 @@ +import ycdl_easy +ycdl_easy.youtube.refresh_all_channels() diff --git a/ycdl_site.py b/ycdl_site.py index b13b10a..4a3e003 100644 --- a/ycdl_site.py +++ b/ycdl_site.py @@ -128,9 +128,8 @@ def get_channel(channel_id, download_filter=None): channel = youtube.get_channel(channel_id) if channel is None: flask.abort(404) - videos = youtube.get_videos(channel_id=channel_id) - if download_filter is not None: - videos = [video for video in videos if video['download'] == download_filter] + videos = youtube.get_videos(channel_id=channel_id, download_filter=download_filter) + for video in videos: published = video['published'] published = datetime.datetime.utcfromtimestamp(published) @@ -153,10 +152,13 @@ def post_mark_video_state(): state = request.form['state'] try: youtube.mark_video_state(video_id, state) - except KeyError: + + except ycdl.NoSuchVideo: flask.abort(404) - except ValueError: + + except ycdl.InvalidVideoState: flask.abort(400) + return make_json_response({'video_id': video_id, 'state': state}) @site.route('/refresh_all_channels', methods=['POST']) @@ -174,6 +176,13 @@ def post_refresh_channel(): channel_id = channel_id.strip() if not channel_id: flask.abort(400) + if not (len(channel_id) == 24 and channel_id.startswith('UC')): + # It seems they have given us a username instead. + try: + channel_id = youtube.youtube.get_user_id(username=channel_id) + except IndexError: + flask.abort(404) + force = request.form.get('force', False) force = helpers.truthystring(force) youtube.refresh_channel(channel_id, force=force) @@ -184,11 +193,11 @@ def post_start_download(): if 'video_id' not in request.form: flask.abort(400) video_id = request.form['video_id'] - video_info = youtube_core.get_video([video_id]) - if video_info == []: + try: + youtube.download_video(video_id) + except ytapi.VideoNotFound: flask.abort(404) - for video in video_info: - youtube.download_video(video) + return make_json_response({'video_id': video_id, 'state': 'downloaded'}) if __name__ == '__main__': diff --git a/ytapi.py b/ytapi.py index 2598226..3071148 100644 --- a/ytapi.py +++ b/ytapi.py @@ -3,6 +3,9 @@ import datetime import helpers +class VideoNotFound(Exception): + pass + class Video: def __init__(self, snippet): self.id = snippet['id'] @@ -33,6 +36,10 @@ class Youtube: ) self.youtube = youtube + def get_user_id(self, username): + user = self.youtube.channels().list(part='snippet', forUsername=username).execute() + return user['items'][0]['id'] + def get_user_name(self, uid): user = self.youtube.channels().list(part='snippet', id=uid).execute() return user['items'][0]['snippet']['title'] @@ -67,15 +74,18 @@ class Youtube: video_ids = [video_ids] else: singular = False - video_ids = helpers.chunk_sequence(video_ids, 50) + results = [] - for chunk in video_ids: + chunks = helpers.chunk_sequence(video_ids, 50) + for chunk in chunks: chunk = ','.join(chunk) data = self.youtube.videos().list(part='snippet', id=chunk).execute() items = data['items'] results += items - #print('Found %d more, %d total' % (len(items), len(results))) results = [Video(snippet) for snippet in results] - if singular and len(results) == 1: - return results[0] + if singular: + if len(results) == 1: + return results[0] + elif len(results) == 0: + raise VideoNotFound(video_ids[0]) return results diff --git a/ytqueue.py b/ytqueue.py index 5de38cf..f4f0861 100644 --- a/ytqueue.py +++ b/ytqueue.py @@ -18,6 +18,7 @@ while True: for filename in queue: yt_id = filename.split('.')[0] command = YOUTUBE_DL.format(id=yt_id) - os.system(command) - os.remove(filename) + exit_code = os.system(command) + if exit_code == 0: + os.remove(filename) time.sleep(10)