voussoir/ycdl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

checkpoint

Browse source
This commit is contained in:
voussoir 2016-12-06 22:11:09 -08:00
parent bd263b2ed7
commit 90eedca0d7
9 changed files with 138 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
download_thumbnails.py Normal file
View file

@ -0,0 +1,13 @@
import os
import ycdl_easy
from voussoirkit import downloady
DIRECTORY = 'C:\\users\\owner\\youtube thumbnails'
videos = ycdl_easy.youtube.get_videos()
for video in videos:
thumbnail_path = os.path.join(DIRECTORY, video['id']) + '.jpg'
if os.path.exists(thumbnail_path):
continue
result = downloady.download_file(video['thumbnail'], thumbnail_path)
print(result)

26
templates/channel.html
View file

@ -11,9 +11,14 @@
#content_body #content_body
{ {
display: flex; display: flex;
flex-grow: 1;
flex-shrink: 0;
flex-basis: auto;
flex-direction: column; flex-direction: column;
width: 1440px;
margin: auto;
max-width: 100%;
} }
.video_card_downloaded, .video_card_downloaded,
.video_card_ignored, .video_card_ignored,
.video_card_pending .video_card_pending
@ -67,8 +72,10 @@
<body> <body>
{{header.make_header()}} {{header.make_header()}}
<div id="content_body"> <div id="content_body">
<button class="refresh_button" onclick="refresh_channel('{{channel['id']}}', false, function(){location.reload()})">Refresh new videos</button> <button class="refresh_button"
<button class="refresh_button" onclick="refresh_channel('{{channel['id']}}', true, function(){location.reload()})">Refresh everything</button> onclick="refresh_channel('{{channel['id']}}', false, function(){location.reload()})">Refresh new videos</button>
<button class="refresh_button"
onclick="refresh_channel('{{channel['id']}}', true, function(){location.reload()})">Refresh everything</button>
<span><a href="/channel/{{channel['id']}}">All</a></span> <span><a href="/channel/{{channel['id']}}">All</a></span>
<span><a href="/channel/{{channel['id']}}/pending">Pending</a></span> <span><a href="/channel/{{channel['id']}}/pending">Pending</a></span>
<span><a href="/channel/{{channel['id']}}/ignored">Ignored</a></span> <span><a href="/channel/{{channel['id']}}/ignored">Ignored</a></span>
@ -107,7 +114,6 @@
class="video_action_ignore" class="video_action_ignore"
onclick="mark_video_state('{{video['id']}}', 'ignored', receive_action_response);" onclick="mark_video_state('{{video['id']}}', 'ignored', receive_action_response);"
>Ignore</button> >Ignore</button>
{% endif %} {% endif %}
</div> </div>
</div> </div>
@ -208,17 +214,5 @@ function start_download(video_id, callback)
data.append("video_id", video_id); data.append("video_id", video_id);
return post(url, data, callback); return post(url, data, callback);
} }
function toggle_dropdown(dropdown)
{
if (dropdown.style.display != "inline-flex")
{
dropdown.style.display = "inline-flex";
}
else
{
dropdown.style.display = "none";
}
}
</script> </script>
</html> </html>

7
templates/channels.html
View file

@ -11,7 +11,13 @@
#content_body #content_body
{ {
display: flex; display: flex;
flex-grow: 1;
flex-shrink: 0;
flex-basis: auto;
flex-direction: column; flex-direction: column;
width: 1440px;
margin: auto;
max-width: 100%;
} }
#new_channel_textbox, #new_channel_textbox,
#new_channel_button #new_channel_button
@ -85,6 +91,7 @@ function refresh_channel(channel_id, force, callback)
data.append("force", force) data.append("force", force)
return post(url, data, callback); return post(url, data, callback);
} }
function refresh_all_channels(force, callback) function refresh_all_channels(force, callback)
{ {
var url = "/refresh_all_channels"; var url = "/refresh_all_channels";

90
ycdl.py
View file

@ -4,16 +4,15 @@ import sqlite3
import ytapi import ytapi
# AVAILABLE FORMATTERS: def YOUTUBE_DL_COMMAND(video_id):
# url, id path = 'C:\\Incoming\\ytqueue\\{id}.ytqueue'.format(id=video_id)
# Note that if the channel has a value in the `directory` column, the bot will open(path, 'w')
# chdir there before executing.
YOUTUBE_DL_COMMAND = 'touch C:\\Incoming\\ytqueue\\{id}.ytqueue'
logging.basicConfig(level=logging.DEBUG) logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
logging.getLogger('googleapiclient.discovery').setLevel(logging.WARNING) logging.getLogger('googleapiclient.discovery').setLevel(logging.WARNING)
logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.WARNING) logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.WARNING)
logging.getLogger('requests.packages.urllib3.util.retry').setLevel(logging.WARNING)
SQL_CHANNEL_COLUMNS = [ SQL_CHANNEL_COLUMNS = [
'id', 'id',
@ -64,6 +63,7 @@ DEFAULT_DBNAME = 'ycdl.db'
ERROR_DATABASE_OUTOFDATE = 'Database is out-of-date. {current} should be {new}' ERROR_DATABASE_OUTOFDATE = 'Database is out-of-date. {current} should be {new}'
def verify_is_abspath(path): def verify_is_abspath(path):
''' '''
TO DO: Determine whether this is actually correct. TO DO: Determine whether this is actually correct.
@ -71,8 +71,16 @@ def verify_is_abspath(path):
if os.path.abspath(path) != path: if os.path.abspath(path) != path:
raise ValueError('Not an abspath') raise ValueError('Not an abspath')
class InvalidVideoState(Exception):
pass
class NoSuchVideo(Exception):
pass
class YCDL: class YCDL:
def __init__(self, youtube, database_filename=None): def __init__(self, youtube, database_filename=None, youtube_dl_function=None):
self.youtube = youtube self.youtube = youtube
if database_filename is None: if database_filename is None:
database_filename = DEFAULT_DBNAME database_filename = DEFAULT_DBNAME
@ -90,6 +98,11 @@ class YCDL:
print(message) print(message)
raise SystemExit raise SystemExit
if youtube_dl_function:
self.youtube_dl_function = youtube_dl_function
else:
self.youtube_dl_function = YOUTUBE_DL_COMMAND
statements = DB_INIT.split(';') statements = DB_INIT.split(';')
for statement in statements: for statement in statements:
self.cur.execute(statement) self.cur.execute(statement)
@ -134,28 +147,46 @@ class YCDL:
return fetch[SQL_CHANNEL['directory']] return fetch[SQL_CHANNEL['directory']]
def download_video(self, video, force=False): def download_video(self, video, force=False):
if not isinstance(video, ytapi.Video): '''
video = self.youtube.get_video(video) Execute the `YOUTUBE_DL_COMMAND`, within the channel's associated
directory if applicable.
'''
# This logic is a little hazier than I would like, but it's all in the
# interest of minimizing unnecessary API calls.
if isinstance(video, ytapi.Video):
video_id = video.id
else:
video_id = video
self.cur.execute('SELECT * FROM videos WHERE id == ?', [video_id])
video_row = self.cur.fetchone()
if video_row is None:
# Since the video was not in the db, we may not know about the channel either.
if not isinstance(video, ytapi.Video):
print('get video')
video = self.youtube.get_video(video)
channel_id = video.author_id
self.cur.execute('SELECT * FROM channels WHERE id == ?', [channel_id])
if self.cur.fetchone() is None:
print('add channel')
self.add_channel(channel_id, get_videos=False, commit=False)
video_row = self.insert_video(video, commit=False)['row']
else:
channel_id = video_row[SQL_VIDEO['author_id']]
self.add_channel(video.author_id, get_videos=False, commit=False) if video_row[SQL_VIDEO['download']] != 'pending' and not force:
status = self.insert_video(video, commit=True)
if status['row'][SQL_VIDEO['download']] != 'pending' and not force:
print('That video does not need to be downloaded.') print('That video does not need to be downloaded.')
return return
download_directory = self.channel_directory(video.author_id) download_directory = self.channel_directory(channel_id)
download_directory = download_directory or os.getcwd() download_directory = download_directory or os.getcwd()
current_directory = os.getcwd() current_directory = os.getcwd()
os.makedirs(download_directory, exist_ok=True) os.makedirs(download_directory, exist_ok=True)
os.chdir(download_directory) os.chdir(download_directory)
url = 'https://www.youtube.com/watch?v={id}'.format(id=video.id) self.youtube_dl_function(video_id)
command = YOUTUBE_DL_COMMAND.format(url=url, id=video.id)
os.system(command)
os.chdir(current_directory) os.chdir(current_directory)
self.cur.execute('UPDATE videos SET download = "downloaded" WHERE id == ?', [video.id]) self.cur.execute('UPDATE videos SET download = "downloaded" WHERE id == ?', [video_id])
self.sql.commit() self.sql.commit()
def get_channel(self, channel_id): def get_channel(self, channel_id):
@ -173,14 +204,29 @@ class YCDL:
channels.sort(key=lambda x: x['name'].lower()) channels.sort(key=lambda x: x['name'].lower())
return channels return channels
def get_videos(self, channel_id=None): def get_videos(self, channel_id=None, download_filter=None):
wheres = []
bindings = []
if channel_id is not None: if channel_id is not None:
self.cur.execute('SELECT * FROM videos WHERE author_id == ?', [channel_id]) wheres.append('author_id')
bindings.append(channel_id)
if download_filter is not None:
wheres.append('download')
bindings.append(download_filter)
if wheres:
wheres = [x + ' == ?' for x in wheres]
wheres = ' WHERE ' + ' AND '.join(wheres)
else: else:
self.cur.execute('SELECT * FROM videos ') wheres = ''
query = 'SELECT * FROM videos' + wheres
self.cur.execute(query, bindings)
videos = self.cur.fetchall() videos = self.cur.fetchall()
if not videos: if not videos:
return [] return []
videos = [{key: video[SQL_VIDEO[key]] for key in SQL_VIDEO} for video in videos] videos = [{key: video[SQL_VIDEO[key]] for key in SQL_VIDEO} for video in videos]
videos.sort(key=lambda x: x['published'], reverse=True) videos.sort(key=lambda x: x['published'], reverse=True)
return videos return videos
@ -214,10 +260,10 @@ class YCDL:
Mark the video as ignored, pending, or downloaded. Mark the video as ignored, pending, or downloaded.
''' '''
if state not in ['ignored', 'pending', 'downloaded']: if state not in ['ignored', 'pending', 'downloaded']:
raise ValueError(state) raise InvalidVideoState(state)
self.cur.execute('SELECT * FROM videos WHERE id == ?', [video_id]) self.cur.execute('SELECT * FROM videos WHERE id == ?', [video_id])
if self.cur.fetchone() is None: if self.cur.fetchone() is None:
raise KeyError(video_id) raise NoSuchVideo(video_id)
self.cur.execute('UPDATE videos SET download = ? WHERE id == ?', [state, video_id]) self.cur.execute('UPDATE videos SET download = ? WHERE id == ?', [state, video_id])
if commit: if commit:
self.sql.commit() self.sql.commit()

3
ycdl_launch.py
View file

@ -4,6 +4,7 @@ gevent.monkey.patch_all()
import gevent.pywsgi import gevent.pywsgi
import gevent.wsgi import gevent.wsgi
import sys import sys
import ycdl_site import ycdl_site
if len(sys.argv) == 2: if len(sys.argv) == 2:
@ -25,5 +26,5 @@ else:
) )
print('Starting server') print('Starting server on port %d' % port)
http.serve_forever() http.serve_forever()

2
ycdl_refresh.py Normal file
View file

@ -0,0 +1,2 @@
import ycdl_easy
ycdl_easy.youtube.refresh_all_channels()

27
ycdl_site.py
View file

@ -128,9 +128,8 @@ def get_channel(channel_id, download_filter=None):
channel = youtube.get_channel(channel_id) channel = youtube.get_channel(channel_id)
if channel is None: if channel is None:
flask.abort(404) flask.abort(404)
videos = youtube.get_videos(channel_id=channel_id) videos = youtube.get_videos(channel_id=channel_id, download_filter=download_filter)
if download_filter is not None:
videos = [video for video in videos if video['download'] == download_filter]
for video in videos: for video in videos:
published = video['published'] published = video['published']
published = datetime.datetime.utcfromtimestamp(published) published = datetime.datetime.utcfromtimestamp(published)
@ -153,10 +152,13 @@ def post_mark_video_state():
state = request.form['state'] state = request.form['state']
try: try:
youtube.mark_video_state(video_id, state) youtube.mark_video_state(video_id, state)
except KeyError:
except ycdl.NoSuchVideo:
flask.abort(404) flask.abort(404)
except ValueError:
except ycdl.InvalidVideoState:
flask.abort(400) flask.abort(400)
return make_json_response({'video_id': video_id, 'state': state}) return make_json_response({'video_id': video_id, 'state': state})
@site.route('/refresh_all_channels', methods=['POST']) @site.route('/refresh_all_channels', methods=['POST'])
@ -174,6 +176,13 @@ def post_refresh_channel():
channel_id = channel_id.strip() channel_id = channel_id.strip()
if not channel_id: if not channel_id:
flask.abort(400) flask.abort(400)
if not (len(channel_id) == 24 and channel_id.startswith('UC')):
# It seems they have given us a username instead.
try:
channel_id = youtube.youtube.get_user_id(username=channel_id)
except IndexError:
flask.abort(404)
force = request.form.get('force', False) force = request.form.get('force', False)
force = helpers.truthystring(force) force = helpers.truthystring(force)
youtube.refresh_channel(channel_id, force=force) youtube.refresh_channel(channel_id, force=force)
@ -184,11 +193,11 @@ def post_start_download():
if 'video_id' not in request.form: if 'video_id' not in request.form:
flask.abort(400) flask.abort(400)
video_id = request.form['video_id'] video_id = request.form['video_id']
video_info = youtube_core.get_video([video_id]) try:
if video_info == []: youtube.download_video(video_id)
except ytapi.VideoNotFound:
flask.abort(404) flask.abort(404)
for video in video_info:
youtube.download_video(video)
return make_json_response({'video_id': video_id, 'state': 'downloaded'}) return make_json_response({'video_id': video_id, 'state': 'downloaded'})
if __name__ == '__main__': if __name__ == '__main__':

20
ytapi.py
View file

@ -3,6 +3,9 @@ import datetime
import helpers import helpers
class VideoNotFound(Exception):
pass
class Video: class Video:
def __init__(self, snippet): def __init__(self, snippet):
self.id = snippet['id'] self.id = snippet['id']
@ -33,6 +36,10 @@ class Youtube:
) )
self.youtube = youtube self.youtube = youtube
def get_user_id(self, username):
user = self.youtube.channels().list(part='snippet', forUsername=username).execute()
return user['items'][0]['id']
def get_user_name(self, uid): def get_user_name(self, uid):
user = self.youtube.channels().list(part='snippet', id=uid).execute() user = self.youtube.channels().list(part='snippet', id=uid).execute()
return user['items'][0]['snippet']['title'] return user['items'][0]['snippet']['title']
@ -67,15 +74,18 @@ class Youtube:
video_ids = [video_ids] video_ids = [video_ids]
else: else:
singular = False singular = False
video_ids = helpers.chunk_sequence(video_ids, 50)
results = [] results = []
for chunk in video_ids: chunks = helpers.chunk_sequence(video_ids, 50)
for chunk in chunks:
chunk = ','.join(chunk) chunk = ','.join(chunk)
data = self.youtube.videos().list(part='snippet', id=chunk).execute() data = self.youtube.videos().list(part='snippet', id=chunk).execute()
items = data['items'] items = data['items']
results += items results += items
#print('Found %d more, %d total' % (len(items), len(results)))
results = [Video(snippet) for snippet in results] results = [Video(snippet) for snippet in results]
if singular and len(results) == 1: if singular:
return results[0] if len(results) == 1:
return results[0]
elif len(results) == 0:
raise VideoNotFound(video_ids[0])
return results return results

5
ytqueue.py
View file

@ -18,6 +18,7 @@ while True:
for filename in queue: for filename in queue:
yt_id = filename.split('.')[0] yt_id = filename.split('.')[0]
command = YOUTUBE_DL.format(id=yt_id) command = YOUTUBE_DL.format(id=yt_id)
os.system(command) exit_code = os.system(command)
os.remove(filename) if exit_code == 0:
os.remove(filename)
time.sleep(10) time.sleep(10)