voussoir/ycdl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

checkpoint

Browse source
This commit is contained in:
voussoir 2016-12-06 22:11:09 -08:00
parent bd263b2ed7
commit 90eedca0d7
9 changed files with 138 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
download_thumbnails.py Normal file
View file

@ -0,0 +1,13 @@
import os
import ycdl_easy
from voussoirkit import downloady
DIRECTORY = 'C:\\users\\owner\\youtube thumbnails'
videos = ycdl_easy.youtube.get_videos()
for video in videos:
thumbnail_path = os.path.join(DIRECTORY, video['id']) + '.jpg'
if os.path.exists(thumbnail_path):
continue
result = downloady.download_file(video['thumbnail'], thumbnail_path)
print(result)

26
templates/channel.html
View file

@ -11,9 +11,14 @@
#content_body
{
display: flex;
flex-grow: 1;
flex-shrink: 0;
flex-basis: auto;
flex-direction: column;
width: 1440px;
margin: auto;
max-width: 100%;
}
.video_card_downloaded,
.video_card_ignored,
.video_card_pending
@ -67,8 +72,10 @@
<body>
{{header.make_header()}}
<div id="content_body">
<button class="refresh_button" onclick="refresh_channel('{{channel['id']}}', false, function(){location.reload()})">Refresh new videos</button>
<button class="refresh_button" onclick="refresh_channel('{{channel['id']}}', true, function(){location.reload()})">Refresh everything</button>
<button class="refresh_button"
onclick="refresh_channel('{{channel['id']}}', false, function(){location.reload()})">Refresh new videos</button>
<button class="refresh_button"
onclick="refresh_channel('{{channel['id']}}', true, function(){location.reload()})">Refresh everything</button>
<span><a href="/channel/{{channel['id']}}">All</a></span>
<span><a href="/channel/{{channel['id']}}/pending">Pending</a></span>
<span><a href="/channel/{{channel['id']}}/ignored">Ignored</a></span>
@ -107,7 +114,6 @@
class="video_action_ignore"
onclick="mark_video_state('{{video['id']}}', 'ignored', receive_action_response);"
>Ignore</button>
{% endif %}
</div>
</div>
@ -208,17 +214,5 @@ function start_download(video_id, callback)
data.append("video_id", video_id);
return post(url, data, callback);
}
function toggle_dropdown(dropdown)
{
if (dropdown.style.display != "inline-flex")
{
dropdown.style.display = "inline-flex";
}
else
{
dropdown.style.display = "none";
}
}
</script>
</html>

7
templates/channels.html
View file

@ -11,7 +11,13 @@
#content_body
{
display: flex;
flex-grow: 1;
flex-shrink: 0;
flex-basis: auto;
flex-direction: column;
width: 1440px;
margin: auto;
max-width: 100%;
}
#new_channel_textbox,
#new_channel_button
@ -85,6 +91,7 @@ function refresh_channel(channel_id, force, callback)
data.append("force", force)
return post(url, data, callback);
}
function refresh_all_channels(force, callback)
{
var url = "/refresh_all_channels";

86
ycdl.py
View file

@ -4,16 +4,15 @@ import sqlite3
import ytapi
# AVAILABLE FORMATTERS:
# url, id
# Note that if the channel has a value in the `directory` column, the bot will
# chdir there before executing.
YOUTUBE_DL_COMMAND = 'touch C:\\Incoming\\ytqueue\\{id}.ytqueue'
def YOUTUBE_DL_COMMAND(video_id):
path = 'C:\\Incoming\\ytqueue\\{id}.ytqueue'.format(id=video_id)
open(path, 'w')
logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger(__name__)
logging.getLogger('googleapiclient.discovery').setLevel(logging.WARNING)
logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.WARNING)
logging.getLogger('requests.packages.urllib3.util.retry').setLevel(logging.WARNING)
SQL_CHANNEL_COLUMNS = [
'id',
@ -64,6 +63,7 @@ DEFAULT_DBNAME = 'ycdl.db'
ERROR_DATABASE_OUTOFDATE = 'Database is out-of-date. {current} should be {new}'
def verify_is_abspath(path):
'''
TO DO: Determine whether this is actually correct.
@ -71,8 +71,16 @@ def verify_is_abspath(path):
if os.path.abspath(path) != path:
raise ValueError('Not an abspath')
class InvalidVideoState(Exception):
pass
class NoSuchVideo(Exception):
pass
class YCDL:
def __init__(self, youtube, database_filename=None):
def __init__(self, youtube, database_filename=None, youtube_dl_function=None):
self.youtube = youtube
if database_filename is None:
database_filename = DEFAULT_DBNAME
@ -90,6 +98,11 @@ class YCDL:
print(message)
raise SystemExit
if youtube_dl_function:
self.youtube_dl_function = youtube_dl_function
else:
self.youtube_dl_function = YOUTUBE_DL_COMMAND
statements = DB_INIT.split(';')
for statement in statements:
self.cur.execute(statement)
@ -134,28 +147,46 @@ class YCDL:
return fetch[SQL_CHANNEL['directory']]
def download_video(self, video, force=False):
'''
Execute the `YOUTUBE_DL_COMMAND`, within the channel's associated
directory if applicable.
'''
# This logic is a little hazier than I would like, but it's all in the
# interest of minimizing unnecessary API calls.
if isinstance(video, ytapi.Video):
video_id = video.id
else:
video_id = video
self.cur.execute('SELECT * FROM videos WHERE id == ?', [video_id])
video_row = self.cur.fetchone()
if video_row is None:
# Since the video was not in the db, we may not know about the channel either.
if not isinstance(video, ytapi.Video):
print('get video')
video = self.youtube.get_video(video)
channel_id = video.author_id
self.cur.execute('SELECT * FROM channels WHERE id == ?', [channel_id])
if self.cur.fetchone() is None:
print('add channel')
self.add_channel(channel_id, get_videos=False, commit=False)
video_row = self.insert_video(video, commit=False)['row']
else:
channel_id = video_row[SQL_VIDEO['author_id']]
self.add_channel(video.author_id, get_videos=False, commit=False)
status = self.insert_video(video, commit=True)
if status['row'][SQL_VIDEO['download']] != 'pending' and not force:
if video_row[SQL_VIDEO['download']] != 'pending' and not force:
print('That video does not need to be downloaded.')
return
download_directory = self.channel_directory(video.author_id)
download_directory = self.channel_directory(channel_id)
download_directory = download_directory or os.getcwd()
current_directory = os.getcwd()
os.makedirs(download_directory, exist_ok=True)
os.chdir(download_directory)
url = 'https://www.youtube.com/watch?v={id}'.format(id=video.id)
command = YOUTUBE_DL_COMMAND.format(url=url, id=video.id)
os.system(command)
self.youtube_dl_function(video_id)
os.chdir(current_directory)
self.cur.execute('UPDATE videos SET download = "downloaded" WHERE id == ?', [video.id])
self.cur.execute('UPDATE videos SET download = "downloaded" WHERE id == ?', [video_id])
self.sql.commit()
def get_channel(self, channel_id):
@ -173,14 +204,29 @@ class YCDL:
channels.sort(key=lambda x: x['name'].lower())
return channels
def get_videos(self, channel_id=None):
def get_videos(self, channel_id=None, download_filter=None):
wheres = []
bindings = []
if channel_id is not None:
self.cur.execute('SELECT * FROM videos WHERE author_id == ?', [channel_id])
wheres.append('author_id')
bindings.append(channel_id)
if download_filter is not None:
wheres.append('download')
bindings.append(download_filter)
if wheres:
wheres = [x + ' == ?' for x in wheres]
wheres = ' WHERE ' + ' AND '.join(wheres)
else:
self.cur.execute('SELECT * FROM videos ')
wheres = ''
query = 'SELECT * FROM videos' + wheres
self.cur.execute(query, bindings)
videos = self.cur.fetchall()
if not videos:
return []
videos = [{key: video[SQL_VIDEO[key]] for key in SQL_VIDEO} for video in videos]
videos.sort(key=lambda x: x['published'], reverse=True)
return videos
@ -214,10 +260,10 @@ class YCDL:
Mark the video as ignored, pending, or downloaded.
'''
if state not in ['ignored', 'pending', 'downloaded']:
raise ValueError(state)
raise InvalidVideoState(state)
self.cur.execute('SELECT * FROM videos WHERE id == ?', [video_id])
if self.cur.fetchone() is None:
raise KeyError(video_id)
raise NoSuchVideo(video_id)
self.cur.execute('UPDATE videos SET download = ? WHERE id == ?', [state, video_id])
if commit:
self.sql.commit()

3
ycdl_launch.py
View file

@ -4,6 +4,7 @@ gevent.monkey.patch_all()
import gevent.pywsgi
import gevent.wsgi
import sys
import ycdl_site
if len(sys.argv) == 2:
@ -25,5 +26,5 @@ else:
)
print('Starting server')
print('Starting server on port %d' % port)
http.serve_forever()

2
ycdl_refresh.py Normal file
View file

@ -0,0 +1,2 @@
import ycdl_easy
ycdl_easy.youtube.refresh_all_channels()

27
ycdl_site.py
View file

@ -128,9 +128,8 @@ def get_channel(channel_id, download_filter=None):
channel = youtube.get_channel(channel_id)
if channel is None:
flask.abort(404)
videos = youtube.get_videos(channel_id=channel_id)
if download_filter is not None:
videos = [video for video in videos if video['download'] == download_filter]
videos = youtube.get_videos(channel_id=channel_id, download_filter=download_filter)
for video in videos:
published = video['published']
published = datetime.datetime.utcfromtimestamp(published)
@ -153,10 +152,13 @@ def post_mark_video_state():
state = request.form['state']
try:
youtube.mark_video_state(video_id, state)
except KeyError:
except ycdl.NoSuchVideo:
flask.abort(404)
except ValueError:
except ycdl.InvalidVideoState:
flask.abort(400)
return make_json_response({'video_id': video_id, 'state': state})
@site.route('/refresh_all_channels', methods=['POST'])
@ -174,6 +176,13 @@ def post_refresh_channel():
channel_id = channel_id.strip()
if not channel_id:
flask.abort(400)
if not (len(channel_id) == 24 and channel_id.startswith('UC')):
# It seems they have given us a username instead.
try:
channel_id = youtube.youtube.get_user_id(username=channel_id)
except IndexError:
flask.abort(404)
force = request.form.get('force', False)
force = helpers.truthystring(force)
youtube.refresh_channel(channel_id, force=force)
@ -184,11 +193,11 @@ def post_start_download():
if 'video_id' not in request.form:
flask.abort(400)
video_id = request.form['video_id']
video_info = youtube_core.get_video([video_id])
if video_info == []:
try:
youtube.download_video(video_id)
except ytapi.VideoNotFound:
flask.abort(404)
for video in video_info:
youtube.download_video(video)
return make_json_response({'video_id': video_id, 'state': 'downloaded'})
if __name__ == '__main__':

18
ytapi.py
View file

@ -3,6 +3,9 @@ import datetime
import helpers
class VideoNotFound(Exception):
pass
class Video:
def __init__(self, snippet):
self.id = snippet['id']
@ -33,6 +36,10 @@ class Youtube:
)
self.youtube = youtube
def get_user_id(self, username):
user = self.youtube.channels().list(part='snippet', forUsername=username).execute()
return user['items'][0]['id']
def get_user_name(self, uid):
user = self.youtube.channels().list(part='snippet', id=uid).execute()
return user['items'][0]['snippet']['title']
@ -67,15 +74,18 @@ class Youtube:
video_ids = [video_ids]
else:
singular = False
video_ids = helpers.chunk_sequence(video_ids, 50)
results = []
for chunk in video_ids:
chunks = helpers.chunk_sequence(video_ids, 50)
for chunk in chunks:
chunk = ','.join(chunk)
data = self.youtube.videos().list(part='snippet', id=chunk).execute()
items = data['items']
results += items
#print('Found %d more, %d total' % (len(items), len(results)))
results = [Video(snippet) for snippet in results]
if singular and len(results) == 1:
if singular:
if len(results) == 1:
return results[0]
elif len(results) == 0:
raise VideoNotFound(video_ids[0])
return results

3
ytqueue.py
View file

@ -18,6 +18,7 @@ while True:
for filename in queue:
yt_id = filename.split('.')[0]
command = YOUTUBE_DL.format(id=yt_id)
os.system(command)
exit_code = os.system(command)
if exit_code == 0:
os.remove(filename)
time.sleep(10)