voussoir/ycdl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Changes

Browse source 
- Add thumbnails to the card (dynamically loaded)
- Add button for embedding the video
- Store video duration in db (use update script)
- Separate links for filter views with and without query.
This commit is contained in:
voussoir 2018-12-17 19:17:53 -08:00
parent ba1961349c
commit f1f12423b1
10 changed files with 483 additions and 332 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
frontends/ycdl_flask/templates/channel.html
View file

@ -77,15 +77,15 @@
onclick="refresh_channel('{{channel['id']}}', false, function(){location.reload()})">Refresh new videos</button></span>
<span><button class="refresh_button"
onclick="refresh_channel('{{channel['id']}}', true, function(){location.reload()})">Refresh everything</button></span>
<span><a href="/channel/{{channel['id']}}{{query_string}}">All</a></span>
<span><a href="/channel/{{channel['id']}}/pending{{query_string}}">Pending</a></span>
<span><a href="/channel/{{channel['id']}}/ignored{{query_string}}">Ignored</a></span>
<span><a href="/channel/{{channel['id']}}/downloaded{{query_string}}">Downloaded</a></span>
<span><a href="/channel/{{channel['id']}}">All</a> <a href="/channel/{{channel['id']}}{{query_string}}">(?q)</a></span>
<span><a href="/channel/{{channel['id']}}/pending">Pending</a> <a href="/channel/{{channel['id']}}/pending{{query_string}}">(?q)</a></span>
<span><a href="/channel/{{channel['id']}}/ignored">Ignored</a> <a href="/channel/{{channel['id']}}/ignored{{query_string}}">(?q)</a></span>
<span><a href="/channel/{{channel['id']}}/downloaded">Downloaded</a> <a href="/channel/{{channel['id']}}/downloaded{{query_string}}">(?q)</a></span>
{% else %}
<span><a href="/videos{{query_string}}">All</a></span>
<span><a href="/videos/pending{{query_string}}">Pending</a></span>
<span><a href="/videos/ignored{{query_string}}">Ignored</a></span>
<span><a href="/videos/downloaded{{query_string}}">Downloaded</a></span>
<span><a href="/videos">All</a> <a href="/videos{{query_string}}">(?q)</a></span>
<span><a href="/videos/pending">Pending</a> <a href="/videos/pending{{query_string}}">(?q)</a></span>
<span><a href="/videos/ignored">Ignored</a> <a href="/videos/ignored{{query_string}}">(?q)</a></span>
<span><a href="/videos/downloaded">Downloaded</a> <a href="/videos/downloaded{{query_string}}">(?q)</a></span>
{% endif %}
<span>{{videos|length}} items</span>
@ -94,15 +94,11 @@
<div id="video_card_{{video['id']}}"
data-ytid="{{video['id']}}"
onclick="onclick_select(event)"
{% if video['download'] == "downloaded" %}
class="video_card video_card_downloaded"
{% elif video['download'] == "ignored" %}
class="video_card video_card_ignored"
{% else %}
class="video_card video_card_pending"
{% endif %}
class="video_card video_card_{{video['download']}}"
>
<img src="http://i3.ytimg.com/vi/{{video['id']}}/default.jpg" height="100px">
<a href="https://www.youtube.com/watch?v={{video['id']}}">{{video['_published_str']}} - {{video['title']}}</a>
<span>({{video['duration'] | seconds_to_hms}})</span>
{% if channel is none %}
<a href="/channel/{{video['author_id']}}">(Chan)</a>
{% endif %}
@ -134,6 +130,10 @@
onclick="action_button_passthrough(event, mark_video_state, 'ignored')"
>Ignore</button>
</div>
<br>
<button class="show_embed_button" onclick="toggle_embed_video('{{video.id}}');">Embed</button>
<button class="hide_embed_button hidden" onclick="toggle_embed_video('{{video.id}}');">Unembed</button>
<br>
</div>
{% endfor %}
</div>
@ -147,6 +147,28 @@ var video_card_first_selected = null;
var video_card_selections = [];
var video_cards = Array.from(document.getElementById("video_cards").children);
function toggle_embed_video(video_id)
{
var video_card = document.getElementById("video_card_" + video_id);
var show_button = video_card.getElementsByClassName("show_embed_button")[0];
var hide_button = video_card.getElementsByClassName("hide_embed_button")[0];
var embeds = video_card.getElementsByTagName("iframe");
if (embeds.length == 0)
{
var html = `<iframe width="711" height="400" src="https://www.youtube.com/embed/${video_id}" frameborder="0" allow="encrypted-media" allowfullscreen></iframe>`
var embed = html_to_element(html);
video_card.appendChild(embed);
show_button.classList.add("hidden");
hide_button.classList.remove("hidden");
}
else
{
video_card.removeChild(embeds[0]);
show_button.classList.remove("hidden");
hide_button.classList.add("hidden");
}
}
function deselect_all()
{
var video_card_first_selected = null;
@ -157,6 +179,13 @@ function deselect_all()
video_card_selections = [];
}
function html_to_element(html)
{
var template = document.createElement("template");
template.innerHTML = html;
return template.content.firstChild;
}
function onclick_select(event)
{
if (!event.target.classList.contains("video_card"))

21
frontends/ycdl_flask/ycdl_flask/jinja_filters.py Normal file
View file

@ -0,0 +1,21 @@
import math
def seconds_to_hms(seconds):
'''
Convert integer number of seconds to an hh:mm:ss string.
Only the necessary fields are used.
'''
if seconds is None:
return '???'
seconds = math.ceil(seconds)
(minutes, seconds) = divmod(seconds, 60)
(hours, minutes) = divmod(minutes, 60)
parts = []
if hours:
parts.append(hours)
if minutes:
parts.append(minutes)
parts.append(seconds)
hms = ':'.join(f'{part:02d}' for part in parts)
return hms

4
frontends/ycdl_flask/ycdl_flask/ycdl_flask.py
View file

@ -18,6 +18,8 @@ import ycdl
from voussoirkit import pathclass
from . import jinja_filters
root_dir = pathclass.Path(__file__).parent.parent
TEMPLATE_DIR = root_dir.with_child('templates')
@ -37,6 +39,7 @@ site.config.update(
TEMPLATES_AUTO_RELOAD=True,
)
site.jinja_env.add_extension('jinja2.ext.do')
site.jinja_env.filters['seconds_to_hms'] = jinja_filters.seconds_to_hms
site.debug = True
####################################################################################################
@ -192,6 +195,7 @@ def post_mark_video_state():
youtube.mark_video_state(video_id, state)
except ycdl.NoSuchVideo:
traceback.print_exc()
flask.abort(404)
except ycdl.InvalidVideoState:

1
frontends/ycdl_flask/ycdl_flask_launch.py
View file

@ -5,7 +5,6 @@ import gevent.monkey
gevent.monkey.patch_all()
import gevent.pywsgi
import gevent.wsgi
import sys
import ycdl_flask

1
requirements.txt
View file

@ -1,3 +1,4 @@
flask
gevent
https://github.com/voussoir/else/raw/master/_voussoirkit/voussoirkit.zip
isodate

80
utilities/database_upgrader.py Normal file
View file

@ -0,0 +1,80 @@
import argparse
import os
import sqlite3
import sys
import ycdl
def upgrade_1_to_2(sql):
'''
In this version, a column `tagged_at` was added to the Photos table, to keep
track of the last time the photo's tags were edited (added or removed).
'''
cur = sql.cursor()
cur.executescript('''
ALTER TABLE videos RENAME TO videos_old;
CREATE TABLE videos(
id TEXT,
published INT,
author_id TEXT,
title TEXT,
description TEXT,
duration INT,
thumbnail TEXT,
download TEXT
);
INSERT INTO videos SELECT
id,
published,
author_id,
title,
description,
NULL,
thumbnail,
download
FROM videos_old;
DROP TABLE videos_old;
''')
def upgrade_all(database_filepath):
'''
Given the directory containing a phototagger database, apply all of the
needed upgrade_x_to_y functions in order.
'''
sql = sqlite3.connect(database_filepath)
cur = sql.cursor()
cur.execute('PRAGMA user_version')
current_version = cur.fetchone()[0]
needed_version = ycdl.ycdl.DATABASE_VERSION
if current_version == needed_version:
print('Already up to date with version %d.' % needed_version)
return
for version_number in range(current_version + 1, needed_version + 1):
print('Upgrading from %d to %d.' % (current_version, version_number))
upgrade_function = 'upgrade_%d_to_%d' % (current_version, version_number)
upgrade_function = eval(upgrade_function)
upgrade_function(sql)
sql.cursor().execute('PRAGMA user_version = %d' % version_number)
sql.commit()
current_version = version_number
print('Upgrades finished.')
def upgrade_all_argparse(args):
return upgrade_all(database_filepath=args.database_filepath)
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument('database_filepath')
parser.set_defaults(func=upgrade_all_argparse)
args = parser.parse_args(argv)
args.func(args)
if __name__ == '__main__':
raise SystemExit(main(sys.argv[1:]))

9
utilities/download_thumbnails.py
View file

@ -1,12 +1,15 @@
import bot
import logging
logging.basicConfig(level=logging.WARNING)
import bot3 as bot
import os
import traceback
import ycdl
import ytapi
import ycdl_repl
from voussoirkit import downloady
youtube_core = ytapi.Youtube(bot.YOUTUBE_KEY)
youtube_core = ycdl.ytapi.Youtube(bot.YOUTUBE_KEY)
youtube = ycdl.YCDL(youtube_core)
DIRECTORY = '.\\youtube thumbnails'

295
ycdl/__init__.py
View file

@ -1,296 +1,5 @@
import logging
import os
import sqlite3
from . import helpers
from . import ycdl
from . import ytapi
def YOUTUBE_DL_COMMAND(video_id):
path = 'D:\\Incoming\\ytqueue\\{id}.ytqueue'.format(id=video_id)
open(path, 'w')
logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger(__name__)
logging.getLogger('googleapiclient.discovery').setLevel(logging.WARNING)
logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.WARNING)
logging.getLogger('requests.packages.urllib3.util.retry').setLevel(logging.WARNING)
SQL_CHANNEL_COLUMNS = [
'id',
'name',
'directory',
]
SQL_VIDEO_COLUMNS = [
'id',
'published',
'author_id',
'title',
'description',
'thumbnail',
'download',
]
SQL_CHANNEL = {key:index for (index, key) in enumerate(SQL_CHANNEL_COLUMNS)}
SQL_VIDEO = {key:index for (index, key) in enumerate(SQL_VIDEO_COLUMNS)}
DATABASE_VERSION = 1
DB_INIT = '''
PRAGMA count_changes = OFF;
PRAGMA cache_size = 10000;
PRAGMA user_version = {user_version};
CREATE TABLE IF NOT EXISTS channels(
id TEXT,
name TEXT,
directory TEXT COLLATE NOCASE
);
CREATE TABLE IF NOT EXISTS videos(
id TEXT,
published INT,
author_id TEXT,
title TEXT,
description TEXT,
thumbnail TEXT,
download TEXT
);
CREATE INDEX IF NOT EXISTS index_channel_id on channels(id);
CREATE INDEX IF NOT EXISTS index_video_id on videos(id);
CREATE INDEX IF NOT EXISTS index_video_published on videos(published);
CREATE INDEX IF NOT EXISTS index_video_download on videos(download);
'''.format(user_version=DATABASE_VERSION)
DEFAULT_DBNAME = 'ycdl.db'
ERROR_DATABASE_OUTOFDATE = 'Database is out-of-date. {current} should be {new}'
def verify_is_abspath(path):
'''
TO DO: Determine whether this is actually correct.
'''
if os.path.abspath(path) != path:
raise ValueError('Not an abspath')
class InvalidVideoState(Exception):
pass
class NoSuchVideo(Exception):
pass
class YCDL:
def __init__(self, youtube, database_filename=None, youtube_dl_function=None):
self.youtube = youtube
if database_filename is None:
database_filename = DEFAULT_DBNAME
existing_database = os.path.exists(database_filename)
self.sql = sqlite3.connect(database_filename)
self.cur = self.sql.cursor()
if existing_database:
self.cur.execute('PRAGMA user_version')
existing_version = self.cur.fetchone()[0]
if existing_version != DATABASE_VERSION:
message = ERROR_DATABASE_OUTOFDATE
message = message.format(current=existing_version, new=DATABASE_VERSION)
print(message)
raise SystemExit
if youtube_dl_function:
self.youtube_dl_function = youtube_dl_function
else:
self.youtube_dl_function = YOUTUBE_DL_COMMAND
statements = DB_INIT.split(';')
for statement in statements:
self.cur.execute(statement)
self.sql.commit()
def add_channel(
self,
channel_id,
commit=True,
download_directory=None,
get_videos=False,
name=None,
):
if self.get_channel(channel_id) is not None:
return
if name is None:
name = self.youtube.get_user_name(channel_id)
data = [None] * len(SQL_CHANNEL)
data[SQL_CHANNEL['id']] = channel_id
data[SQL_CHANNEL['name']] = name
if download_directory is not None:
verify_is_abspath(download_directory)
data[SQL_CHANNEL['directory']] = download_directory
self.cur.execute('INSERT INTO channels VALUES(?, ?, ?)', data)
if get_videos:
self.refresh_channel(channel_id, commit=False)
if commit:
self.sql.commit()
def channel_has_pending(self, channel_id):
query = 'SELECT * FROM videos WHERE author_id == ? AND download == "pending"'
self.cur.execute(query, [channel_id])
return self.cur.fetchone() is not None
def channel_directory(self, channel_id):
self.cur.execute('SELECT * FROM channels WHERE id == ?', [channel_id])
fetch = self.cur.fetchone()
if fetch is None:
return None
return fetch[SQL_CHANNEL['directory']]
def download_video(self, video, force=False):
'''
Execute the `YOUTUBE_DL_COMMAND`, within the channel's associated
directory if applicable.
'''
# This logic is a little hazier than I would like, but it's all in the
# interest of minimizing unnecessary API calls.
if isinstance(video, ytapi.Video):
video_id = video.id
else:
video_id = video
self.cur.execute('SELECT * FROM videos WHERE id == ?', [video_id])
video_row = self.cur.fetchone()
if video_row is None:
# Since the video was not in the db, we may not know about the channel either.
if not isinstance(video, ytapi.Video):
print('get video')
video = self.youtube.get_video(video)
channel_id = video.author_id
self.cur.execute('SELECT * FROM channels WHERE id == ?', [channel_id])
if self.cur.fetchone() is None:
print('add channel')
self.add_channel(channel_id, get_videos=False, commit=False)
video_row = self.insert_video(video, commit=False)['row']
else:
channel_id = video_row[SQL_VIDEO['author_id']]
if video_row[SQL_VIDEO['download']] != 'pending' and not force:
print('That video does not need to be downloaded.')
return
current_directory = os.getcwd()
download_directory = self.channel_directory(channel_id)
download_directory = download_directory or current_directory
os.makedirs(download_directory, exist_ok=True)
os.chdir(download_directory)
self.youtube_dl_function(video_id)
os.chdir(current_directory)
self.cur.execute('UPDATE videos SET download = "downloaded" WHERE id == ?', [video_id])
self.sql.commit()
def get_channel(self, channel_id):
self.cur.execute('SELECT * FROM channels WHERE id == ?', [channel_id])
fetch = self.cur.fetchone()
if not fetch:
return None
fetch = {key: fetch[SQL_CHANNEL[key]] for key in SQL_CHANNEL}
return fetch
def get_channels(self):
self.cur.execute('SELECT * FROM channels')
channels = self.cur.fetchall()
channels = [{key: channel[SQL_CHANNEL[key]] for key in SQL_CHANNEL} for channel in channels]
channels.sort(key=lambda x: x['name'].lower())
return channels
def get_video(self, video_id):
self.cur.execute('SELECT * FROM videos WHERE id == ?', [video_id])
video = self.cur.fetchone()
video = {key: video[SQL_VIDEO[key]] for key in SQL_VIDEO}
return video
def get_videos(self, channel_id=None, download_filter=None):
wheres = []
bindings = []
if channel_id is not None:
wheres.append('author_id')
bindings.append(channel_id)
if download_filter is not None:
wheres.append('download')
bindings.append(download_filter)
if wheres:
wheres = [x + ' == ?' for x in wheres]
wheres = ' WHERE ' + ' AND '.join(wheres)
else:
wheres = ''
query = 'SELECT * FROM videos' + wheres
self.cur.execute(query, bindings)
videos = self.cur.fetchall()
if not videos:
return []
videos = [{key: video[SQL_VIDEO[key]] for key in SQL_VIDEO} for video in videos]
videos.sort(key=lambda x: x['published'], reverse=True)
return videos
def insert_video(self, video, *, add_channel=True, commit=True):
if not isinstance(video, ytapi.Video):
video = self.youtube.get_video(video)
if add_channel:
self.add_channel(video.author_id, get_videos=False, commit=False)
self.cur.execute('SELECT * FROM videos WHERE id == ?', [video.id])
fetch = self.cur.fetchone()
if fetch is not None:
return {'new': False, 'row': fetch}
data = [None] * len(SQL_VIDEO)
data[SQL_VIDEO['id']] = video.id
data[SQL_VIDEO['published']] = video.published
data[SQL_VIDEO['author_id']] = video.author_id
data[SQL_VIDEO['title']] = video.title
data[SQL_VIDEO['description']] = video.description
data[SQL_VIDEO['thumbnail']] = video.thumbnail['url']
data[SQL_VIDEO['download']] = 'pending'
self.cur.execute('INSERT INTO videos VALUES(?, ?, ?, ?, ?, ?, ?)', data)
if commit:
self.sql.commit()
return {'new': True, 'row': data}
def mark_video_state(self, video_id, state, commit=True):
'''
Mark the video as ignored, pending, or downloaded.
'''
if state not in ['ignored', 'pending', 'downloaded']:
raise InvalidVideoState(state)
self.cur.execute('SELECT * FROM videos WHERE id == ?', [video_id])
if self.cur.fetchone() is None:
raise NoSuchVideo(video_id)
self.cur.execute('UPDATE videos SET download = ? WHERE id == ?', [state, video_id])
if commit:
self.sql.commit()
def refresh_all_channels(self, force=False, commit=True):
for channel in self.get_channels():
self.refresh_channel(channel['id'], force=force, commit=commit)
if commit:
self.sql.commit()
def refresh_channel(self, channel_id, force=False, commit=True):
video_generator = self.youtube.get_user_videos(uid=channel_id)
log.debug('Refreshing channel: %s', channel_id)
for video in video_generator:
status = self.insert_video(video, commit=False)
if not force and not status['new']:
break
if commit:
self.sql.commit()
YCDL = ycdl.YCDL

300
ycdl/ycdl.py Normal file
View file

@ -0,0 +1,300 @@
import logging
import os
import sqlite3
from . import helpers
from . import ytapi
def YOUTUBE_DL_COMMAND(video_id):
path = 'D:\\Incoming\\ytqueue\\{id}.ytqueue'.format(id=video_id)
open(path, 'w')
logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger(__name__)
logging.getLogger('googleapiclient.discovery').setLevel(logging.WARNING)
logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.WARNING)
logging.getLogger('requests.packages.urllib3.util.retry').setLevel(logging.WARNING)
SQL_CHANNEL_COLUMNS = [
'id',
'name',
'directory',
]
SQL_VIDEO_COLUMNS = [
'id',
'published',
'author_id',
'title',
'description',
'duration',
'thumbnail',
'download',
]
SQL_CHANNEL = {key:index for (index, key) in enumerate(SQL_CHANNEL_COLUMNS)}
SQL_VIDEO = {key:index for (index, key) in enumerate(SQL_VIDEO_COLUMNS)}
DATABASE_VERSION = 2
DB_INIT = '''
PRAGMA count_changes = OFF;
PRAGMA cache_size = 10000;
PRAGMA user_version = {user_version};
CREATE TABLE IF NOT EXISTS channels(
id TEXT,
name TEXT,
directory TEXT COLLATE NOCASE
);
CREATE TABLE IF NOT EXISTS videos(
id TEXT,
published INT,
author_id TEXT,
title TEXT,
description TEXT,
duration INT,
thumbnail TEXT,
download TEXT
);
CREATE INDEX IF NOT EXISTS index_channel_id on channels(id);
CREATE INDEX IF NOT EXISTS index_video_author on videos(author_id);
CREATE INDEX IF NOT EXISTS index_video_id on videos(id);
CREATE INDEX IF NOT EXISTS index_video_published on videos(published);
CREATE INDEX IF NOT EXISTS index_video_download on videos(download);
'''.format(user_version=DATABASE_VERSION)
DEFAULT_DBNAME = 'ycdl.db'
ERROR_DATABASE_OUTOFDATE = 'Database is out-of-date. {current} should be {new}'
def verify_is_abspath(path):
'''
TO DO: Determine whether this is actually correct.
'''
if os.path.abspath(path) != path:
raise ValueError('Not an abspath')
class InvalidVideoState(Exception):
pass
class NoSuchVideo(Exception):
pass
class YCDL:
def __init__(self, youtube, database_filename=None, youtube_dl_function=None):
self.youtube = youtube
if database_filename is None:
database_filename = DEFAULT_DBNAME
existing_database = os.path.exists(database_filename)
self.sql = sqlite3.connect(database_filename)
self.cur = self.sql.cursor()
if existing_database:
self.cur.execute('PRAGMA user_version')
existing_version = self.cur.fetchone()[0]
if existing_version != DATABASE_VERSION:
message = ERROR_DATABASE_OUTOFDATE
message = message.format(current=existing_version, new=DATABASE_VERSION)
print(message)
raise SystemExit
if youtube_dl_function:
self.youtube_dl_function = youtube_dl_function
else:
self.youtube_dl_function = YOUTUBE_DL_COMMAND
statements = DB_INIT.split(';')
for statement in statements:
self.cur.execute(statement)
self.sql.commit()
def add_channel(
self,
channel_id,
commit=True,
download_directory=None,
get_videos=False,
name=None,
):
if self.get_channel(channel_id) is not None:
return
if name is None:
name = self.youtube.get_user_name(channel_id)
data = [None] * len(SQL_CHANNEL)
data[SQL_CHANNEL['id']] = channel_id
data[SQL_CHANNEL['name']] = name
if download_directory is not None:
verify_is_abspath(download_directory)
data[SQL_CHANNEL['directory']] = download_directory
self.cur.execute('INSERT INTO channels VALUES(?, ?, ?)', data)
if get_videos:
self.refresh_channel(channel_id, commit=False)
if commit:
self.sql.commit()
def channel_has_pending(self, channel_id):
query = 'SELECT * FROM videos WHERE author_id == ? AND download == "pending"'
self.cur.execute(query, [channel_id])
return self.cur.fetchone() is not None
def channel_directory(self, channel_id):
self.cur.execute('SELECT * FROM channels WHERE id == ?', [channel_id])
fetch = self.cur.fetchone()
if fetch is None:
return None
return fetch[SQL_CHANNEL['directory']]
def download_video(self, video, force=False):
'''
Execute the `YOUTUBE_DL_COMMAND`, within the channel's associated
directory if applicable.
'''
# This logic is a little hazier than I would like, but it's all in the
# interest of minimizing unnecessary API calls.
if isinstance(video, ytapi.Video):
video_id = video.id
else:
video_id = video
self.cur.execute('SELECT * FROM videos WHERE id == ?', [video_id])
video_row = self.cur.fetchone()
if video_row is None:
# Since the video was not in the db, we may not know about the channel either.
if not isinstance(video, ytapi.Video):
print('get video')
video = self.youtube.get_video(video)
channel_id = video.author_id
self.cur.execute('SELECT * FROM channels WHERE id == ?', [channel_id])
if self.cur.fetchone() is None:
print('add channel')
self.add_channel(channel_id, get_videos=False, commit=False)
video_row = self.insert_video(video, commit=False)['row']
else:
channel_id = video_row[SQL_VIDEO['author_id']]
if video_row[SQL_VIDEO['download']] != 'pending' and not force:
print('That video does not need to be downloaded.')
return
current_directory = os.getcwd()
download_directory = self.channel_directory(channel_id)
download_directory = download_directory or current_directory
os.makedirs(download_directory, exist_ok=True)
os.chdir(download_directory)
self.youtube_dl_function(video_id)
os.chdir(current_directory)
self.cur.execute('UPDATE videos SET download = "downloaded" WHERE id == ?', [video_id])
self.sql.commit()
def get_channel(self, channel_id):
self.cur.execute('SELECT * FROM channels WHERE id == ?', [channel_id])
fetch = self.cur.fetchone()
if not fetch:
return None
fetch = {key: fetch[SQL_CHANNEL[key]] for key in SQL_CHANNEL}
return fetch
def get_channels(self):
self.cur.execute('SELECT * FROM channels')
channels = self.cur.fetchall()
channels = [{key: channel[SQL_CHANNEL[key]] for key in SQL_CHANNEL} for channel in channels]
channels.sort(key=lambda x: x['name'].lower())
return channels
def get_video(self, video_id):
self.cur.execute('SELECT * FROM videos WHERE id == ?', [video_id])
video = self.cur.fetchone()
video = {key: video[SQL_VIDEO[key]] for key in SQL_VIDEO}
return video
def get_videos(self, channel_id=None, download_filter=None):
wheres = []
bindings = []
if channel_id is not None:
wheres.append('author_id')
bindings.append(channel_id)
if download_filter is not None:
wheres.append('download')
bindings.append(download_filter)
if wheres:
wheres = [x + ' == ?' for x in wheres]
wheres = ' WHERE ' + ' AND '.join(wheres)
else:
wheres = ''
query = 'SELECT * FROM videos' + wheres
self.cur.execute(query, bindings)
videos = self.cur.fetchall()
if not videos:
return []
videos = [{key: video[SQL_VIDEO[key]] for key in SQL_VIDEO} for video in videos]
videos.sort(key=lambda x: x['published'], reverse=True)
return videos
def insert_video(self, video, *, add_channel=True, commit=True):
if not isinstance(video, ytapi.Video):
video = self.youtube.get_video(video)
if add_channel:
self.add_channel(video.author_id, get_videos=False, commit=False)
self.cur.execute('SELECT * FROM videos WHERE id == ?', [video.id])
fetch = self.cur.fetchone()
if fetch is not None:
return {'new': False, 'row': fetch}
data = [None] * len(SQL_VIDEO)
data[SQL_VIDEO['id']] = video.id
data[SQL_VIDEO['published']] = video.published
data[SQL_VIDEO['author_id']] = video.author_id
data[SQL_VIDEO['title']] = video.title
data[SQL_VIDEO['description']] = video.description
data[SQL_VIDEO['duration']] = video.duration
data[SQL_VIDEO['thumbnail']] = video.thumbnail['url']
data[SQL_VIDEO['download']] = 'pending'
self.cur.execute('INSERT INTO videos VALUES(?, ?, ?, ?, ?, ?, ?, ?)', data)
if commit:
self.sql.commit()
return {'new': True, 'row': data}
def mark_video_state(self, video_id, state, commit=True):
'''
Mark the video as ignored, pending, or downloaded.
'''
if state not in ['ignored', 'pending', 'downloaded']:
raise InvalidVideoState(state)
self.cur.execute('SELECT * FROM videos WHERE id == ?', [video_id])
if self.cur.fetchone() is None:
raise NoSuchVideo(video_id)
self.cur.execute('UPDATE videos SET download = ? WHERE id == ?', [state, video_id])
if commit:
self.sql.commit()
def refresh_all_channels(self, force=False, commit=True):
for channel in self.get_channels():
self.refresh_channel(channel['id'], force=force, commit=commit)
if commit:
self.sql.commit()
def refresh_channel(self, channel_id, force=False, commit=True):
video_generator = self.youtube.get_user_videos(uid=channel_id)
log.debug('Refreshing channel: %s', channel_id)
for video in video_generator:
status = self.insert_video(video, commit=False)
if not force and not status['new']:
break
if commit:
self.sql.commit()

45
ycdl/ytapi.py
View file

@ -1,5 +1,6 @@
import apiclient.discovery
import datetime
import isodate
from . import helpers
@ -7,21 +8,24 @@ class VideoNotFound(Exception):
pass
class Video:
def __init__(self, snippet):
self.id = snippet['id']
def __init__(self, data):
self.id = data['id']
snippet = data['snippet']
content_details = data['contentDetails']
snippet = snippet['snippet']
self.title = snippet['title'] or '[untitled]'
self.description = snippet['description']
self.author_id = snippet['channelId']
self.author_name = snippet['channelTitle']
# Something like '2016-10-01T21:00:01'
self.published_string = snippet['publishedAt']
published = snippet['publishedAt']
published = published.split('.')[0]
published = snippet['publishedAt'].split('.')[0]
published = datetime.datetime.strptime(published, '%Y-%m-%dT%H:%M:%S')
self.published = published.timestamp()
self.duration = isodate.parse_duration(content_details['duration']).seconds
thumbnails = snippet['thumbnails']
best_thumbnail = max(thumbnails, key=lambda x: thumbnails[x]['width'] * thumbnails[x]['height'])
self.thumbnail = thumbnails[best_thumbnail]
@ -54,38 +58,39 @@ class Youtube:
user = self.youtube.channels().list(part='contentDetails', id=uid).execute()
upload_playlist = user['items'][0]['contentDetails']['relatedPlaylists']['uploads']
page_token = None
total = 0
while True:
items = self.youtube.playlistItems().list(
response = self.youtube.playlistItems().list(
maxResults=50,
pageToken=page_token,
part='contentDetails',
playlistId=upload_playlist,
).execute()
page_token = items.get('nextPageToken', None)
new = [item['contentDetails']['videoId'] for item in items['items']]
count = len(new)
new = self.get_video(new)
new.sort(key=lambda x: x.published, reverse=True)
yield from new
#print('Found %d more, %d total' % (count, len(videos)))
page_token = response.get('nextPageToken', None)
video_ids = [item['contentDetails']['videoId'] for item in response['items']]
videos = self.get_video(video_ids)
videos.sort(key=lambda x: x.published, reverse=True)
yield from videos
count = len(videos)
total += count
print(f'Found {count} more, {total} total')
if page_token is None or count < 50:
break
def get_related_videos(self, video_id, part='id,snippet', count=50):
def get_related_videos(self, video_id, count=50):
if isinstance(video_id, Video):
video_id = video_id.id
results = self.youtube.search().list(
part=part,
part='id',
relatedToVideoId=video_id,
type='video',
maxResults=count,
).execute()
videos = []
for related in results['items']:
related['id'] = related['id']['videoId']
video = Video(related)
videos.append(video)
related = [rel['id']['videoId'] for rel in results['items']]
videos = self.get_video(related)
return videos
def get_video(self, video_ids):
@ -99,7 +104,7 @@ class Youtube:
chunks = helpers.chunk_sequence(video_ids, 50)
for chunk in chunks:
chunk = ','.join(chunk)
data = self.youtube.videos().list(part='snippet', id=chunk).execute()
data = self.youtube.videos().list(part='id,contentDetails,snippet', id=chunk).execute()
items = data['items']
results.extend(items)
results = [Video(snippet) for snippet in results]