Changes

- Add thumbnails to the card (dynamically loaded)
- Add button for embedding the video
- Store video duration in db (use update script)
- Separate links for filter views with and without query.

frontends/ycdl_flask/templates/channel.html

@@ -77,15 +77,15 @@
     onclick="refresh_channel('{{channel['id']}}', false, function(){location.reload()})">Refresh new videos</button></span>
     <span><button class="refresh_button"
     onclick="refresh_channel('{{channel['id']}}', true, function(){location.reload()})">Refresh everything</button></span>
-    <span><a href="/channel/{{channel['id']}}{{query_string}}">All</a></span>
+    <span><a href="/channel/{{channel['id']}}">All</a> <a href="/channel/{{channel['id']}}{{query_string}}">(?q)</a></span>
-    <span><a href="/channel/{{channel['id']}}/pending{{query_string}}">Pending</a></span>
+    <span><a href="/channel/{{channel['id']}}/pending">Pending</a> <a href="/channel/{{channel['id']}}/pending{{query_string}}">(?q)</a></span>
-    <span><a href="/channel/{{channel['id']}}/ignored{{query_string}}">Ignored</a></span>
+    <span><a href="/channel/{{channel['id']}}/ignored">Ignored</a> <a href="/channel/{{channel['id']}}/ignored{{query_string}}">(?q)</a></span>
-    <span><a href="/channel/{{channel['id']}}/downloaded{{query_string}}">Downloaded</a></span>
+    <span><a href="/channel/{{channel['id']}}/downloaded">Downloaded</a> <a href="/channel/{{channel['id']}}/downloaded{{query_string}}">(?q)</a></span>
     {% else %}
-    <span><a href="/videos{{query_string}}">All</a></span>
+    <span><a href="/videos">All</a> <a href="/videos{{query_string}}">(?q)</a></span>
-    <span><a href="/videos/pending{{query_string}}">Pending</a></span>
+    <span><a href="/videos/pending">Pending</a> <a href="/videos/pending{{query_string}}">(?q)</a></span>
-    <span><a href="/videos/ignored{{query_string}}">Ignored</a></span>
+    <span><a href="/videos/ignored">Ignored</a> <a href="/videos/ignored{{query_string}}">(?q)</a></span>
-    <span><a href="/videos/downloaded{{query_string}}">Downloaded</a></span>
+    <span><a href="/videos/downloaded">Downloaded</a> <a href="/videos/downloaded{{query_string}}">(?q)</a></span>
     {% endif %}
     <span>{{videos|length}} items</span>
     
@@ -94,15 +94,11 @@
         <div id="video_card_{{video['id']}}"
         data-ytid="{{video['id']}}"
         onclick="onclick_select(event)"
-        {% if video['download'] == "downloaded" %}
+        class="video_card video_card_{{video['download']}}"
-        class="video_card video_card_downloaded"
-        {% elif video['download'] == "ignored" %}
-        class="video_card video_card_ignored"
-        {% else %}
-        class="video_card video_card_pending"
-        {% endif %}
         >
+            <img src="http://i3.ytimg.com/vi/{{video['id']}}/default.jpg" height="100px">
             <a href="https://www.youtube.com/watch?v={{video['id']}}">{{video['_published_str']}} - {{video['title']}}</a>
+            <span>({{video['duration'] | seconds_to_hms}})</span>
             {% if channel is none %}
             <a href="/channel/{{video['author_id']}}">(Chan)</a>
             {% endif %}
@@ -134,6 +130,10 @@
                 onclick="action_button_passthrough(event, mark_video_state, 'ignored')"
                 >Ignore</button>
             </div>
+            <br>
+            <button class="show_embed_button" onclick="toggle_embed_video('{{video.id}}');">Embed</button>
+            <button class="hide_embed_button hidden" onclick="toggle_embed_video('{{video.id}}');">Unembed</button>
+            <br>
         </div>
         {% endfor %}
     </div>
@@ -147,6 +147,28 @@ var video_card_first_selected = null;
 var video_card_selections = [];
 var video_cards = Array.from(document.getElementById("video_cards").children);
 
+function toggle_embed_video(video_id)
+{
+    var video_card = document.getElementById("video_card_" + video_id);
+    var show_button = video_card.getElementsByClassName("show_embed_button")[0];
+    var hide_button = video_card.getElementsByClassName("hide_embed_button")[0];
+    var embeds = video_card.getElementsByTagName("iframe");
+    if (embeds.length == 0)
+    {
+        var html = `<iframe width="711" height="400" src="https://www.youtube.com/embed/${video_id}" frameborder="0" allow="encrypted-media" allowfullscreen></iframe>`
+        var embed = html_to_element(html);
+        video_card.appendChild(embed);
+        show_button.classList.add("hidden");
+        hide_button.classList.remove("hidden");
+    }
+    else
+    {
+        video_card.removeChild(embeds[0]);
+        show_button.classList.remove("hidden");
+        hide_button.classList.add("hidden");
+    }
+}
+
 function deselect_all()
 {
     var video_card_first_selected = null;
@@ -157,6 +179,13 @@ function deselect_all()
     video_card_selections = [];
 }
 
+function html_to_element(html)
+{
+    var template = document.createElement("template");
+    template.innerHTML = html;
+    return template.content.firstChild;
+}
+
 function onclick_select(event)
 {
     if (!event.target.classList.contains("video_card"))

frontends/ycdl_flask/ycdl_flask/jinja_filters.py

@@ -0,0 +1,21 @@
+import math
+
+def seconds_to_hms(seconds):
+    '''
+    Convert integer number of seconds to an hh:mm:ss string.
+    Only the necessary fields are used.
+    '''
+    if seconds is None:
+        return '???'
+
+    seconds = math.ceil(seconds)
+    (minutes, seconds) = divmod(seconds, 60)
+    (hours, minutes) = divmod(minutes, 60)
+    parts = []
+    if hours:
+        parts.append(hours)
+    if minutes:
+        parts.append(minutes)
+    parts.append(seconds)
+    hms = ':'.join(f'{part:02d}' for part in parts)
+    return hms

frontends/ycdl_flask/ycdl_flask/ycdl_flask.py

@@ -18,6 +18,8 @@ import ycdl
 
 from voussoirkit import pathclass
 
+from . import jinja_filters
+
 root_dir = pathclass.Path(__file__).parent.parent
 
 TEMPLATE_DIR = root_dir.with_child('templates')
@@ -37,6 +39,7 @@ site.config.update(
     TEMPLATES_AUTO_RELOAD=True,
 )
 site.jinja_env.add_extension('jinja2.ext.do')
+site.jinja_env.filters['seconds_to_hms'] = jinja_filters.seconds_to_hms
 site.debug = True
 
 ####################################################################################################
@@ -192,6 +195,7 @@ def post_mark_video_state():
         youtube.mark_video_state(video_id, state)
 
     except ycdl.NoSuchVideo:
+        traceback.print_exc()
         flask.abort(404)
 
     except ycdl.InvalidVideoState:

frontends/ycdl_flask/ycdl_flask_launch.py

@@ -5,7 +5,6 @@ import gevent.monkey
 gevent.monkey.patch_all()
 
 import gevent.pywsgi
-import gevent.wsgi
 import sys
 
 import ycdl_flask

requirements.txt

@@ -1,3 +1,4 @@
 flask
 gevent
 https://github.com/voussoir/else/raw/master/_voussoirkit/voussoirkit.zip
+isodate

utilities/database_upgrader.py

@@ -0,0 +1,80 @@
+import argparse
+import os
+import sqlite3
+import sys
+
+import ycdl
+
+def upgrade_1_to_2(sql):
+    '''
+    In this version, a column `tagged_at` was added to the Photos table, to keep
+    track of the last time the photo's tags were edited (added or removed).
+    '''
+    cur = sql.cursor()
+    cur.executescript('''
+        ALTER TABLE videos RENAME TO videos_old;
+        CREATE TABLE videos(
+            id TEXT,
+            published INT,
+            author_id TEXT,
+            title TEXT,
+            description TEXT,
+            duration INT,
+            thumbnail TEXT,
+            download TEXT
+        );
+        INSERT INTO videos SELECT
+            id,
+            published,
+            author_id,
+            title,
+            description,
+            NULL,
+            thumbnail,
+            download
+        FROM videos_old;
+        DROP TABLE videos_old;
+    ''')
+
+def upgrade_all(database_filepath):
+    '''
+    Given the directory containing a phototagger database, apply all of the
+    needed upgrade_x_to_y functions in order.
+    '''
+    sql = sqlite3.connect(database_filepath)
+
+    cur = sql.cursor()
+
+    cur.execute('PRAGMA user_version')
+    current_version = cur.fetchone()[0]
+    needed_version = ycdl.ycdl.DATABASE_VERSION
+
+    if current_version == needed_version:
+        print('Already up to date with version %d.' % needed_version)
+        return
+
+    for version_number in range(current_version + 1, needed_version + 1):
+        print('Upgrading from %d to %d.' % (current_version, version_number))
+        upgrade_function = 'upgrade_%d_to_%d' % (current_version, version_number)
+        upgrade_function = eval(upgrade_function)
+        upgrade_function(sql)
+        sql.cursor().execute('PRAGMA user_version = %d' % version_number)
+        sql.commit()
+        current_version = version_number
+    print('Upgrades finished.')
+
+
+def upgrade_all_argparse(args):
+    return upgrade_all(database_filepath=args.database_filepath)
+
+def main(argv):
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('database_filepath')
+    parser.set_defaults(func=upgrade_all_argparse)
+
+    args = parser.parse_args(argv)
+    args.func(args)
+
+if __name__ == '__main__':
+    raise SystemExit(main(sys.argv[1:]))

utilities/download_thumbnails.py

@@ -1,12 +1,15 @@
-import bot
+import logging
+logging.basicConfig(level=logging.WARNING)
+
+import bot3 as bot
 import os
 import traceback
 import ycdl
-import ytapi
 import ycdl_repl
 from voussoirkit import downloady
 
-youtube_core = ytapi.Youtube(bot.YOUTUBE_KEY)
+
+youtube_core = ycdl.ytapi.Youtube(bot.YOUTUBE_KEY)
 youtube = ycdl.YCDL(youtube_core)
 
 DIRECTORY = '.\\youtube thumbnails'

ycdl/__init__.py

@@ -1,296 +1,5 @@
-import logging
-import os
-import sqlite3
-
 from . import helpers
+from . import ycdl
 from . import ytapi
 
-def YOUTUBE_DL_COMMAND(video_id):
+YCDL = ycdl.YCDL
-    path = 'D:\\Incoming\\ytqueue\\{id}.ytqueue'.format(id=video_id)
-    open(path, 'w')
-
-logging.basicConfig(level=logging.DEBUG)
-log = logging.getLogger(__name__)
-logging.getLogger('googleapiclient.discovery').setLevel(logging.WARNING)
-logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.WARNING)
-logging.getLogger('requests.packages.urllib3.util.retry').setLevel(logging.WARNING)
-
-SQL_CHANNEL_COLUMNS = [
-    'id',
-    'name',
-    'directory',
-]
-
-SQL_VIDEO_COLUMNS = [
-    'id',
-    'published',
-    'author_id',
-    'title',
-    'description',
-    'thumbnail',
-    'download',
-]
-
-SQL_CHANNEL = {key:index for (index, key) in enumerate(SQL_CHANNEL_COLUMNS)}
-SQL_VIDEO = {key:index for (index, key) in enumerate(SQL_VIDEO_COLUMNS)}
-
-DATABASE_VERSION = 1
-DB_INIT = '''
-PRAGMA count_changes = OFF;
-PRAGMA cache_size = 10000;
-PRAGMA user_version = {user_version};
-CREATE TABLE IF NOT EXISTS channels(
-    id TEXT,
-    name TEXT,
-    directory TEXT COLLATE NOCASE
-);
-CREATE TABLE IF NOT EXISTS videos(
-    id TEXT,
-    published INT,
-    author_id TEXT,
-    title TEXT,
-    description TEXT,
-    thumbnail TEXT,
-    download TEXT
-);
-
-CREATE INDEX IF NOT EXISTS index_channel_id on channels(id);
-CREATE INDEX IF NOT EXISTS index_video_id on videos(id);
-CREATE INDEX IF NOT EXISTS index_video_published on videos(published);
-CREATE INDEX IF NOT EXISTS index_video_download on videos(download);
-'''.format(user_version=DATABASE_VERSION)
-
-DEFAULT_DBNAME = 'ycdl.db'
-
-ERROR_DATABASE_OUTOFDATE = 'Database is out-of-date. {current} should be {new}'
-
-
-def verify_is_abspath(path):
-    '''
-    TO DO: Determine whether this is actually correct.
-    '''
-    if os.path.abspath(path) != path:
-        raise ValueError('Not an abspath')
-
-
-class InvalidVideoState(Exception):
-    pass
-
-class NoSuchVideo(Exception):
-    pass
-
-
-class YCDL:
-    def __init__(self, youtube, database_filename=None, youtube_dl_function=None):
-        self.youtube = youtube
-        if database_filename is None:
-            database_filename = DEFAULT_DBNAME
-
-        existing_database = os.path.exists(database_filename)
-        self.sql = sqlite3.connect(database_filename)
-        self.cur = self.sql.cursor()
-
-        if existing_database:
-            self.cur.execute('PRAGMA user_version')
-            existing_version = self.cur.fetchone()[0]
-            if existing_version != DATABASE_VERSION:
-                message = ERROR_DATABASE_OUTOFDATE
-                message = message.format(current=existing_version, new=DATABASE_VERSION)
-                print(message)
-                raise SystemExit
-
-        if youtube_dl_function:
-            self.youtube_dl_function = youtube_dl_function
-        else:
-            self.youtube_dl_function = YOUTUBE_DL_COMMAND
-
-        statements = DB_INIT.split(';')
-        for statement in statements:
-            self.cur.execute(statement)
-        self.sql.commit()
-
-    def add_channel(
-            self,
-            channel_id,
-            commit=True,
-            download_directory=None,
-            get_videos=False,
-            name=None,
-        ):
-        if self.get_channel(channel_id) is not None:
-            return
-
-        if name is None:
-            name = self.youtube.get_user_name(channel_id)
-
-        data = [None] * len(SQL_CHANNEL)
-        data[SQL_CHANNEL['id']] = channel_id
-        data[SQL_CHANNEL['name']] = name
-        if download_directory is not None:
-            verify_is_abspath(download_directory)
-        data[SQL_CHANNEL['directory']] = download_directory
-
-        self.cur.execute('INSERT INTO channels VALUES(?, ?, ?)', data)
-        if get_videos:
-            self.refresh_channel(channel_id, commit=False)
-        if commit:
-            self.sql.commit()
-
-    def channel_has_pending(self, channel_id):
-        query = 'SELECT * FROM videos WHERE author_id == ? AND download == "pending"'
-        self.cur.execute(query, [channel_id])
-        return self.cur.fetchone() is not None
-
-    def channel_directory(self, channel_id):
-        self.cur.execute('SELECT * FROM channels WHERE id == ?', [channel_id])
-        fetch = self.cur.fetchone()
-        if fetch is None:
-            return None
-        return fetch[SQL_CHANNEL['directory']]
-
-    def download_video(self, video, force=False):
-        '''
-        Execute the `YOUTUBE_DL_COMMAND`, within the channel's associated
-        directory if applicable.
-        '''
-        # This logic is a little hazier than I would like, but it's all in the
-        # interest of minimizing unnecessary API calls.
-        if isinstance(video, ytapi.Video):
-            video_id = video.id
-        else:
-            video_id = video
-        self.cur.execute('SELECT * FROM videos WHERE id == ?', [video_id])
-        video_row = self.cur.fetchone()
-        if video_row is None:
-            # Since the video was not in the db, we may not know about the channel either.
-            if not isinstance(video, ytapi.Video):
-                print('get video')
-                video = self.youtube.get_video(video)
-            channel_id = video.author_id
-            self.cur.execute('SELECT * FROM channels WHERE id == ?', [channel_id])
-            if self.cur.fetchone() is None:
-                print('add channel')
-                self.add_channel(channel_id, get_videos=False, commit=False)
-            video_row = self.insert_video(video, commit=False)['row']
-        else:
-            channel_id = video_row[SQL_VIDEO['author_id']]
-
-        if video_row[SQL_VIDEO['download']] != 'pending' and not force:
-            print('That video does not need to be downloaded.')
-            return
-
-        current_directory = os.getcwd()
-        download_directory = self.channel_directory(channel_id)
-        download_directory = download_directory or current_directory
-
-        os.makedirs(download_directory, exist_ok=True)
-        os.chdir(download_directory)
-
-        self.youtube_dl_function(video_id)
-
-        os.chdir(current_directory)
-
-        self.cur.execute('UPDATE videos SET download = "downloaded" WHERE id == ?', [video_id])
-        self.sql.commit()
-
-    def get_channel(self, channel_id):
-        self.cur.execute('SELECT * FROM channels WHERE id == ?', [channel_id])
-        fetch = self.cur.fetchone()
-        if not fetch:
-            return None
-        fetch = {key: fetch[SQL_CHANNEL[key]] for key in SQL_CHANNEL}
-        return fetch
-
-    def get_channels(self):
-        self.cur.execute('SELECT * FROM channels')
-        channels = self.cur.fetchall()
-        channels = [{key: channel[SQL_CHANNEL[key]] for key in SQL_CHANNEL} for channel in channels]
-        channels.sort(key=lambda x: x['name'].lower())
-        return channels
-
-    def get_video(self, video_id):
-        self.cur.execute('SELECT * FROM videos WHERE id == ?', [video_id])
-        video = self.cur.fetchone()
-        video = {key: video[SQL_VIDEO[key]] for key in SQL_VIDEO}
-        return video
-
-    def get_videos(self, channel_id=None, download_filter=None):
-        wheres = []
-        bindings = []
-        if channel_id is not None:
-            wheres.append('author_id')
-            bindings.append(channel_id)
-
-        if download_filter is not None:
-            wheres.append('download')
-            bindings.append(download_filter)
-
-        if wheres:
-            wheres = [x + ' == ?' for x in wheres]
-            wheres = ' WHERE ' + ' AND '.join(wheres)
-        else:
-            wheres = ''
-
-        query = 'SELECT * FROM videos' + wheres
-        self.cur.execute(query, bindings)
-        videos = self.cur.fetchall()
-        if not videos:
-            return []
-
-        videos = [{key: video[SQL_VIDEO[key]] for key in SQL_VIDEO} for video in videos]
-        videos.sort(key=lambda x: x['published'], reverse=True)
-        return videos
-
-    def insert_video(self, video, *, add_channel=True, commit=True):
-        if not isinstance(video, ytapi.Video):
-            video = self.youtube.get_video(video)
-
-        if add_channel:
-            self.add_channel(video.author_id, get_videos=False, commit=False)
-        self.cur.execute('SELECT * FROM videos WHERE id == ?', [video.id])
-        fetch = self.cur.fetchone()
-        if fetch is not None:
-            return {'new': False, 'row': fetch}
-
-        data = [None] * len(SQL_VIDEO)
-        data[SQL_VIDEO['id']] = video.id
-        data[SQL_VIDEO['published']] = video.published
-        data[SQL_VIDEO['author_id']] = video.author_id
-        data[SQL_VIDEO['title']] = video.title
-        data[SQL_VIDEO['description']] = video.description
-        data[SQL_VIDEO['thumbnail']] = video.thumbnail['url']
-        data[SQL_VIDEO['download']] = 'pending'
-
-        self.cur.execute('INSERT INTO videos VALUES(?, ?, ?, ?, ?, ?, ?)', data)
-        if commit:
-            self.sql.commit()
-        return {'new': True, 'row': data}
-
-    def mark_video_state(self, video_id, state, commit=True):
-        '''
-        Mark the video as ignored, pending, or downloaded.
-        '''
-        if state not in ['ignored', 'pending', 'downloaded']:
-            raise InvalidVideoState(state)
-        self.cur.execute('SELECT * FROM videos WHERE id == ?', [video_id])
-        if self.cur.fetchone() is None:
-            raise NoSuchVideo(video_id)
-        self.cur.execute('UPDATE videos SET download = ? WHERE id == ?', [state, video_id])
-        if commit:
-            self.sql.commit()
-
-    def refresh_all_channels(self, force=False, commit=True):
-        for channel in self.get_channels():
-            self.refresh_channel(channel['id'], force=force, commit=commit)
-        if commit:
-            self.sql.commit()
-
-    def refresh_channel(self, channel_id, force=False, commit=True):
-        video_generator = self.youtube.get_user_videos(uid=channel_id)
-        log.debug('Refreshing channel: %s', channel_id)
-        for video in video_generator:
-            status = self.insert_video(video, commit=False)
-            if not force and not status['new']:
-                break
-        if commit:
-            self.sql.commit()

ycdl/ycdl.py

@@ -0,0 +1,300 @@
+import logging
+import os
+import sqlite3
+
+from . import helpers
+from . import ytapi
+
+def YOUTUBE_DL_COMMAND(video_id):
+    path = 'D:\\Incoming\\ytqueue\\{id}.ytqueue'.format(id=video_id)
+    open(path, 'w')
+
+logging.basicConfig(level=logging.DEBUG)
+log = logging.getLogger(__name__)
+logging.getLogger('googleapiclient.discovery').setLevel(logging.WARNING)
+logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.WARNING)
+logging.getLogger('requests.packages.urllib3.util.retry').setLevel(logging.WARNING)
+
+SQL_CHANNEL_COLUMNS = [
+    'id',
+    'name',
+    'directory',
+]
+
+SQL_VIDEO_COLUMNS = [
+    'id',
+    'published',
+    'author_id',
+    'title',
+    'description',
+    'duration',
+    'thumbnail',
+    'download',
+]
+
+SQL_CHANNEL = {key:index for (index, key) in enumerate(SQL_CHANNEL_COLUMNS)}
+SQL_VIDEO = {key:index for (index, key) in enumerate(SQL_VIDEO_COLUMNS)}
+
+DATABASE_VERSION = 2
+DB_INIT = '''
+PRAGMA count_changes = OFF;
+PRAGMA cache_size = 10000;
+PRAGMA user_version = {user_version};
+CREATE TABLE IF NOT EXISTS channels(
+    id TEXT,
+    name TEXT,
+    directory TEXT COLLATE NOCASE
+);
+CREATE TABLE IF NOT EXISTS videos(
+    id TEXT,
+    published INT,
+    author_id TEXT,
+    title TEXT,
+    description TEXT,
+    duration INT,
+    thumbnail TEXT,
+    download TEXT
+);
+
+CREATE INDEX IF NOT EXISTS index_channel_id on channels(id);
+CREATE INDEX IF NOT EXISTS index_video_author on videos(author_id);
+CREATE INDEX IF NOT EXISTS index_video_id on videos(id);
+CREATE INDEX IF NOT EXISTS index_video_published on videos(published);
+CREATE INDEX IF NOT EXISTS index_video_download on videos(download);
+'''.format(user_version=DATABASE_VERSION)
+
+DEFAULT_DBNAME = 'ycdl.db'
+
+ERROR_DATABASE_OUTOFDATE = 'Database is out-of-date. {current} should be {new}'
+
+
+def verify_is_abspath(path):
+    '''
+    TO DO: Determine whether this is actually correct.
+    '''
+    if os.path.abspath(path) != path:
+        raise ValueError('Not an abspath')
+
+
+class InvalidVideoState(Exception):
+    pass
+
+class NoSuchVideo(Exception):
+    pass
+
+
+class YCDL:
+    def __init__(self, youtube, database_filename=None, youtube_dl_function=None):
+        self.youtube = youtube
+        if database_filename is None:
+            database_filename = DEFAULT_DBNAME
+
+        existing_database = os.path.exists(database_filename)
+        self.sql = sqlite3.connect(database_filename)
+        self.cur = self.sql.cursor()
+
+        if existing_database:
+            self.cur.execute('PRAGMA user_version')
+            existing_version = self.cur.fetchone()[0]
+            if existing_version != DATABASE_VERSION:
+                message = ERROR_DATABASE_OUTOFDATE
+                message = message.format(current=existing_version, new=DATABASE_VERSION)
+                print(message)
+                raise SystemExit
+
+        if youtube_dl_function:
+            self.youtube_dl_function = youtube_dl_function
+        else:
+            self.youtube_dl_function = YOUTUBE_DL_COMMAND
+
+        statements = DB_INIT.split(';')
+        for statement in statements:
+            self.cur.execute(statement)
+        self.sql.commit()
+
+    def add_channel(
+            self,
+            channel_id,
+            commit=True,
+            download_directory=None,
+            get_videos=False,
+            name=None,
+        ):
+        if self.get_channel(channel_id) is not None:
+            return
+
+        if name is None:
+            name = self.youtube.get_user_name(channel_id)
+
+        data = [None] * len(SQL_CHANNEL)
+        data[SQL_CHANNEL['id']] = channel_id
+        data[SQL_CHANNEL['name']] = name
+        if download_directory is not None:
+            verify_is_abspath(download_directory)
+        data[SQL_CHANNEL['directory']] = download_directory
+
+        self.cur.execute('INSERT INTO channels VALUES(?, ?, ?)', data)
+        if get_videos:
+            self.refresh_channel(channel_id, commit=False)
+        if commit:
+            self.sql.commit()
+
+    def channel_has_pending(self, channel_id):
+        query = 'SELECT * FROM videos WHERE author_id == ? AND download == "pending"'
+        self.cur.execute(query, [channel_id])
+        return self.cur.fetchone() is not None
+
+    def channel_directory(self, channel_id):
+        self.cur.execute('SELECT * FROM channels WHERE id == ?', [channel_id])
+        fetch = self.cur.fetchone()
+        if fetch is None:
+            return None
+        return fetch[SQL_CHANNEL['directory']]
+
+    def download_video(self, video, force=False):
+        '''
+        Execute the `YOUTUBE_DL_COMMAND`, within the channel's associated
+        directory if applicable.
+        '''
+        # This logic is a little hazier than I would like, but it's all in the
+        # interest of minimizing unnecessary API calls.
+        if isinstance(video, ytapi.Video):
+            video_id = video.id
+        else:
+            video_id = video
+        self.cur.execute('SELECT * FROM videos WHERE id == ?', [video_id])
+        video_row = self.cur.fetchone()
+        if video_row is None:
+            # Since the video was not in the db, we may not know about the channel either.
+            if not isinstance(video, ytapi.Video):
+                print('get video')
+                video = self.youtube.get_video(video)
+            channel_id = video.author_id
+            self.cur.execute('SELECT * FROM channels WHERE id == ?', [channel_id])
+            if self.cur.fetchone() is None:
+                print('add channel')
+                self.add_channel(channel_id, get_videos=False, commit=False)
+            video_row = self.insert_video(video, commit=False)['row']
+        else:
+            channel_id = video_row[SQL_VIDEO['author_id']]
+
+        if video_row[SQL_VIDEO['download']] != 'pending' and not force:
+            print('That video does not need to be downloaded.')
+            return
+
+        current_directory = os.getcwd()
+        download_directory = self.channel_directory(channel_id)
+        download_directory = download_directory or current_directory
+
+        os.makedirs(download_directory, exist_ok=True)
+        os.chdir(download_directory)
+
+        self.youtube_dl_function(video_id)
+
+        os.chdir(current_directory)
+
+        self.cur.execute('UPDATE videos SET download = "downloaded" WHERE id == ?', [video_id])
+        self.sql.commit()
+
+    def get_channel(self, channel_id):
+        self.cur.execute('SELECT * FROM channels WHERE id == ?', [channel_id])
+        fetch = self.cur.fetchone()
+        if not fetch:
+            return None
+        fetch = {key: fetch[SQL_CHANNEL[key]] for key in SQL_CHANNEL}
+        return fetch
+
+    def get_channels(self):
+        self.cur.execute('SELECT * FROM channels')
+        channels = self.cur.fetchall()
+        channels = [{key: channel[SQL_CHANNEL[key]] for key in SQL_CHANNEL} for channel in channels]
+        channels.sort(key=lambda x: x['name'].lower())
+        return channels
+
+    def get_video(self, video_id):
+        self.cur.execute('SELECT * FROM videos WHERE id == ?', [video_id])
+        video = self.cur.fetchone()
+        video = {key: video[SQL_VIDEO[key]] for key in SQL_VIDEO}
+        return video
+
+    def get_videos(self, channel_id=None, download_filter=None):
+        wheres = []
+        bindings = []
+        if channel_id is not None:
+            wheres.append('author_id')
+            bindings.append(channel_id)
+
+        if download_filter is not None:
+            wheres.append('download')
+            bindings.append(download_filter)
+
+        if wheres:
+            wheres = [x + ' == ?' for x in wheres]
+            wheres = ' WHERE ' + ' AND '.join(wheres)
+        else:
+            wheres = ''
+
+        query = 'SELECT * FROM videos' + wheres
+        self.cur.execute(query, bindings)
+        videos = self.cur.fetchall()
+        if not videos:
+            return []
+
+        videos = [{key: video[SQL_VIDEO[key]] for key in SQL_VIDEO} for video in videos]
+        videos.sort(key=lambda x: x['published'], reverse=True)
+        return videos
+
+    def insert_video(self, video, *, add_channel=True, commit=True):
+        if not isinstance(video, ytapi.Video):
+            video = self.youtube.get_video(video)
+
+        if add_channel:
+            self.add_channel(video.author_id, get_videos=False, commit=False)
+        self.cur.execute('SELECT * FROM videos WHERE id == ?', [video.id])
+        fetch = self.cur.fetchone()
+        if fetch is not None:
+            return {'new': False, 'row': fetch}
+
+        data = [None] * len(SQL_VIDEO)
+        data[SQL_VIDEO['id']] = video.id
+        data[SQL_VIDEO['published']] = video.published
+        data[SQL_VIDEO['author_id']] = video.author_id
+        data[SQL_VIDEO['title']] = video.title
+        data[SQL_VIDEO['description']] = video.description
+        data[SQL_VIDEO['duration']] = video.duration
+        data[SQL_VIDEO['thumbnail']] = video.thumbnail['url']
+        data[SQL_VIDEO['download']] = 'pending'
+
+        self.cur.execute('INSERT INTO videos VALUES(?, ?, ?, ?, ?, ?, ?, ?)', data)
+        if commit:
+            self.sql.commit()
+        return {'new': True, 'row': data}
+
+    def mark_video_state(self, video_id, state, commit=True):
+        '''
+        Mark the video as ignored, pending, or downloaded.
+        '''
+        if state not in ['ignored', 'pending', 'downloaded']:
+            raise InvalidVideoState(state)
+        self.cur.execute('SELECT * FROM videos WHERE id == ?', [video_id])
+        if self.cur.fetchone() is None:
+            raise NoSuchVideo(video_id)
+        self.cur.execute('UPDATE videos SET download = ? WHERE id == ?', [state, video_id])
+        if commit:
+            self.sql.commit()
+
+    def refresh_all_channels(self, force=False, commit=True):
+        for channel in self.get_channels():
+            self.refresh_channel(channel['id'], force=force, commit=commit)
+        if commit:
+            self.sql.commit()
+
+    def refresh_channel(self, channel_id, force=False, commit=True):
+        video_generator = self.youtube.get_user_videos(uid=channel_id)
+        log.debug('Refreshing channel: %s', channel_id)
+        for video in video_generator:
+            status = self.insert_video(video, commit=False)
+            if not force and not status['new']:
+                break
+        if commit:
+            self.sql.commit()

ycdl/ytapi.py

@@ -1,5 +1,6 @@
 import apiclient.discovery
 import datetime
+import isodate
 
 from . import helpers
 
@@ -7,21 +8,24 @@ class VideoNotFound(Exception):
     pass
 
 class Video:
-    def __init__(self, snippet):
+    def __init__(self, data):
-        self.id = snippet['id']
+        self.id = data['id']
+
+        snippet = data['snippet']
+        content_details = data['contentDetails']
 
-        snippet = snippet['snippet']
         self.title = snippet['title'] or '[untitled]'
         self.description = snippet['description']
         self.author_id = snippet['channelId']
         self.author_name = snippet['channelTitle']
         # Something like '2016-10-01T21:00:01'
         self.published_string = snippet['publishedAt']
-        published = snippet['publishedAt']
+        published = snippet['publishedAt'].split('.')[0]
-        published = published.split('.')[0]
         published = datetime.datetime.strptime(published, '%Y-%m-%dT%H:%M:%S')
         self.published = published.timestamp()
 
+        self.duration = isodate.parse_duration(content_details['duration']).seconds
+
         thumbnails = snippet['thumbnails']
         best_thumbnail = max(thumbnails, key=lambda x: thumbnails[x]['width'] * thumbnails[x]['height'])
         self.thumbnail = thumbnails[best_thumbnail]
@@ -54,38 +58,39 @@ class Youtube:
             user = self.youtube.channels().list(part='contentDetails', id=uid).execute()
         upload_playlist = user['items'][0]['contentDetails']['relatedPlaylists']['uploads']
         page_token = None
+        total = 0
         while True:
-            items = self.youtube.playlistItems().list(
+            response = self.youtube.playlistItems().list(
                 maxResults=50,
                 pageToken=page_token,
                 part='contentDetails',
                 playlistId=upload_playlist,
             ).execute()
-            page_token = items.get('nextPageToken', None)
+            page_token = response.get('nextPageToken', None)
-            new = [item['contentDetails']['videoId'] for item in items['items']]
+            video_ids = [item['contentDetails']['videoId'] for item in response['items']]
-            count = len(new)
+            videos = self.get_video(video_ids)
-            new = self.get_video(new)
+            videos.sort(key=lambda x: x.published, reverse=True)
-            new.sort(key=lambda x: x.published, reverse=True)
+            yield from videos
-            yield from new
+
-            #print('Found %d more, %d total' % (count, len(videos)))
+            count = len(videos)
+            total += count
+            print(f'Found {count} more, {total} total')
             if page_token is None or count < 50:
                 break
 
-    def get_related_videos(self, video_id, part='id,snippet', count=50):
+    def get_related_videos(self, video_id, count=50):
         if isinstance(video_id, Video):
             video_id = video_id.id
 
         results = self.youtube.search().list(
-            part=part,
+            part='id',
             relatedToVideoId=video_id,
             type='video',
             maxResults=count,
         ).execute()
         videos = []
-        for related in results['items']:
+        related = [rel['id']['videoId'] for rel in results['items']]
-            related['id'] = related['id']['videoId']
+        videos = self.get_video(related)
-            video = Video(related)
-            videos.append(video)
         return videos
 
     def get_video(self, video_ids):
@@ -99,7 +104,7 @@ class Youtube:
         chunks = helpers.chunk_sequence(video_ids, 50)
         for chunk in chunks:
             chunk = ','.join(chunk)
-            data = self.youtube.videos().list(part='snippet', id=chunk).execute()
+            data = self.youtube.videos().list(part='id,contentDetails,snippet', id=chunk).execute()
             items = data['items']
             results.extend(items)
         results = [Video(snippet) for snippet in results]