Ignore youtube shorts on a per-channel basis.

2023-08-09 23:24:47 -07:00 · 2023-08-09 23:24:47 -07:00 · 4de461dbf1
commit 4de461dbf1
parent 94ee832a27
11 changed files with 222 additions and 28 deletions
--- a/README.md
+++ b/README.md
@ -31,10 +31,11 @@ The reason for this is that youtube-dl is extremely configurable. Every user mig

 - Web interface with video embeds
 - Commandline interface for scripted use
- "Sub-box" page where newest videos from all channels are listed in order
+- "New videos" feed page where newest videos from all channels are listed in order (wow, no algorithm!)
 - Sort videos by date, duration, views, or random
 - Background thread will refresh channels over time
- Automark channels as ignore or download
+- Mark channels as auto-download or auto-ignore
+- Automatically remove shorts from feed
 - Free yourself from Youtube's awful recommendation system

 ## Your API key
--- a/frontends/ycdl_flask/backend/common.py
+++ b/frontends/ycdl_flask/backend/common.py
@ -6,6 +6,7 @@ import flask; from flask import request
 import functools
 import threading
 import time
+import traceback

 from voussoirkit import flasktools
 from voussoirkit import pathclass
@ -121,7 +122,45 @@ def refresher_thread(rate):
        refresh_job.start()
        last_refresh = time.time()

+def ignore_shorts_thread(rate):
+    last_commit_id = None
+    while True:
+        if ycdldb.last_commit_id == last_commit_id:
+            # log.debug('Sleeping again due to no new commits.')
+            time.sleep(5 * rate)
+            continue
+
+        last_commit_id = ycdldb.last_commit_id
+
+        log.info('Starting shorts job.')
+        videos = ycdldb.get_videos_by_sql('''
+        SELECT * FROM videos
+        LEFT JOIN channels ON channels.id = videos.author_id
+        WHERE is_shorts IS NULL AND duration < 62 AND state = "pending" AND channels.ignore_shorts = 1
+        ORDER BY published DESC
+        LIMIT 10
+        ''')
+        videos = list(videos)
+        if len(videos) == 0:
+            time.sleep(rate)
+            continue
+
+        with ycdldb.transaction:
+            for video in videos:
+                try:
+                    is_shorts = ycdl.ytapi.video_is_shorts(video.id)
+                except Exception as exc:
+                    log.warning(traceback.format_exc())
+                pairs = {'id': video.id, 'is_shorts': int(is_shorts)}
+                if is_shorts:
+                    pairs['state'] = 'ignored'
+                ycdldb.update(table=ycdl.objects.Video, pairs=pairs, where_key='id')
+        time.sleep(rate)
+
 def start_refresher_thread(rate):
    log.info('Starting refresher thread, once per %d seconds.', rate)
    refresher = threading.Thread(target=refresher_thread, args=[rate], daemon=True)
    refresher.start()
+
+    shorts_killer = threading.Thread(target=ignore_shorts_thread, args=[60], daemon=True)
+    shorts_killer.start()
--- a/frontends/ycdl_flask/backend/endpoints/channel_endpoints.py
+++ b/frontends/ycdl_flask/backend/endpoints/channel_endpoints.py
@ -211,6 +211,22 @@ def post_set_download_directory(channel_id):
    response = {'id': channel.id, 'download_directory': abspath}
    return flasktools.json_response(response)

+@flasktools.required_fields(['ignore_shorts'], forbid_whitespace=True)
+@site.route('/channel/<channel_id>/set_ignore_shorts', methods=['POST'])
+def post_set_ignore_shorts(channel_id):
+    ignore_shorts = request.form['ignore_shorts']
+    channel = common.ycdldb.get_channel(channel_id)
+
+    try:
+        ignore_shorts = stringtools.truthystring(ignore_shorts)
+        with common.ycdldb.transaction:
+            channel.set_ignore_shorts(ignore_shorts)
+    except (ValueError, TypeError):
+        flask.abort(400)
+
+    response = {'id': channel.id, 'ignore_shorts': channel.ignore_shorts}
+    return flasktools.json_response(response)
+
@flasktools.required_fields(['name'], forbid_whitespace=False)
@site.route('/channel/<channel_id>/set_name', methods=['POST'])
 def post_set_name(channel_id):
--- a/frontends/ycdl_flask/static/js/api.js
+++ b/frontends/ycdl_flask/static/js/api.js
@ -73,6 +73,16 @@ function set_download_directory(channel_id, download_directory, callback)
    });
 }

+api.channels.set_ignore_shorts =
+function set_ignore_shorts(channel_id, ignore_shorts, callback)
+{
+    return http.post({
+        url: `/channel/${channel_id}/set_ignore_shorts`,
+        data: {"ignore_shorts": ignore_shorts},
+        callback: callback,
+    });
+}
+
 api.channels.set_name =
 function set_name(channel_id, name, callback)
 {
--- a/frontends/ycdl_flask/static/js/common.js
+++ b/frontends/ycdl_flask/static/js/common.js
@ -88,13 +88,13 @@ function hms_render_colons(hours, minutes, seconds)
 }

 common.seconds_to_hms =
-function seconds_to_hms(seconds, args)
+function seconds_to_hms({
+    seconds,
+    renderer=common.hms_render_colons,
+    force_minutes=false,
+    force_hours=false,
+})
 {
-    args = args || {};
-    const renderer = args["renderer"] || common.hms_render_colons;
-    const force_minutes = args["force_minutes"] || false;
-    const force_hours = args["force_hours"] || false;
-
    if (seconds > 0 && seconds < 1)
    {
        seconds = 1;
--- a/frontends/ycdl_flask/templates/channel.html
+++ b/frontends/ycdl_flask/templates/channel.html
@ -266,6 +266,12 @@ https://stackoverflow.com/a/35153397
        <span id="set_automark_spinner" class="hidden">Working...</span>
    </div>

+    <div>
+        {% set checked = 'checked' if channel.ignore_shorts else '' %}
+        <label><input type="checkbox" id="set_ignore_shorts_checkbox" {{checked}} onchange="return set_ignore_shorts_form(event);"/> Automatically ignore shorts (short vertical videos).</label>
+        <span id="set_ignore_shorts_spinner" class="hidden">Working...</span>
+    </div>
+
    <div>
        <input type="text" id="set_queuefile_extension_input" placeholder="Queuefile extension" value="{{channel.queuefile_extension or ''}}"/>
        <button id="set_queuefile_extension_button" class="button_with_spinner" onclick="return set_queuefile_extension_form(event);">Set extension</button>
@ -661,6 +667,22 @@ function set_download_directory_callback(response)
    }
 }

+function set_ignore_shorts_form(event)
+{
+    set_ignore_shorts_spinner.show();
+    api.channels.set_ignore_shorts(CHANNEL_ID, event.target.checked, set_ignore_shorts_callback);
+}
+
+function set_ignore_shorts_callback(response)
+{
+    set_ignore_shorts_spinner.hide();
+    if (response.meta.status != 200)
+    {
+        alert(JSON.stringify(response));
+        return;
+    }
+}
+
 function set_name_form(event)
 {
    const name = set_name_input.value.trim();
@ -751,6 +773,9 @@ if (CHANNEL_ID)

    var set_autorefresh_spinner = document.getElementById("set_autorefresh_spinner");
    set_autorefresh_spinner = new spinners.Spinner(set_autorefresh_spinner);
+
+    var set_ignore_shorts_spinner = document.getElementById("set_ignore_shorts_spinner");
+    set_ignore_shorts_spinner = new spinners.Spinner(set_ignore_shorts_spinner);
 }

 ////////////////////////////////////////////////////////////////////////////////////////////////////
--- a/utilities/database_upgrader.py
+++ b/utilities/database_upgrader.py
@ -347,6 +347,72 @@ def upgrade_10_to_11(ycdldb):

    m.go()

+def upgrade_11_to_12(ycdldb):
+    '''
+    In this version, the `ignore_shorts` column was added to the channels table
+    and `is_shorts` was added to the videos table.
+    '''
+    m = Migrator(ycdldb)
+
+    m.tables['channels']['create'] = '''
+    CREATE TABLE IF NOT EXISTS channels(
+        id TEXT,
+        name TEXT,
+        uploads_playlist TEXT,
+        download_directory TEXT COLLATE NOCASE,
+        queuefile_extension TEXT COLLATE NOCASE,
+        automark TEXT,
+        autorefresh INT,
+        last_refresh INT,
+        ignore_shorts INT NOT NULL
+    );
+    '''
+    m.tables['channels']['transfer'] = '''
+    INSERT INTO channels SELECT
+        id,
+        name,
+        uploads_playlist,
+        download_directory,
+        queuefile_extension,
+        automark,
+        autorefresh,
+        last_refresh,
+        1
+    FROM channels_old;
+    '''
+    m.tables['videos']['create'] = '''
+    CREATE TABLE IF NOT EXISTS videos(
+        id TEXT,
+        published INT,
+        author_id TEXT,
+        title TEXT,
+        description TEXT,
+        duration INT,
+        views INT,
+        thumbnail TEXT,
+        live_broadcast TEXT,
+        state TEXT,
+        is_shorts INT
+    );
+    '''
+    m.tables['videos']['transfer'] = '''
+    INSERT INTO videos SELECT
+        id,
+        published,
+        author_id,
+        title,
+        description,
+        duration,
+        views,
+        thumbnail,
+        live_broadcast,
+        state,
+        NULL
+    FROM videos_old;
+    '''
+
+    m.go()
+
 def upgrade_all(data_directory):
    '''
    Given the directory containing a ycdl database, apply all of the
--- a/ycdl/constants.py
+++ b/ycdl/constants.py
@ -1,6 +1,6 @@
 from voussoirkit import sqlhelpers

-DATABASE_VERSION = 11
+DATABASE_VERSION = 12

 DB_INIT = f'''
 CREATE TABLE IF NOT EXISTS channels(
@ -11,7 +11,8 @@ CREATE TABLE IF NOT EXISTS channels(
    queuefile_extension TEXT COLLATE NOCASE,
    automark TEXT,
    autorefresh INT,
-    last_refresh INT
+    last_refresh INT,
+    ignore_shorts INT NOT NULL
 );
 CREATE INDEX IF NOT EXISTS index_channel_id on channels(id);
 ----------------------------------------------------------------------------------------------------
@ -25,7 +26,8 @@ CREATE TABLE IF NOT EXISTS videos(
    views INT,
    thumbnail TEXT,
    live_broadcast TEXT,
-    state TEXT
+    state TEXT,
+    is_shorts INT
 );
 CREATE INDEX IF NOT EXISTS index_video_author_published on videos(author_id, published);
 CREATE INDEX IF NOT EXISTS index_video_author_state_published on videos(author_id, state, published);
--- a/ycdl/objects.py
+++ b/ycdl/objects.py
@ -36,6 +36,7 @@ class Channel(ObjectBase):
        self.queuefile_extension = self.normalize_queuefile_extension(db_row['queuefile_extension'])
        self.automark = db_row['automark'] or 'pending'
        self.autorefresh = stringtools.truthystring(db_row['autorefresh'])
+        self.ignore_shorts = bool(db_row['ignore_shorts'])

    def __repr__(self):
        return f'Channel:{self.id}'
@ -139,7 +140,7 @@ class Channel(ObjectBase):
        log.info('Deleting %s.', self)

        self.ycdldb.delete(table='videos', pairs={'author_id': self.id})
-        self.ycdldb.delete(table='channels', pairs={'id': self.id})
+        self.ycdldb.delete(table=Channel, pairs={'id': self.id})
        self.deleted = True

    def get_most_recent_video_id(self) -> str:
@ -249,7 +250,7 @@ class Channel(ObjectBase):
            'id': self.id,
            'last_refresh': timetools.now().timestamp(),
        }
-        self.ycdldb.update(table='channels', pairs=pairs, where_key='id')
+        self.ycdldb.update(table=Channel, pairs=pairs, where_key='id')

    def reset_uploads_playlist_id(self):
        '''
@ -267,7 +268,7 @@ class Channel(ObjectBase):
            'id': self.id,
            'automark': state,
        }
-        self.ycdldb.update(table='channels', pairs=pairs, where_key='id')
+        self.ycdldb.update(table=Channel, pairs=pairs, where_key='id')
        self.automark = state

    @worms.atomic
@ -278,7 +279,7 @@ class Channel(ObjectBase):
            'id': self.id,
            'autorefresh': autorefresh,
        }
-        self.ycdldb.update(table='channels', pairs=pairs, where_key='id')
+        self.ycdldb.update(table=Channel, pairs=pairs, where_key='id')
        self.autorefresh = autorefresh

    @worms.atomic
@ -289,9 +290,21 @@ class Channel(ObjectBase):
            'id': self.id,
            'download_directory': download_directory.absolute_path if download_directory else None,
        }
-        self.ycdldb.update(table='channels', pairs=pairs, where_key='id')
+        self.ycdldb.update(table=Channel, pairs=pairs, where_key='id')
        self.download_directory = download_directory

+    @worms.atomic
+    def set_ignore_shorts(self, ignore_shorts: bool):
+        if not isinstance(ignore_shorts, bool):
+            raise TypeError(ignore_shorts)
+
+        pairs = {
+            'id': self.id,
+            'ignore_shorts': int(ignore_shorts)
+        }
+        self.ycdldb.update(table=Channel, pairs=pairs, where_key='id')
+        self.ignore_shorts = ignore_shorts
+
    @worms.atomic
    def set_name(self, name):
        name = self.normalize_name(name)
@ -300,7 +313,7 @@ class Channel(ObjectBase):
            'id': self.id,
            'name': name,
        }
-        self.ycdldb.update(table='channels', pairs=pairs, where_key='id')
+        self.ycdldb.update(table=Channel, pairs=pairs, where_key='id')
        self.name = name

    @worms.atomic
@ -311,7 +324,7 @@ class Channel(ObjectBase):
            'id': self.id,
            'queuefile_extension': queuefile_extension,
        }
-        self.ycdldb.update(table='channels', pairs=pairs, where_key='id')
+        self.ycdldb.update(table=Channel, pairs=pairs, where_key='id')
        self.queuefile_extension = queuefile_extension

    @worms.atomic
@ -324,7 +337,7 @@ class Channel(ObjectBase):
            'id': self.id,
            'uploads_playlist': playlist_id,
        }
-        self.ycdldb.update(table='channels', pairs=pairs, where_key='id')
+        self.ycdldb.update(table=Channel, pairs=pairs, where_key='id')
        self.uploads_playlist = playlist_id

 class Video(ObjectBase):
--- a/ycdl/ycdldb.py
+++ b/ycdl/ycdldb.py
@ -30,6 +30,7 @@ class YCDLDBChannelMixin:
            *,
            automark='pending',
            download_directory=None,
+            ignore_shorts=True,
            queuefile_extension=None,
            get_videos=False,
            name=None,
@ -67,6 +68,7 @@ class YCDLDBChannelMixin:
            'queuefile_extension': queuefile_extension,
            'automark': automark,
            'autorefresh': True,
+            'ignore_shorts': int(bool(ignore_shorts)),
        }
        self.insert(table='channels', pairs=data)

@ -393,14 +395,15 @@ class YCDLDBVideoMixin:
            'thumbnail': video.thumbnail['url'],
            'live_broadcast': video.live_broadcast,
            'state': download_status,
+            'is_shorts': None,
        }

        if existing:
            log.loud('Updating Video %s.', video)
-            self.update(table='videos', pairs=data, where_key='id')
+            self.update(objects.Video, pairs=data, where_key='id')
        else:
            log.loud('Inserting Video %s.', video)
-            self.insert(table='videos', pairs=data)
+            self.insert(objects.Video, pairs=data)

        # Override the cached copy with the new copy so that the cache contains
        # updated information (view counts etc.).
--- a/ycdl/ytapi.py
+++ b/ycdl/ytapi.py
@ -1,10 +1,16 @@
 import googleapiclient.discovery
 import isodate
+import requests
 import typing

 from voussoirkit import gentools
+from voussoirkit import httperrors
 from voussoirkit import vlogging

+log = vlogging.getLogger(__name__)
+
+session = requests.Session()
+
 def int_none(x):
    if x is None:
        return None
@ -60,7 +66,6 @@ class Youtube:
            serviceName='youtube',
            version='v3',
        )
-        self.log = vlogging.getLogger(__name__)

    def _playlist_paginator(self, playlist_id):
        page_token = None
@ -131,21 +136,35 @@ class Youtube:
        chunks = gentools.chunk_generator(video_ids, 50)
        total_snippets = 0
        for chunk in chunks:
-            self.log.debug('Requesting batch of %d video ids.', len(chunk))
-            self.log.loud(chunk)
+            log.debug('Requesting batch of %d video ids.', len(chunk))
+            log.loud(chunk)
            chunk = ','.join(chunk)
            data = self.youtube.videos().list(
                part='id,contentDetails,snippet,statistics',
                id=chunk,
            ).execute()
            snippets = data['items']
-            self.log.debug('Got batch of %d snippets.', len(snippets))
+            log.debug('Got batch of %d snippets.', len(snippets))
            total_snippets += len(snippets)
-            self.log.loud(snippets)
+            log.loud(snippets)
            for snippet in snippets:
+                log.loud('%s', snippet)
                try:
                    video = Video(snippet)
                    yield video
                except KeyError as exc:
-                    self.log.warning(f'KEYERROR: {exc} not in {snippet}')
-        self.log.debug('Finished getting a total of %d snippets.', total_snippets)
+                    log.warning(f'KEYERROR: {exc} not in {snippet}')
+        log.debug('Finished getting a total of %d snippets.', total_snippets)
+
+def video_is_shorts(video_id) -> bool:
+    url = f'https://www.youtube.com/shorts/{video_id}'
+    log.loud('Checking if %s is shorts.', video_id)
+    response = session.head(url)
+    httperrors.raise_for_status(response)
+
+    if response.status_code == 200:
+        return True
+    elif response.status_code == 303:
+        return False
+
+    raise ValueError('Unexpected status code %s', response.status_code)