From 6068d644b98db93e436a0e64f36031b287ab4eab Mon Sep 17 00:00:00 2001
From: Ethan Dalool <git@voussoir.net>
Date: Sun, 5 Sep 2021 01:26:34 -0700
Subject: [PATCH] Add autorefresh to Channel to include/exclude in refresh_all
 loop.

---
 .../backend/endpoints/channel_endpoints.py    | 14 +++++++++
 frontends/ycdl_flask/static/js/api.js         |  8 +++++
 frontends/ycdl_flask/templates/channel.html   | 23 ++++++++++++++
 frontends/ycdl_flask/templates/channels.html  |  5 ++-
 utilities/database_upgrader.py                | 31 +++++++++++++++++++
 ycdl/constants.py                             |  5 +--
 ycdl/objects.py                               | 21 +++++++++++++
 ycdl/ycdldb.py                                | 13 +++++---
 8 files changed, 112 insertions(+), 8 deletions(-)

diff --git a/frontends/ycdl_flask/backend/endpoints/channel_endpoints.py b/frontends/ycdl_flask/backend/endpoints/channel_endpoints.py
index 225f0d0..adc1342 100644
--- a/frontends/ycdl_flask/backend/endpoints/channel_endpoints.py
+++ b/frontends/ycdl_flask/backend/endpoints/channel_endpoints.py
@@ -159,6 +159,20 @@ def post_set_automark(channel_id):
 
     return flasktools.make_json_response({})
 
+@flasktools.required_fields(['autorefresh'], forbid_whitespace=True)
+@site.route('/channel/<channel_id>/set_autorefresh', methods=['POST'])
+def post_set_autorefresh(channel_id):
+    autorefresh = request.form['autorefresh']
+    channel = common.ycdldb.get_channel(channel_id)
+
+    try:
+        autorefresh = stringtools.truthystring(autorefresh)
+        channel.set_autorefresh(autorefresh)
+    except (ValueError, TypeError):
+        flask.abort(400)
+
+    return flasktools.make_json_response({})
+
 @site.route('/channel/<channel_id>/set_download_directory', methods=['POST'])
 def post_set_download_directory(channel_id):
     download_directory = request.form['download_directory']
diff --git a/frontends/ycdl_flask/static/js/api.js b/frontends/ycdl_flask/static/js/api.js
index c887dee..3cdba57 100644
--- a/frontends/ycdl_flask/static/js/api.js
+++ b/frontends/ycdl_flask/static/js/api.js
@@ -43,6 +43,14 @@ function set_automark(channel_id, state, callback)
     return common.post(url, data, callback);
 }
 
+api.channels.set_autorefresh =
+function set_autorefresh(channel_id, autorefresh, callback)
+{
+    const url = `/channel/${channel_id}/set_autorefresh`;
+    const data = {"autorefresh": autorefresh};
+    return common.post(url, data, callback);
+}
+
 api.channels.set_download_directory =
 function set_download_directory(channel_id, download_directory, callback)
 {
diff --git a/frontends/ycdl_flask/templates/channel.html b/frontends/ycdl_flask/templates/channel.html
index 9f74766..ec31943 100644
--- a/frontends/ycdl_flask/templates/channel.html
+++ b/frontends/ycdl_flask/templates/channel.html
@@ -259,6 +259,11 @@ https://stackoverflow.com/a/35153397
     </div> <!-- tab-videos -->
 
     <div class="tab" data-tab-title="Settings">
+    <div>
+        <label><input type="checkbox" id="set_autorefresh_checkbox" {{"checked" if channel.autorefresh else ""}} onchange="return set_autorefresh_form(event);"/> Automatically refresh this channel regularly.</label>
+        <span id="set_autorefresh_spinner" class="hidden">Working...</span>
+    </div>
+
     <div>
         New videos are:
         <select onchange="return set_automark_form(event);">
@@ -615,6 +620,21 @@ function set_automark_callback(response)
     set_automark_spinner.hide();
 }
 
+function set_autorefresh_form(event)
+{
+    set_autorefresh_spinner.show();
+    api.channels.set_autorefresh(CHANNEL_ID, event.target.checked, set_autorefresh_callback);
+}
+
+function set_autorefresh_callback(response)
+{
+    if (response.meta.status != 200)
+    {
+        alert(JSON.stringify(response));
+    }
+    set_autorefresh_spinner.hide();
+}
+
 function set_download_directory_form(event)
 {
     const download_directory = set_download_directory_input.value.trim();
@@ -653,6 +673,9 @@ if (CHANNEL_ID)
 
     var set_automark_spinner = document.getElementById("set_automark_spinner");
     set_automark_spinner = new spinner.Spinner(set_automark_spinner);
+
+    var set_autorefresh_spinner = document.getElementById("set_autorefresh_spinner");
+    set_autorefresh_spinner = new spinner.Spinner(set_autorefresh_spinner);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/frontends/ycdl_flask/templates/channels.html b/frontends/ycdl_flask/templates/channels.html
index 1999d2c..220082a 100644
--- a/frontends/ycdl_flask/templates/channels.html
+++ b/frontends/ycdl_flask/templates/channels.html
@@ -63,7 +63,10 @@
     {% endif %}
         <a href="/channel/{{channel.id}}">{{channel.name}}</a> <a href="/channel/{{channel.id}}/pending">(p)</a>
         {% if channel.automark not in [none, "pending"] %}
-        <span>automark: {{channel.automark}}</span>
+        <span>(automark: {{channel.automark}})</span>
+        {% endif %}
+        {% if not channel.autorefresh %}
+        <span>(autorefresh: no)</span>
         {% endif %}
     </div>
     {% endfor %}
diff --git a/utilities/database_upgrader.py b/utilities/database_upgrader.py
index a90d861..59f7d2a 100644
--- a/utilities/database_upgrader.py
+++ b/utilities/database_upgrader.py
@@ -284,6 +284,37 @@ def upgrade_8_to_9(ycdldb):
 
     m.go()
 
+def upgrade_9_to_10(ycdldb):
+    '''
+    In this version, the `autorefresh` column was added to the channels table.
+    '''
+    m = Migrator(ycdldb)
+
+    m.tables['channels']['create'] = '''
+    CREATE TABLE IF NOT EXISTS channels(
+        id TEXT,
+        name TEXT,
+        uploads_playlist TEXT,
+        download_directory TEXT COLLATE NOCASE,
+        queuefile_extension TEXT COLLATE NOCASE,
+        automark TEXT,
+        autorefresh INT
+    );
+    '''
+    m.tables['channels']['transfer'] = '''
+    INSERT INTO channels SELECT
+        id,
+        name,
+        uploads_playlist,
+        download_directory,
+        queuefile_extension,
+        automark,
+        1
+    FROM channels_old;
+    '''
+
+    m.go()
+
 def upgrade_all(data_directory):
     '''
     Given the directory containing a ycdl database, apply all of the
diff --git a/ycdl/constants.py b/ycdl/constants.py
index 9a7f117..430bf6b 100644
--- a/ycdl/constants.py
+++ b/ycdl/constants.py
@@ -1,6 +1,6 @@
 from voussoirkit import sqlhelpers
 
-DATABASE_VERSION = 9
+DATABASE_VERSION = 10
 DB_VERSION_PRAGMA = f'''
 PRAGMA user_version = {DATABASE_VERSION};
 '''
@@ -21,7 +21,8 @@ CREATE TABLE IF NOT EXISTS channels(
     uploads_playlist TEXT,
     download_directory TEXT COLLATE NOCASE,
     queuefile_extension TEXT COLLATE NOCASE,
-    automark TEXT
+    automark TEXT,
+    autorefresh INT
 );
 CREATE TABLE IF NOT EXISTS videos(
     id TEXT,
diff --git a/ycdl/objects.py b/ycdl/objects.py
index 552ef7e..735a580 100644
--- a/ycdl/objects.py
+++ b/ycdl/objects.py
@@ -2,6 +2,7 @@ import datetime
 import typing
 
 from voussoirkit import pathclass
+from voussoirkit import stringtools
 
 from . import constants
 from . import exceptions
@@ -42,6 +43,7 @@ class Channel(Base):
         )
         self.queuefile_extension = self.normalize_queuefile_extension(db_row['queuefile_extension'])
         self.automark = db_row['automark'] or 'pending'
+        self.autorefresh = stringtools.truthystring(db_row['autorefresh'])
 
     def __repr__(self):
         return f'Channel:{self.id}'
@@ -204,6 +206,25 @@ class Channel(Base):
         if commit:
             self.ycdldb.commit()
 
+    def set_autorefresh(self, autorefresh, commit=True):
+        if isinstance(autorefresh, int):
+            if autorefresh not in {0, 1}:
+                raise ValueError(f'autorefresh should be a boolean, not {autorefresh}.')
+            autorefresh = bool(autorefresh)
+
+        if not isinstance(autorefresh, bool):
+            raise TypeError(f'autorefresh should be a boolean, not {autorefresh}.')
+
+        pairs = {
+            'id': self.id,
+            'autorefresh': autorefresh,
+        }
+        self.ycdldb.sql_update(table='channels', pairs=pairs, where_key='id')
+        self.autorefresh = autorefresh
+
+        if commit:
+            self.ycdldb.commit()
+
     def set_download_directory(self, download_directory, commit=True):
         download_directory = self.normalize_download_directory(download_directory)
 
diff --git a/ycdl/ycdldb.py b/ycdl/ycdldb.py
index 4462da2..ee50497 100644
--- a/ycdl/ycdldb.py
+++ b/ycdl/ycdldb.py
@@ -146,7 +146,8 @@ class YCDLDBChannelMixin:
             'uploads_playlist': self.youtube.get_user_uploads_playlist_id(channel_id),
             'download_directory': download_directory.absolute_path if download_directory else None,
             'queuefile_extension': queuefile_extension,
-            'automark': "pending",
+            'automark': 'pending',
+            'autorefresh': True,
         }
         self.sql_insert(table='channels', data=data)
 
@@ -169,7 +170,7 @@ class YCDLDBChannelMixin:
     def get_channels_by_sql(self, query, bindings=None):
         return self.get_things_by_sql('channel', query, bindings)
 
-    def _rss_assisted_refresh(self, skip_failures=False, commit=True):
+    def _rss_assisted_refresh(self, channels, skip_failures=False, commit=True):
         '''
         Youtube provides RSS feeds for every channel. These feeds do not
         require the API token and seem to have generous ratelimits, or
@@ -216,7 +217,7 @@ class YCDLDBChannelMixin:
                 )
                 traditional(channel)
 
-        new_ids = (id for channel in self.get_channels() for id in assisted(channel))
+        new_ids = (id for channel in channels for id in assisted(channel))
         for video in self.youtube.get_videos(new_ids):
             self.ingest_video(video, commit=False)
 
@@ -235,11 +236,13 @@ class YCDLDBChannelMixin:
         ):
         self.log.info('Refreshing all channels.')
 
+        channels = self.get_channels_by_sql('SELECT * FROM channels WHERE autorefresh == 1')
+
         if rss_assisted and not force:
-            return self._rss_assisted_refresh(skip_failures=skip_failures, commit=commit)
+            return self._rss_assisted_refresh(channels, skip_failures=skip_failures, commit=commit)
 
         excs = []
-        for channel in self.get_channels():
+        for channel in channels:
             try:
                 channel.refresh(force=force, commit=commit)
             except Exception as exc: