Use generators instead of lists for loading / searching videos.

Previously, creating the list meant that all database rows would be
loaded and objected even if you had a limit in place on your page
query, which was a huge waste.

frontends/ycdl_flask/backend/endpoints/channel_endpoints.py

@@ -1,5 +1,6 @@
 import datetime
 import flask; from flask import request
+import itertools
 import traceback
 
 import ycdl
@@ -50,10 +51,12 @@ def get_channel(channel_id=None, download_filter=None):
     if limit is not None:
         try:
             limit = int(limit)
-            videos = videos[:limit]
+            videos = itertools.islice(videos, limit)
         except ValueError:
             pass
 
+    videos = list(videos)
+
     for video in videos:
         published = video.published
         published = datetime.datetime.utcfromtimestamp(published)

ycdl/ycdldb.py

@@ -307,9 +307,14 @@ class YCDLDBVideoMixin:
             orderbys = ' ORDER BY ' + orderbys
 
         query = 'SELECT * FROM videos' + wheres + orderbys
+
+        print(query, bindings)
+        explain = self.sql_execute('EXPLAIN QUERY PLAN ' + query, bindings)
+        print('\n'.join(str(x) for x in explain.fetchall()))
+
         rows = self.sql_select(query, bindings)
-        videos = [self.get_cached_instance('video', row) for row in rows]
-        return videos
+        for row in rows:
+            yield self.get_cached_instance('video', row)
 
     def insert_playlist(self, playlist_id, commit=True):
         video_generator = self.youtube.get_playlist_videos(playlist_id)