From 4911cee9106bf13074e4aadf64c430d608d4b009 Mon Sep 17 00:00:00 2001 From: Ethan Dalool Date: Wed, 1 Jul 2020 17:53:11 -0700 Subject: [PATCH] Use generators instead of lists for loading / searching videos. Previously, creating the list meant that all database rows would be loaded and objected even if you had a limit in place on your page query, which was a huge waste. --- .../ycdl_flask/backend/endpoints/channel_endpoints.py | 5 ++++- ycdl/ycdldb.py | 9 +++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/frontends/ycdl_flask/backend/endpoints/channel_endpoints.py b/frontends/ycdl_flask/backend/endpoints/channel_endpoints.py index 9cb3a7a..f6cfbbc 100644 --- a/frontends/ycdl_flask/backend/endpoints/channel_endpoints.py +++ b/frontends/ycdl_flask/backend/endpoints/channel_endpoints.py @@ -1,5 +1,6 @@ import datetime import flask; from flask import request +import itertools import traceback import ycdl @@ -50,10 +51,12 @@ def get_channel(channel_id=None, download_filter=None): if limit is not None: try: limit = int(limit) - videos = videos[:limit] + videos = itertools.islice(videos, limit) except ValueError: pass + videos = list(videos) + for video in videos: published = video.published published = datetime.datetime.utcfromtimestamp(published) diff --git a/ycdl/ycdldb.py b/ycdl/ycdldb.py index abf5737..36a8511 100644 --- a/ycdl/ycdldb.py +++ b/ycdl/ycdldb.py @@ -307,9 +307,14 @@ class YCDLDBVideoMixin: orderbys = ' ORDER BY ' + orderbys query = 'SELECT * FROM videos' + wheres + orderbys + + print(query, bindings) + explain = self.sql_execute('EXPLAIN QUERY PLAN ' + query, bindings) + print('\n'.join(str(x) for x in explain.fetchall())) + rows = self.sql_select(query, bindings) - videos = [self.get_cached_instance('video', row) for row in rows] - return videos + for row in rows: + yield self.get_cached_instance('video', row) def insert_playlist(self, playlist_id, commit=True): video_generator = self.youtube.get_playlist_videos(playlist_id)