voussoir/ycdl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Improve usage of generators in ytapi.

Browse source
This commit is contained in:
voussoir 2021-03-31 19:35:13 -07:00
parent 1626e13c08
commit 75a17a6361
No known key found for this signature in database
GPG key ID: 5F7554F8C26DACCB
2 changed files with 18 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
ycdl/objects.py
View file

@ -88,7 +88,8 @@ class Channel(Base):
else: else:
try: try:
video_generator = self._rss_assisted_videos() video_generator = self._rss_assisted_videos()
except exceptions.RSSAssistFailed: except exceptions.RSSAssistFailed as exc:
self.ycdldb.log.debug('Caught %s.', exc)
video_generator = self.ycdldb.youtube.get_playlist_videos(self.uploads_playlist) video_generator = self.ycdldb.youtube.get_playlist_videos(self.uploads_playlist)
seen_ids = set() seen_ids = set()
@ -96,7 +97,7 @@ class Channel(Base):
seen_ids.add(video.id) seen_ids.add(video.id)
status = self.ycdldb.ingest_video(video, commit=False) status = self.ycdldb.ingest_video(video, commit=False)
if not (force or status['new']): if (not status['new']) and (not force):
break break
# Now we will refresh some other IDs that may not have been refreshed # Now we will refresh some other IDs that may not have been refreshed

42
ycdl/ytapi.py
View file

@ -80,9 +80,7 @@ class Youtube:
paginator = self._playlist_paginator(playlist_id) paginator = self._playlist_paginator(playlist_id)
video_ids = (item['contentDetails']['videoId'] for item in paginator) video_ids = (item['contentDetails']['videoId'] for item in paginator)
videos = self.get_videos(video_ids) videos = self.get_videos(video_ids)
videos.sort(key=lambda x: x.published, reverse=True) return videos
yield from videos
def get_related_videos(self, video_id, count=50): def get_related_videos(self, video_id, count=50):
if isinstance(video_id, Video): if isinstance(video_id, Video):
@ -121,15 +119,13 @@ class Youtube:
yield from self.get_playlist_videos(self.get_user_uploads_playlist_id(uid)) yield from self.get_playlist_videos(self.get_user_uploads_playlist_id(uid))
def get_video(self, video_id): def get_video(self, video_id):
videos = self.get_videos([video_id]) try:
video = next(self.get_videos([video_id]))
if len(videos) == 1: return video
return videos[0] except StopIteration:
elif len(videos) == 0: raise VideoNotFound(video_id) from None
raise VideoNotFound(video_id)
def get_videos(self, video_ids): def get_videos(self, video_ids):
snippets = []
chunks = gentools.chunk_generator(video_ids, 50) chunks = gentools.chunk_generator(video_ids, 50)
for chunk in chunks: for chunk in chunks:
self.log.debug('Requesting batch of %d video ids.', len(chunk)) self.log.debug('Requesting batch of %d video ids.', len(chunk))
@ -139,20 +135,12 @@ class Youtube:
part='id,contentDetails,snippet,statistics', part='id,contentDetails,snippet,statistics',
id=chunk, id=chunk,
).execute() ).execute()
items = data['items'] snippets = data['items']
self.log.debug('Got %d snippets.', len(items)) self.log.debug('Got %d snippets.', len(snippets))
self.log.loud(items) self.log.loud(snippets)
snippets.extend(items) for snippet in snippets:
try:
videos = [] video = Video(snippet)
broken = [] yield video
for snippet in snippets: except KeyError as exc:
try: self.log.warning(f'KEYERROR: {exc} not in {snippet}')
videos.append(Video(snippet))
except KeyError as exc:
print(f'KEYERROR: {exc} not in {snippet}')
broken.append(snippet)
if broken:
# print('broken:', broken)
pass
return videos