Add some more docstrings here and there.
This commit is contained in:
parent
07ca340c80
commit
b5160cb6c4
3 changed files with 46 additions and 13 deletions
|
@ -64,6 +64,10 @@ def fetchgenerator(cursor):
|
||||||
yield item
|
yield item
|
||||||
|
|
||||||
def generator_chunker(generator, chunk_size):
|
def generator_chunker(generator, chunk_size):
|
||||||
|
'''
|
||||||
|
Given an item generator, yield lists of length chunk_size, except maybe
|
||||||
|
the last one.
|
||||||
|
'''
|
||||||
chunk = []
|
chunk = []
|
||||||
for item in generator:
|
for item in generator:
|
||||||
chunk.append(item)
|
chunk.append(item)
|
||||||
|
|
|
@ -7,7 +7,22 @@ from . import exceptions
|
||||||
from . import tsdb
|
from . import tsdb
|
||||||
|
|
||||||
|
|
||||||
|
def _listify(x):
|
||||||
|
'''
|
||||||
|
The user may have given us a string containing multiple subreddits / users.
|
||||||
|
Try to split that up into a list of names.
|
||||||
|
'''
|
||||||
|
if not x:
|
||||||
|
return []
|
||||||
|
if isinstance(x, str):
|
||||||
|
return common.split_any(x, ['+', ' ', ','])
|
||||||
|
return x
|
||||||
|
|
||||||
def generator_printer(generator):
|
def generator_printer(generator):
|
||||||
|
'''
|
||||||
|
Given a generator that produces livestream update steps, print them out.
|
||||||
|
This yields None because print returns None.
|
||||||
|
'''
|
||||||
prev_message_length = 0
|
prev_message_length = 0
|
||||||
for step in generator:
|
for step in generator:
|
||||||
newtext = '%s: +%ds, %dc' % (step['tsdb'].filepath.basename, step['new_submissions'], step['new_comments'])
|
newtext = '%s: +%ds, %dc' % (step['tsdb'].filepath.basename, step['new_submissions'], step['new_comments'])
|
||||||
|
@ -24,8 +39,16 @@ def generator_printer(generator):
|
||||||
yield None
|
yield None
|
||||||
|
|
||||||
def cycle_generators(generators, only_once, sleepy):
|
def cycle_generators(generators, only_once, sleepy):
|
||||||
|
'''
|
||||||
|
Given multiple generators, yield an item from each one, cycling through
|
||||||
|
them in a round-robin fashion.
|
||||||
|
|
||||||
|
This is useful if you want to convert multiple livestream generators into a
|
||||||
|
single generator that take turns updating each of them and yields all of
|
||||||
|
their items.
|
||||||
|
'''
|
||||||
while True:
|
while True:
|
||||||
for (index, generator) in enumerate(generators):
|
for generator in generators:
|
||||||
yield next(generator)
|
yield next(generator)
|
||||||
if only_once:
|
if only_once:
|
||||||
break
|
break
|
||||||
|
@ -45,18 +68,12 @@ def livestream(
|
||||||
Continuously get posts from this source and insert them into the database.
|
Continuously get posts from this source and insert them into the database.
|
||||||
|
|
||||||
as_a_generator:
|
as_a_generator:
|
||||||
return a generator where every iteration does a single livestream loop.
|
Return a generator where every iteration does a single livestream loop
|
||||||
This is good if you want to manage multiple livestreams yourself by
|
and yields the return value of TSDB.insert (A summary of new
|
||||||
calling `next` on each of them, instead of getting stuck in here.
|
submission & comment count).
|
||||||
|
This is useful if you want to manage the generator yourself.
|
||||||
|
Otherwise, this function will run the generator forever.
|
||||||
'''
|
'''
|
||||||
|
|
||||||
def _listify(x):
|
|
||||||
if x is None:
|
|
||||||
return []
|
|
||||||
if isinstance(x, str):
|
|
||||||
return common.split_any(x, ['+', ' ', ','])
|
|
||||||
return x
|
|
||||||
|
|
||||||
subreddits = _listify(subreddit)
|
subreddits = _listify(subreddit)
|
||||||
usernames = _listify(username)
|
usernames = _listify(username)
|
||||||
kwargs = {
|
kwargs = {
|
||||||
|
@ -65,6 +82,7 @@ def livestream(
|
||||||
'limit': limit,
|
'limit': limit,
|
||||||
'params': {'show': 'all'},
|
'params': {'show': 'all'},
|
||||||
}
|
}
|
||||||
|
|
||||||
subreddit_generators = [
|
subreddit_generators = [
|
||||||
_livestream_as_a_generator(subreddit=subreddit, username=None, **kwargs) for subreddit in subreddits
|
_livestream_as_a_generator(subreddit=subreddit, username=None, **kwargs) for subreddit in subreddits
|
||||||
]
|
]
|
||||||
|
@ -72,12 +90,13 @@ def livestream(
|
||||||
_livestream_as_a_generator(subreddit=None, username=username, **kwargs) for username in usernames
|
_livestream_as_a_generator(subreddit=None, username=username, **kwargs) for username in usernames
|
||||||
]
|
]
|
||||||
generators = subreddit_generators + user_generators
|
generators = subreddit_generators + user_generators
|
||||||
|
|
||||||
if as_a_generator:
|
if as_a_generator:
|
||||||
if len(generators) == 1:
|
if len(generators) == 1:
|
||||||
return generators[0]
|
return generators[0]
|
||||||
return generators
|
return generators
|
||||||
|
|
||||||
generator = cycle_generators(generators, only_once, sleepy)
|
generator = cycle_generators(generators, only_once=only_once, sleepy=sleepy)
|
||||||
generator = generator_printer(generator)
|
generator = generator_printer(generator)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
|
@ -274,6 +274,9 @@ def html_from_tree(tree, sort=None):
|
||||||
return page
|
return page
|
||||||
|
|
||||||
def html_helper_permalink(item):
|
def html_helper_permalink(item):
|
||||||
|
'''
|
||||||
|
Given a submission or a comment, return an <a> tag for its permalink.
|
||||||
|
'''
|
||||||
link = 'https://www.reddit.com/r/%s/comments/' % item.subreddit
|
link = 'https://www.reddit.com/r/%s/comments/' % item.subreddit
|
||||||
if item.object_type == 'submission':
|
if item.object_type == 'submission':
|
||||||
link += item.idstr[3:]
|
link += item.idstr[3:]
|
||||||
|
@ -283,6 +286,10 @@ def html_helper_permalink(item):
|
||||||
return link
|
return link
|
||||||
|
|
||||||
def html_helper_urlortext(submission):
|
def html_helper_urlortext(submission):
|
||||||
|
'''
|
||||||
|
Given a submission, return either an <a> tag for its url, or its
|
||||||
|
markdown-rendered selftext.
|
||||||
|
'''
|
||||||
if submission.url:
|
if submission.url:
|
||||||
text = '<a href="{url}">{url}</a>'.format(url=submission.url)
|
text = '<a href="{url}">{url}</a>'.format(url=submission.url)
|
||||||
elif submission.selftext:
|
elif submission.selftext:
|
||||||
|
@ -293,6 +300,9 @@ def html_helper_urlortext(submission):
|
||||||
return text
|
return text
|
||||||
|
|
||||||
def html_helper_userlink(item):
|
def html_helper_userlink(item):
|
||||||
|
'''
|
||||||
|
Given a submission or comment, return an <a> tag for its author, or [deleted].
|
||||||
|
'''
|
||||||
name = item.author
|
name = item.author
|
||||||
if name.lower() == '[deleted]':
|
if name.lower() == '[deleted]':
|
||||||
return '[deleted]'
|
return '[deleted]'
|
||||||
|
|
Loading…
Reference in a new issue