voussoir
/
timesearch
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improve division of labor with _as_generator, add some debugs. 

Previously, `livestream` was doing validation and building the
database and then passing it to `_as_a_generator`.
Since that's the only place those things are used, just move them
into there leaving `livestream` looking more simple.
master
Ethan Dalool 2017-11-21 21:00:02 -08:00
parent d59c3d5310
commit 60a153fba7
1 changed files with 38 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
timesearch/livestream.py
View File

@ -9,7 +9,6 @@ from . import tsdb
def livestream(
subreddit=None,
username=None,
verbose=False,
as_a_generator=False,
do_submissions=True,
do_comments=True,
@ -18,40 +17,21 @@ def livestream(
sleepy=30,
):
'''
Continuously get posts from this source
and insert them into the database
Continuously get posts from this source and insert them into the database.
as_a_generator:
return a generator where every iteration does a single livestream loop.
This is good if you want to manage multiple livestreams yourself by
calling `next` on each of them, instead of getting stuck in here.
'''
if bool(subreddit) == bool(username):
raise Exception('Require either username / subreddit parameter, but not both')
if bool(do_submissions) is bool(do_comments) is False:
raise Exception('Require do_submissions and/or do_comments parameter')
common.bot.login(common.r)
if subreddit:
print('Getting subreddit %s' % subreddit)
database = tsdb.TSDB.for_subreddit(subreddit)
subreddit = common.r.subreddit(subreddit)
submissions = subreddit.new if do_submissions else None
comments = subreddit.comments if do_comments else None
else:
print('Getting redditor %s' % username)
database = tsdb.TSDB.for_user(username)
user = common.r.redditor(username)
submissions = user.submissions.new if do_submissions else None
comments = user.comments.new if do_comments else None
generator = _livestream_as_a_generator(
database,
submission_function=submissions,
comment_function=comments,
subreddit=subreddit,
username=username,
do_submissions=do_submissions,
do_comments=do_comments,
limit=limit,
params={'show': 'all'},
verbose=verbose,
)
if as_a_generator:
return generator
@ -62,15 +42,13 @@ def livestream(
newtext = '%ds, %dc' % (step['new_submissions'], step['new_comments'])
totalnew = step['new_submissions'] + step['new_comments']
status = '{now} +{new}'.format(now=common.human(common.get_now()), new=newtext)
print(status, end='', flush=True)
if totalnew == 0 and verbose is False:
print(status, end='')
if totalnew == 0 and common.log.level != common.logging.DEBUG:
# Since there were no news, allow the next line to overwrite status
print('\r', end='')
print('\r', end='', flush=True)
else:
print()
if verbose:
print('Loop finished.')
if only_once:
break
time.sleep(sleepy)
@ -92,22 +70,40 @@ hangman = lambda: livestream(
)
def _livestream_as_a_generator(
database,
submission_function,
comment_function,
subreddit,
username,
do_submissions,
do_comments,
limit,
params,
verbose,
):
if bool(subreddit) == bool(username):
raise Exception('Require either username / subreddit parameter, but not both')
if bool(do_submissions) is bool(do_comments) is False:
raise Exception('Require do_submissions and/or do_comments parameter')
common.bot.login(common.r)
if subreddit:
common.log.debug('Getting subreddit %s', subreddit)
database = tsdb.TSDB.for_subreddit(subreddit)
subreddit = common.r.subreddit(subreddit)
submission_function = subreddit.new if do_submissions else None
comment_function = subreddit.comments if do_comments else None
else:
common.log.debug('Getting redditor %s', username)
database = tsdb.TSDB.for_user(username)
user = common.r.redditor(username)
submission_function = user.submissions.new if do_submissions else None
comment_function = user.comments.new if do_comments else None
while True:
#common.r.handler.clear_cache()
try:
items = _livestream_helper(
submission_function=submission_function,
comment_function=comment_function,
limit=limit,
params=params,
verbose=verbose,
)
newitems = database.insert(items)
yield newitems
@ -116,11 +112,9 @@ def _livestream_as_a_generator(
print('Retrying in 5...')
time.sleep(5)
def _livestream_helper(
submission_function=None,
comment_function=None,
verbose=False,
*args,
**kwargs,
):
@ -135,22 +129,22 @@ def _livestream_helper(
results = []
if submission_function:
if verbose:
print('Getting submissions', args, kwargs)
common.log.debug('Getting submissions %s %s', args, kwargs)
this_kwargs = copy.deepcopy(kwargs)
submission_batch = submission_function(*args, **this_kwargs)
results.extend(submission_batch)
if comment_function:
if verbose:
print('Getting comments', args, kwargs)
common.log.debug('Getting comments %s %s', args, kwargs)
this_kwargs = copy.deepcopy(kwargs)
comment_batch = comment_function(*args, **this_kwargs)
results.extend(comment_batch)
if verbose:
print('Collected. Saving...')
common.log.debug('Got %d posts', len(results))
return results
def livestream_argparse(args):
if args.verbose:
common.log.setLevel(common.logging.DEBUG)
if args.submissions is args.comments is False:
args.submissions = True
args.comments = True
@ -169,7 +163,6 @@ def livestream_argparse(args):
do_comments=args.comments,
do_submissions=args.submissions,
limit=limit,
verbose=args.verbose,
only_once=args.once,
sleepy=common.int_none(args.sleepy),
)