From 60a153fba72153d565af6973244e24fa9e82367b Mon Sep 17 00:00:00 2001 From: Ethan Dalool Date: Tue, 21 Nov 2017 21:00:02 -0800 Subject: [PATCH] Improve division of labor with _as_generator, add some debugs. Previously, `livestream` was doing validation and building the database and then passing it to `_as_a_generator`. Since that's the only place those things are used, just move them into there leaving `livestream` looking more simple. --- timesearch/livestream.py | 83 ++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 45 deletions(-) diff --git a/timesearch/livestream.py b/timesearch/livestream.py index acb5928..5a7307e 100644 --- a/timesearch/livestream.py +++ b/timesearch/livestream.py @@ -9,7 +9,6 @@ from . import tsdb def livestream( subreddit=None, username=None, - verbose=False, as_a_generator=False, do_submissions=True, do_comments=True, @@ -18,40 +17,21 @@ def livestream( sleepy=30, ): ''' - Continuously get posts from this source - and insert them into the database + Continuously get posts from this source and insert them into the database. as_a_generator: return a generator where every iteration does a single livestream loop. This is good if you want to manage multiple livestreams yourself by calling `next` on each of them, instead of getting stuck in here. ''' - if bool(subreddit) == bool(username): - raise Exception('Require either username / subreddit parameter, but not both') - if bool(do_submissions) is bool(do_comments) is False: - raise Exception('Require do_submissions and/or do_comments parameter') - common.bot.login(common.r) - - if subreddit: - print('Getting subreddit %s' % subreddit) - database = tsdb.TSDB.for_subreddit(subreddit) - subreddit = common.r.subreddit(subreddit) - submissions = subreddit.new if do_submissions else None - comments = subreddit.comments if do_comments else None - else: - print('Getting redditor %s' % username) - database = tsdb.TSDB.for_user(username) - user = common.r.redditor(username) - submissions = user.submissions.new if do_submissions else None - comments = user.comments.new if do_comments else None generator = _livestream_as_a_generator( - database, - submission_function=submissions, - comment_function=comments, + subreddit=subreddit, + username=username, + do_submissions=do_submissions, + do_comments=do_comments, limit=limit, params={'show': 'all'}, - verbose=verbose, ) if as_a_generator: return generator @@ -62,15 +42,13 @@ def livestream( newtext = '%ds, %dc' % (step['new_submissions'], step['new_comments']) totalnew = step['new_submissions'] + step['new_comments'] status = '{now} +{new}'.format(now=common.human(common.get_now()), new=newtext) - print(status, end='', flush=True) - if totalnew == 0 and verbose is False: + print(status, end='') + if totalnew == 0 and common.log.level != common.logging.DEBUG: # Since there were no news, allow the next line to overwrite status - print('\r', end='') + print('\r', end='', flush=True) else: print() - if verbose: - print('Loop finished.') if only_once: break time.sleep(sleepy) @@ -92,22 +70,40 @@ hangman = lambda: livestream( ) def _livestream_as_a_generator( - database, - submission_function, - comment_function, + subreddit, + username, + do_submissions, + do_comments, limit, params, - verbose, ): + + if bool(subreddit) == bool(username): + raise Exception('Require either username / subreddit parameter, but not both') + if bool(do_submissions) is bool(do_comments) is False: + raise Exception('Require do_submissions and/or do_comments parameter') + common.bot.login(common.r) + + if subreddit: + common.log.debug('Getting subreddit %s', subreddit) + database = tsdb.TSDB.for_subreddit(subreddit) + subreddit = common.r.subreddit(subreddit) + submission_function = subreddit.new if do_submissions else None + comment_function = subreddit.comments if do_comments else None + else: + common.log.debug('Getting redditor %s', username) + database = tsdb.TSDB.for_user(username) + user = common.r.redditor(username) + submission_function = user.submissions.new if do_submissions else None + comment_function = user.comments.new if do_comments else None + while True: - #common.r.handler.clear_cache() try: items = _livestream_helper( submission_function=submission_function, comment_function=comment_function, limit=limit, params=params, - verbose=verbose, ) newitems = database.insert(items) yield newitems @@ -116,11 +112,9 @@ def _livestream_as_a_generator( print('Retrying in 5...') time.sleep(5) - def _livestream_helper( submission_function=None, comment_function=None, - verbose=False, *args, **kwargs, ): @@ -135,22 +129,22 @@ def _livestream_helper( results = [] if submission_function: - if verbose: - print('Getting submissions', args, kwargs) + common.log.debug('Getting submissions %s %s', args, kwargs) this_kwargs = copy.deepcopy(kwargs) submission_batch = submission_function(*args, **this_kwargs) results.extend(submission_batch) if comment_function: - if verbose: - print('Getting comments', args, kwargs) + common.log.debug('Getting comments %s %s', args, kwargs) this_kwargs = copy.deepcopy(kwargs) comment_batch = comment_function(*args, **this_kwargs) results.extend(comment_batch) - if verbose: - print('Collected. Saving...') + common.log.debug('Got %d posts', len(results)) return results def livestream_argparse(args): + if args.verbose: + common.log.setLevel(common.logging.DEBUG) + if args.submissions is args.comments is False: args.submissions = True args.comments = True @@ -169,7 +163,6 @@ def livestream_argparse(args): do_comments=args.comments, do_submissions=args.submissions, limit=limit, - verbose=args.verbose, only_once=args.once, sleepy=common.int_none(args.sleepy), )