Some small whitespace and log tweaks.
This commit is contained in:
parent
55d9c064bd
commit
e2d1f2b9ef
3 changed files with 4 additions and 4 deletions
|
@ -5,7 +5,6 @@ from . import exceptions
|
|||
from . import pushshift
|
||||
from . import tsdb
|
||||
|
||||
|
||||
def get_comments(
|
||||
subreddit=None,
|
||||
username=None,
|
||||
|
@ -74,12 +73,11 @@ def get_comments(
|
|||
elif username:
|
||||
comments = pushshift.get_comments_from_user(username, lower=lower, upper=upper)
|
||||
|
||||
|
||||
if do_supplement:
|
||||
comments = pushshift.supplement_reddit_data(comments, chunk_size=100)
|
||||
comments = common.generator_chunker(comments, 500)
|
||||
|
||||
form = '{lower} ({lower_unix}) - {upper} ({upper_unix}) +{gain}'
|
||||
comments = common.generator_chunker(comments, 500)
|
||||
for chunk in comments:
|
||||
step = database.insert(chunk)
|
||||
message = form.format(
|
||||
|
@ -90,6 +88,7 @@ def get_comments(
|
|||
gain=step['new_comments'],
|
||||
)
|
||||
print(message)
|
||||
|
||||
if specific_submission:
|
||||
query = '''
|
||||
UPDATE submissions
|
||||
|
|
|
@ -238,6 +238,7 @@ def supplement_reddit_data(dummies, chunk_size=100):
|
|||
'''
|
||||
chunks = common.generator_chunker(dummies, chunk_size)
|
||||
for chunk in chunks:
|
||||
common.log.debug('Supplementing %d items with live reddit data.', len(chunk))
|
||||
ids = [item.fullname for item in chunk]
|
||||
live_copies = list(common.r.info(ids))
|
||||
live_copies = {item.fullname: item for item in live_copies}
|
||||
|
|
|
@ -346,7 +346,7 @@ class TSDB:
|
|||
new_values[key] += status
|
||||
|
||||
if commit:
|
||||
common.log.debug('Committing insert')
|
||||
common.log.debug('Committing insert.')
|
||||
self.sql.commit()
|
||||
|
||||
common.log.debug('Done inserting.')
|
||||
|
|
Loading…
Reference in a new issue