Some small whitespace and log tweaks.

master
Ethan Dalool 2020-10-08 14:17:19 -07:00
parent 55d9c064bd
commit e2d1f2b9ef
3 changed files with 4 additions and 4 deletions

View File

@ -5,7 +5,6 @@ from . import exceptions
from . import pushshift
from . import tsdb
def get_comments(
subreddit=None,
username=None,
@ -74,12 +73,11 @@ def get_comments(
elif username:
comments = pushshift.get_comments_from_user(username, lower=lower, upper=upper)
if do_supplement:
comments = pushshift.supplement_reddit_data(comments, chunk_size=100)
comments = common.generator_chunker(comments, 500)
form = '{lower} ({lower_unix}) - {upper} ({upper_unix}) +{gain}'
comments = common.generator_chunker(comments, 500)
for chunk in comments:
step = database.insert(chunk)
message = form.format(
@ -90,6 +88,7 @@ def get_comments(
gain=step['new_comments'],
)
print(message)
if specific_submission:
query = '''
UPDATE submissions

View File

@ -238,6 +238,7 @@ def supplement_reddit_data(dummies, chunk_size=100):
'''
chunks = common.generator_chunker(dummies, chunk_size)
for chunk in chunks:
common.log.debug('Supplementing %d items with live reddit data.', len(chunk))
ids = [item.fullname for item in chunk]
live_copies = list(common.r.info(ids))
live_copies = {item.fullname: item for item in live_copies}

View File

@ -346,7 +346,7 @@ class TSDB:
new_values[key] += status
if commit:
common.log.debug('Committing insert')
common.log.debug('Committing insert.')
self.sql.commit()
common.log.debug('Done inserting.')