voussoir/timesearch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Some small whitespace and log tweaks.

Browse source
This commit is contained in:
Ethan Dalool 2020-10-08 14:17:19 -07:00
parent 55d9c064bd
commit e2d1f2b9ef
3 changed files with 4 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
timesearch_modules/get_comments.py
View file

@ -5,7 +5,6 @@ from . import exceptions
from . import pushshift from . import pushshift
from . import tsdb from . import tsdb
def get_comments( def get_comments(
subreddit=None, subreddit=None,
username=None, username=None,
@ -74,12 +73,11 @@ def get_comments(
elif username: elif username:
comments = pushshift.get_comments_from_user(username, lower=lower, upper=upper) comments = pushshift.get_comments_from_user(username, lower=lower, upper=upper)
if do_supplement: if do_supplement:
comments = pushshift.supplement_reddit_data(comments, chunk_size=100) comments = pushshift.supplement_reddit_data(comments, chunk_size=100)
comments = common.generator_chunker(comments, 500)
form = '{lower} ({lower_unix}) - {upper} ({upper_unix}) +{gain}' form = '{lower} ({lower_unix}) - {upper} ({upper_unix}) +{gain}'
comments = common.generator_chunker(comments, 500)
for chunk in comments: for chunk in comments:
step = database.insert(chunk) step = database.insert(chunk)
message = form.format( message = form.format(
@ -90,6 +88,7 @@ def get_comments(
gain=step['new_comments'], gain=step['new_comments'],
) )
print(message) print(message)
if specific_submission: if specific_submission:
query = ''' query = '''
UPDATE submissions UPDATE submissions

1
timesearch_modules/pushshift.py
View file

@ -238,6 +238,7 @@ def supplement_reddit_data(dummies, chunk_size=100):
''' '''
chunks = common.generator_chunker(dummies, chunk_size) chunks = common.generator_chunker(dummies, chunk_size)
for chunk in chunks: for chunk in chunks:
common.log.debug('Supplementing %d items with live reddit data.', len(chunk))
ids = [item.fullname for item in chunk] ids = [item.fullname for item in chunk]
live_copies = list(common.r.info(ids)) live_copies = list(common.r.info(ids))
live_copies = {item.fullname: item for item in live_copies} live_copies = {item.fullname: item for item in live_copies}

2
timesearch_modules/tsdb.py
View file

@ -346,7 +346,7 @@ class TSDB:
new_values[key] += status new_values[key] += status
if commit: if commit:
common.log.debug('Committing insert') common.log.debug('Committing insert.')
self.sql.commit() self.sql.commit()
common.log.debug('Done inserting.') common.log.debug('Done inserting.')