From eba495e3b74baa5ebe9a0aa07aa8a153b6141600 Mon Sep 17 00:00:00 2001 From: Ethan Dalool Date: Sun, 6 Sep 2020 21:43:48 -0700 Subject: [PATCH] Increase size of chunks going into db to reduce commit frequency. --- timesearch_modules/get_comments.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/timesearch_modules/get_comments.py b/timesearch_modules/get_comments.py index 91c8c71..d206906 100644 --- a/timesearch_modules/get_comments.py +++ b/timesearch_modules/get_comments.py @@ -78,7 +78,8 @@ def get_comments( if do_supplement: comments = pushshift.supplement_reddit_data(comments, chunk_size=100) - comments = common.generator_chunker(comments, 200) + + comments = common.generator_chunker(comments, 500) for chunk in comments: step = database.insert(chunk) message = form.format(