Add readme.

README.md

@@ -0,0 +1,74 @@
+hnarchive
+=========
+
+hnarchive downloads all HN items (threads and comments) into an SQLite database. At this time, my database is 23.18 GiB with just over 25,000,000 items. I'd be happy to share it.
+
+Please `pip install requests` and `pip install voussoirkit`.
+
+According to the [HN API docs](https://github.com/HackerNews/API) there is no enforced ratelimit, so just use a `threads` count that seems polite.
+
+To get started, just run `python hnarchive.py update` and it will start from 1. In the future, you can run `update` on a cronjob or use `livestream` to get new items forever. Note, `update` always starts from the highest ID in the database. If you use `get` to get a range of IDs that is ahead of your update schedule, your next `update` will miss the skipped IDs.
+
+Here are all of the subcommands:
+
+    get:
+        Get items between two IDs, inclusive.
+
+        flags:
+        --lower id:
+            Lower bound item ID.
+
+        --upper id:
+            Upper bound item ID.
+
+        --threads X:
+            Use X threads to download items. Default = 1 thread.
+
+        --commit_period X:
+            Commit the database after every X insertions. Default = 200.
+
+    livestream:
+        Watch for new items in an infinite loop.
+
+        flags:
+        --commit_period X:
+            Commit the database after every X insertions. Default = 200.
+
+    update:
+        Get new items, from the highest ID in the database to the present.
+
+        flags:
+        --threads X:
+            Use X threads to download items. Default = 1 thread.
+
+        --commit_period X:
+            Commit the database after every X insertions. Default = 200.
+
+    update_items:
+        Redownload items to update their scores, descendant counts, etc.
+
+        flags:
+        --days X:
+            Update items where the retrieval date is less than X days ahead of the
+            submission date.
+            Stories are only open for comments for 14 days, so the `descendants`
+            count of any story younger than 14 days should be considered volatile.
+            It seems the upvote button does not disappear at any age, though I
+            don't know whether votes on old submissions will actually count.
+            Regardless, votes and comments tend to solidify within a day or two
+            after submission so a small number should be sufficient.
+
+        --threads X:
+            Use X threads to download items. Default = 1 thread.
+
+        --only_mature:
+            If True, only update items where the submission date is more than 14
+            days ago. Without this, you will be updating items which are very close
+            to the present time, an effort which you may find wasteful.
+
+        --commit_period X:
+            Commit the database after every X insertions. Default = 200.
+
+https://github.com/voussoir/hnarchive
+
+https://gitlab.com/voussoir/hnarchive

hnarchive.py

@@ -255,11 +255,14 @@ hnarchive.py
 
 {get}
 
-{update}
-
 {livestream}
 
+{update}
+
 {update_items}
+
+TO SEE DETAILS ON EACH COMMAND, RUN
+> hnarchive.py <command> --help
 '''.lstrip()
 
 SUB_DOCSTRINGS = dict(
@@ -268,17 +271,26 @@ get:
     Get items between two IDs, inclusive.
 
     flags:
-    --lower:
+    --lower id:
         Lower bound item ID.
 
-    --upper:
+    --upper id:
         Upper bound item ID.
 
     --threads X:
         Use X threads to download items. Default = 1 thread.
 
     --commit_period X:
-        Commit the database after every X insertions. Default = 1000.
+        Commit the database after every X insertions. Default = 200.
+'''.strip(),
+
+livestream='''
+livestream:
+    Watch for new items in an infinite loop.
+
+    flags:
+    --commit_period X:
+        Commit the database after every X insertions. Default = 200.
 '''.strip(),
 
 update='''
@@ -288,6 +300,9 @@ update:
     flags:
     --threads X:
         Use X threads to download items. Default = 1 thread.
+
+    --commit_period X:
+        Commit the database after every X insertions. Default = 200.
 '''.strip(),
 
 update_items='''
@@ -298,11 +313,12 @@ update_items:
     --days X:
         Update items where the retrieval date is less than X days ahead of the
         submission date.
-        Stories are only open for comments for 14 days.
+        Stories are only open for comments for 14 days, so the `descendants`
+        count of any story younger than 14 days should be considered volatile.
         It seems the upvote button does not disappear at any age, though I
         don't know whether votes on old submissions will actually count.
-        Regardless, votes tend to solidify within a day or two after
+        Regardless, votes and comments tend to solidify within a day or two
-        submission so a small number should be sufficient.
+        after submission so a small number should be sufficient.
 
     --threads X:
         Use X threads to download items. Default = 1 thread.
@@ -311,11 +327,9 @@ update_items:
         If True, only update items where the submission date is more than 14
         days ago. Without this, you will be updating items which are very close
         to the present time, an effort which you may find wasteful.
-'''.strip(),
 
-livestream='''
+    --commit_period X:
-livestream:
+        Commit the database after every X insertions. Default = 200.
-    Watch for new items in an infinite loop.
 '''.strip(),
 )
 
@@ -335,7 +349,7 @@ def get_argparse(args):
 
 def livestream_argparse(args):
     try:
-        insert_items(livestream())
+        insert_items(livestream(), commit_period=args.commit_period)
     except KeyboardInterrupt:
         commit()
 
@@ -350,7 +364,7 @@ def update_argparse(args):
             ids = range(lower, upper+1)
             items = get_items(ids, threads=args.threads)
 
-            insert_items(items)
+            insert_items(items, commit_period=args.commit_period)
     except KeyboardInterrupt:
         commit()
 
@@ -366,11 +380,12 @@ def update_items_argparse(args):
     cur = sql.execute(query, bindings)
     ids = cur.fetchall()
 
+    log.info('Updating %d items.', len(ids))
     ids = [id for (id,) in ids]
     items = get_items(ids, threads=args.threads)
 
     try:
-        insert_items(items)
+        insert_items(items, commit_period=args.commit_period)
     except KeyboardInterrupt:
         commit()
 
@@ -384,20 +399,23 @@ def main(argv):
     p_get.add_argument('--lower', type=int, default=None)
     p_get.add_argument('--upper', type=int, default=None)
     p_get.add_argument('--threads', type=int, default=None)
-    p_get.add_argument('--commit_period', '--commit-period', type=int, default=1000)
+    p_get.add_argument('--commit_period', '--commit-period', type=int, default=200)
     p_get.set_defaults(func=get_argparse)
 
     p_livestream = subparsers.add_parser('livestream')
+    p_livestream.add_argument('--commit_period', '--commit-period', type=int, default=200)
     p_livestream.set_defaults(func=livestream_argparse)
 
     p_update = subparsers.add_parser('update')
     p_update.add_argument('--threads', type=int, default=None)
+    p_update.add_argument('--commit_period', '--commit-period', type=int, default=200)
     p_update.set_defaults(func=update_argparse)
 
     p_update_items = subparsers.add_parser('update_items', aliases=['update-items'])
     p_update_items.add_argument('--days', type=float, required=True)
     p_update_items.add_argument('--threads', type=int, default=None)
     p_update_items.add_argument('--only_mature', '--only-mature', action='store_true')
+    p_update_items.add_argument('--commit_period', '--commit-period', type=int, default=200)
     p_update_items.set_defaults(func=update_items_argparse)
 
     return betterhelp.subparser_main(

requirements.txt

@@ -0,0 +1,2 @@
+requests
+voussoirkit