From ef15e1a64496a9a44c398266de75387b8b250e4a Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 9 Jul 2016 21:38:49 -0700 Subject: [PATCH] else --- Javascript/reddit_live_new.html | 105 +++++++++++++++++++----- OpenDirDL/README.md | 3 + OpenDirDL/opendirdl.py | 140 +++++++++++++++++++++----------- QuickTips/generators.md | 91 +++++++++++++++++---- ServerReference/simpleserver.py | 119 +++++++++++++++++++-------- SpinalTap/spinal.py | 25 +++++- 6 files changed, 367 insertions(+), 116 deletions(-) diff --git a/Javascript/reddit_live_new.html b/Javascript/reddit_live_new.html index f8a23b7..df63e64 100644 --- a/Javascript/reddit_live_new.html +++ b/Javascript/reddit_live_new.html @@ -7,7 +7,7 @@
- + @@ -32,14 +32,14 @@ body margin-right: 10%; padding: 5px; } + + #control_panel { background-color: #284142; padding: 5px; } -#workspace -{ -} + a { @@ -53,11 +53,26 @@ a:hover text-decoration: underline; } +.delete_button +{ + position: absolute; + top: 0px; + right: 0px; + height: 100%; + width: 5%; + + border: 0px; + background-color: #b53030; +} + .submission { + position: relative; + padding: 10px; padding-top: 20px; padding-bottom: 20px; + padding-right: 6%; /*for the delete button*/ margin: 10px; margin-top: 20px; @@ -76,8 +91,21 @@ a:hover */ var CHECK_DELAY = 30 * 1000; +var CONTROL_PANEL = document.getElementById("control_panel"); +CONTROL_PANEL.default_color = CONTROL_PANEL.style.backgroundColor; var WORKSPACE = document.getElementById("workspace"); +var id_cache = []; +var id_cache_size = 20; + +var first_loop = true; +var unread_notification_count = 0; +var subreddit = ""; +var check_timer = null; + +var page_focused_cached; + + var HTTPClient = function() { /* Thanks ttgagne http://stackoverflow.com/a/22076667 */ @@ -86,10 +114,20 @@ var HTTPClient = function() { var request = new XMLHttpRequest(); request.onreadystatechange = function() - { - if (request.readyState == 4 && request.status == 200) + { + // console.log(request.readyState); + // console.log(request.status); + if (request.readyState == 4) { - callback(request.responseText); + if (request.status == 200) + { + CONTROL_PANEL.style.backgroundColor = CONTROL_PANEL.default_color; + callback(request.responseText); + } + else + { + CONTROL_PANEL.style.backgroundColor = "#f00"; + } } } request.open("GET", url, asynchronous); @@ -98,21 +136,21 @@ var HTTPClient = function() } } -function apply_to_page(text) +function apply_to_page(response_json) { - var j = JSON.parse(text); + var j = JSON.parse(response_json); var submissions = j["data"]["children"]; submissions.reverse(); // newest last var new_items = 0; for (var index = 0; index < submissions.length; index += 1) { var submission = submissions[index]["data"]; - if (done_ids.has(submission["id"])) + if (id_cache.includes(submission["id"])) { continue; } - done_ids.add(submission["id"]); + id_cache.push(submission["id"]); if (first_loop) { @@ -128,11 +166,21 @@ function apply_to_page(text) anchor.href = "https://reddit.com/r/" + submission["subreddit"] + "/comments/" + submission["id"]; anchor.target = "_blank"; + var delete_button = document.createElement("button") + delete_button.className = "delete_button"; + delete_button.div = div; + delete_button.innerHTML = "X"; + delete_button.onclick = function() + { + this.div.parentElement.removeChild(this.div); + } + var timestamp = document.createElement("span"); var submission_time = new Date(submission["created_utc"]) timestamp.innerHTML = "" + submission_time.getHours() + ":" + submission_time.getMinutes(); div.appendChild(anchor); + div.appendChild(delete_button); //WORKSPACE.insertBefore(div, WORKSPACE.firstChild); WORKSPACE.appendChild(div); } @@ -142,6 +190,12 @@ function apply_to_page(text) unread_notification_count += new_items; update_title(); } + + while (id_cache.length < id_cache_size) + { + id_cache.shift(); + } + first_loop = false; } @@ -160,7 +214,7 @@ function check_once() console.log("no subreddit"); return; } - var url = "https://api.reddit.com/r/" + subreddit + "/new.json"; + var url = "https://api.reddit.com" + subreddit + "/new.json"; session.get(url, apply_to_page); } @@ -196,6 +250,18 @@ function page_focused_fresh() return page_focused_cached; } +function sort_submission_comparator(submission_1, submission_2) +{ + created_1 = submission_1["created_utc"]; + created_2 = submission_2["created_utc"]; + + if (created_1 < created_2) + {return -1;} + if (created_1 > created_2) + {return 1;} + return 0; +} + function start() { console.log("start"); @@ -203,10 +269,14 @@ function start() clear_workspace(); var field = document.getElementById("subreddit_field"); var text = field.value; - text = text.replace("/r/", "").replace("r/", ""); + text = text.replace("/u/", "/user/"); + if (text.indexOf("/") == -1) + { + text = "/r/" + text; + } subreddit = text; var link = document.getElementById("browser_link"); - var url = "https://reddit.com/r/" + subreddit + "/new"; + var url = "https://reddit.com" + subreddit + "/new"; link.href = url; link.innerHTML = url; update_title(); @@ -242,14 +312,7 @@ function visibility_property() return null; } -var done_ids = new Set(); -var first_loop = true; -var unread_notification_count = 0; -var subreddit = ""; -var check_timer = null; - -var page_focused_cached; page_focused_fresh(); var my_visibility_property = visibility_property(); diff --git a/OpenDirDL/README.md b/OpenDirDL/README.md index 771a32c..fb63aa1 100644 --- a/OpenDirDL/README.md +++ b/OpenDirDL/README.md @@ -1,6 +1,9 @@ Open Dir DL =========== +- 2016 07 08 + - Fixed bug in which trees wouldn't generate on server:port urls. + - 2016 07 04 - Added new argparse command "tree" diff --git a/OpenDirDL/opendirdl.py b/OpenDirDL/opendirdl.py index e27cb48..fade867 100644 --- a/OpenDirDL/opendirdl.py +++ b/OpenDirDL/opendirdl.py @@ -80,9 +80,14 @@ measure: flags: -f | --fullscan: - When included, perform HEAD requests when a file's size is not known. - If this flag is not included, and some file's size is unkown, you will - receive a printed note. + When included, perform HEAD requests on all files to update their size. + + -n | --new_only: + When included, perform HEAD requests only on files that haven't gotten one + yet. + + If a file's size is not known by the time this operation completes, you will + receive a printed note. tree: Print the file / folder tree. @@ -120,6 +125,7 @@ import requests import shutil import sqlite3 ## ~tkinter +import traceback import urllib.parse FILENAME_BADCHARS = '/\\:*?"<>|' @@ -166,6 +172,7 @@ SKIPPABLE_FILETYPES = [ '.tar', '.ttf', '.txt', +'.wav', '.webm', '.wma', '.zip', @@ -229,6 +236,11 @@ SQL_CONTENT_TYPE = 3 SQL_DO_DOWNLOAD = 4 +UNMEASURED_WARNING = ''' +Note: %d files do not have a stored Content-Length. +Run `measure` with `-f`|`--fullscan` or `-n`|`--new_only` to HEAD request those files. +'''.strip() + ## DOWNLOADER ###################################################################################### ## ## class Downloader: @@ -293,6 +305,12 @@ class Generic: setattr(self, kwarg, kwargs[kwarg]) +class TreeExistingChild(Exception): + pass + +class TreeInvalidIdentifier(Exception): + pass + class TreeNode: def __init__(self, identifier, data, parent=None): assert isinstance(identifier, str) @@ -329,9 +347,9 @@ class TreeNode: def check_child_availability(self, identifier): if ':' in identifier: - raise Exception('Only roots may have a colon') + raise TreeInvalidIdentifier('Only roots may have a colon') if identifier in self.children: - raise Exception('Node %s already has child %s' % (self.identifier, identifier)) + raise TreeExistingChild('Node %s already has child %s' % (self.identifier, identifier)) def detach(self): del self.parent.children[self.identifier] @@ -363,8 +381,11 @@ class TreeNode: for node in self.walk(customsort): print(node.abspath()) - def sorted_children(self): - keys = sorted(self.children.keys()) + def sorted_children(self, customsort=None): + if customsort: + keys = sorted(self.children.keys(), key=customsort) + else: + keys = sorted(self.children.keys()) for key in keys: yield (key, self.children[key]) @@ -388,7 +409,7 @@ class Walker: if databasename is None or databasename == "": self.domain = url_to_filepath(walkurl)['root'] databasename = self.domain + '.db' - databasename = databasename.replace(':', '') + databasename = databasename.replace(':', '#') self.databasename = databasename safeprint('Opening %s' % self.databasename) @@ -451,6 +472,10 @@ class Walker: else: url = urllib.parse.urljoin(self.walkurl, url) + if url in self.seen_directories: + # We already picked this up at some point + return + if not url.startswith(self.walkurl): # Don't follow external links or parent directory. safeprint('Skipping "%s" due to external url.' % url) @@ -480,11 +505,14 @@ class Walker: return raise content_type = head.headers.get('Content-Type', '?') - - if content_type.startswith('text/html') and head.url.endswith('/'): + #print(content_type) + if content_type.startswith('text/html'):# and head.url.endswith('/'): # This is an index page, so extract links and queue them. response = do_get(url) hrefs = self.extract_hrefs(response) + # Just in case the URL we used is different than the real one, + # such as missing a trailing slash, add both. + self.seen_directories.add(url) self.seen_directories.add(head.url) added = 0 for href in hrefs: @@ -660,6 +688,7 @@ def smart_insert(sql, cur, url=None, head=None, commit=True): if is_new: cur.execute('INSERT INTO urls VALUES(?, ?, ?, ?, ?)', data) else: + print(url) command = ''' UPDATE urls SET content_length = coalesce(?, content_length), @@ -832,34 +861,37 @@ def list_basenames_argparse(args): outputfile=args.outputfile, ) -def measure(databasename, fullscan=False): +def measure(databasename, fullscan=False, new_only=False): ''' Given a database, print the sum of all Content-Lengths. - If `fullscan`, then URLs with no Content-Length will be - HEAD requested, and the result will be saved back into the file. + URLs will be HEAD requested if: + `new_only` is True and the file has no stored content length, or + `fullscan` is True and `new_only` is False ''' if isinstance(fullscan, str): fullscan = bool(fullscan) totalsize = 0 sql = sqlite3.connect(databasename) - cur1 = sql.cursor() - cur2 = sql.cursor() - cur2.execute('SELECT * FROM urls WHERE do_download == 1') + cur = sql.cursor() + + if new_only: + cur.execute('SELECT * FROM urls WHERE do_download == 1 AND content_length IS NULL') + else: + cur.execute('SELECT * FROM urls WHERE do_download == 1') + + items = cur.fetchall() + filecount = 0 unmeasured_file_count = 0 try: - while True: - fetch = cur2.fetchone() - if fetch is None: - break - + for fetch in items: size = fetch[SQL_CONTENT_LENGTH] - if fullscan: + if fullscan or new_only: url = fetch[SQL_URL] head = do_head(url, raise_for_status=False) - fetch = smart_insert(sql, cur1, head=head, commit=False) + fetch = smart_insert(sql, cur, head=head, commit=True) size = fetch[SQL_CONTENT_LENGTH] if size is None: safeprint('"%s" is not revealing Content-Length' % url) @@ -881,14 +913,14 @@ def measure(databasename, fullscan=False): totalsize_string = '{} ({:,} bytes) in {:,} files'.format(short_string, totalsize, filecount) print(totalsize_string) if unmeasured_file_count > 0: - print('Note: %d files do not have a stored Content-Length.' % unmeasured_file_count) - print('Run `measure` with `-f` or `--fullscan` to HEAD request those files.') + print(UNMEASURED_WARNING % unmeasured_file_count) return totalsize def measure_argparse(args): return measure( databasename=args.databasename, fullscan=args.fullscan, + new_only=args.new_only, ) def remove_pattern(args): @@ -913,9 +945,13 @@ def tree(databasename, output_filename=None): path_parts = url_to_filepath(items[0][SQL_URL]) root_identifier = path_parts['root'] - #print('Root', root_identifier) + print('Root', root_identifier) root_data = {'name': root_identifier, 'item_type': 'directory'} - tree = TreeNode(identifier=root_identifier, data=root_data) + root_identifier = root_identifier.replace(':', '') + tree = TreeNode( + identifier=root_identifier, + data=root_data + ) node_map = {} unmeasured_file_count = 0 @@ -923,20 +959,27 @@ def tree(databasename, output_filename=None): for item in items: path = url_to_filepath(item[SQL_URL]) scheme = path['scheme'] + + # I join and re-split because 'folder' may contain slashes of its own + # and I want to break all the pieces path = '\\'.join([path['root'], path['folder'], path['filename']]) parts = path.split('\\') + #print(path) for (index, part) in enumerate(parts): - index += 1 - this_path = '/'.join(parts[:index]) - parent_path = '/'.join(parts[:index-1]) + this_path = '/'.join(parts[:index + 1]) + parent_path = '/'.join(parts[:index]) #safeprint('this:' + this_path) #safeprint('parent:' + parent_path) + #input() data = { 'name': part, 'url': scheme + '://' + this_path, } - if index == len(parts): + this_identifier = this_path.replace(':', '') + parent_identifier = parent_path.replace(':', '') + + if (index + 1) == len(parts): data['item_type'] = 'file' if item[SQL_CONTENT_LENGTH]: data['size'] = item[SQL_CONTENT_LENGTH] @@ -948,29 +991,29 @@ def tree(databasename, output_filename=None): # Ensure this comment is in a node of its own - this_node = node_map.get(this_path, None) + this_node = node_map.get(this_identifier, None) if this_node: # This ID was detected as a parent of a previous iteration # Now we're actually filling it in. this_node.data = data else: - this_node = TreeNode(this_path, data) - node_map[this_path] = this_node + this_node = TreeNode(this_identifier, data) + node_map[this_identifier] = this_node # Attach this node to the parent. - if parent_path == root_identifier: + if parent_identifier == root_identifier: try: tree.add_child(this_node) - except: + except TreeExistingChild: pass else: - parent_node = node_map.get(parent_path, None) + parent_node = node_map.get(parent_identifier, None) if not parent_node: - parent_node = TreeNode(parent_path, data=None) - node_map[parent_path] = parent_node + parent_node = TreeNode(parent_identifier, data=None) + node_map[parent_identifier] = parent_node try: parent_node.add_child(this_node) - except: + except TreeExistingChild: pass this_node.parent = parent_node #print(this_node.data) @@ -997,7 +1040,7 @@ def tree(databasename, output_filename=None): if node.data['item_type'] == 'directory': div_id = hashit(node.identifier, 16) line = '' - line += '
' + line += '