diff --git a/BaseNumber/basenumber.py b/BaseNumber/basenumber.py index 1043a68..80af20d 100644 --- a/BaseNumber/basenumber.py +++ b/BaseNumber/basenumber.py @@ -37,7 +37,6 @@ def from_base(number, base, alphabet=None): except ValueError: decimal_pos = len(number) - result = 0 for (index, character) in enumerate(number): if index == decimal_pos: diff --git a/Javascript/opendir_image.js b/Javascript/opendir_image.js index b05c389..1d372b3 100644 --- a/Javascript/opendir_image.js +++ b/Javascript/opendir_image.js @@ -5,7 +5,6 @@ the page, and displays them in a nice gallery. Designed for use on open directory listings, but works in many places. */ var seen_urls = new Set(); -var all_urls = []; var image_height = 200; var video_height = 300; var audio_width = 1000; @@ -18,20 +17,24 @@ VIDEO_TYPES = new RegExp(VIDEO_TYPES, "i"); var has_started = false; -var CSS = ""; -CSS += "audio, video { display: block; }"; -CSS += "audio { width: $audio_width$px; }"; -CSS += "video { height: $video_height$px; }"; -CSS += "img { display: block; max-width: 100% }"; - -CSS += ".control_panel { background-color: #aaa; min-height: 10px; width: 100%; }"; -CSS += ".workspace { background-color: #ddd; min-height: 10px; float: left; }"; -CSS += ".arealabel { position:absolute; right: 0; bottom: 0; opacity: 0.8; background-color: #000; color: #fff; }"; -CSS += ".delete_button { color: #d00; font-family: Arial; font-size: 11px; left: 0; position: absolute; top: 0; width: 25px; }"; -CSS += ".load_button { position: absolute; top: 10%; width: 100%; height: 80%; word-wrap: break-word; }"; -CSS += ".odi_anchor { display: block; }"; -CSS += ".odi_image_div, .odi_media_div { display: inline-block; margin: 5px; float: left; position: relative; background-color: #aaa; }"; -CSS += ".odi_image_div { min-width: $image_height$px; }"; +var CSS = "" ++ "body { background-color: #fff; }" ++ "audio, video { display: block; }" ++ "audio { width: $audio_width$px; }" ++ "video { height: $video_height$px; }" ++ "img { display: block; height: $image_height$px; max-width: 100% }" ++ ".control_panel { position: relative; background-color: #aaa; min-height: 10px; width: 100%; }" ++ ".workspace { background-color: #ddd; min-height: 10px; float: left; }" ++ ".arealabel { position:absolute; right: 0; bottom: 0; opacity: 0.8; background-color: #000; color: #fff; }" ++ ".delete_button { color: #d00; font-family: Arial; font-size: 11px; left: 0; position: absolute; top: 0; width: 25px; }" ++ ".ingest { position:absolute; right: 5px; top: 5px; height: 100%; width: 30% }" ++ ".ingestbox { position:relative; height: 75%; width:100%; box-sizing: border-box; }" ++ ".urldumpbox { overflow-y: scroll; height: 300px; width: 90% }" ++ ".load_button { position: absolute; top: 10%; width: 100%; height: 80%; word-wrap: break-word; }" ++ ".odi_anchor { display: block; }" ++ ".odi_image_div, .odi_media_div { display: inline-block; margin: 5px; float: left; position: relative; background-color: #aaa; }" ++ ".odi_image_div { min-width: $image_height$px; }" +; function apply_css() { @@ -64,7 +67,7 @@ function array_remove(a, item) function clear_page() { - /* Remove EVERYTHING, insert own css */ + /* Remove EVERYTHING */ console.log("clearing page"); document.removeChild(document.documentElement); @@ -122,145 +125,160 @@ function create_command_box_button(boxname, label, operation) var div = document.createElement("div"); div.appendChild(box); div.appendChild(button); + div.box = box; + div.button = button; return div; } +function create_odi_div(url) +{ + var div = null; + var paramless_url = url.split("?")[0]; + var basename = get_basename(url); + + if (paramless_url.match(IMAGE_TYPES)) + { + var div = document.createElement("div"); + div.id = generate_id(32); + div.className = "odi_image_div"; + div.odi_type = "image"; + + var a = document.createElement("a"); + a.className = "odi_anchor"; + a.odi_div = div; + a.href = url; + a.target = "_blank"; + + var img = document.createElement("img"); + img.odi_div = div; + img.anchor = a; + img.border = 0; + + img.lazy_src = url; + img.src = ""; + + var arealabel = document.createElement("span"); + arealabel.className = "arealabel"; + arealabel.odi_div = div; + arealabel.innerHTML = "0x0"; + img.arealabel = arealabel; + + var load_button = document.createElement("button"); + load_button.className = "load_button"; + load_button.odi_div = div; + load_button.innerHTML = basename; + load_button.onclick = function() + { + this.parentElement.removeChild(this); + lazy_load_one(this.odi_div); + }; + + div.image = img; + div.anchor = a; + a.appendChild(img); + a.appendChild(arealabel); + div.appendChild(a); + div.appendChild(load_button); + } + else + { + if (paramless_url.match(AUDIO_TYPES)) + { + var mediatype = "audio"; + } + else if (paramless_url.match(VIDEO_TYPES)) + { + var mediatype = "video"; + } + else + { + return null; + } + + var div = document.createElement("div"); + div.id = generate_id(32); + div.className = "odi_media_div"; + div.odi_type = "media"; + div.mediatype = mediatype; + + var center = document.createElement("center"); + center.odi_div = div; + + var a = document.createElement("a"); + a.odi_div = div; + a.innerHTML = get_basename(url); + a.target = "_blank"; + a.style.display = "block"; + a.href = url; + + var media = document.createElement(mediatype); + media.odi_div = div; + media.controls = true; + media.preload = "none"; + + sources = get_alternate_sources(url); + for (var sourceindex = 0; sourceindex < sources.length; sourceindex += 1) + { + source = document.createElement("source"); + source.src = sources[sourceindex]; + source.odi_div = div; + media.appendChild(source); + } + + div.media = media; + div.anchor = a; + center.appendChild(a); + div.appendChild(center); + div.appendChild(media); + } + + if (div == null) + { + return null; + } + + div.url = url; + div.basename = basename; + + button = document.createElement("button"); + button.className = "delete_button"; + button.odi_div = div; + button.innerHTML = "X"; + button.onclick = function() + { + delete_odi_div(this); + }; + div.appendChild(button); + return div; +} function create_odi_divs(urls) { image_divs = []; media_divs = []; + odi_divs = []; for (var index = 0; index < urls.length; index += 1) { url = urls[index]; + var paramless_url = url.split("?")[0]; if (!url) { continue; } - /*console.log("Building for " + url);*/ - var div = null; - var paramless_url = url.split("?")[0]; - var basename = get_basename(url); - - if (paramless_url.match(IMAGE_TYPES)) - { - var div = document.createElement("div"); - div.id = generate_id(32); - div.className = "odi_image_div"; - div.odi_type = "image"; - - var a = document.createElement("a"); - a.className = "odi_anchor"; - a.odi_div = div; - a.href = url; - a.target = "_blank"; - - var img = document.createElement("img"); - img.odi_div = div; - img.anchor = a; - img.border = 0; - img.height = image_height; - - img.lazy_src = url; - img.src = ""; - - var arealabel = document.createElement("span"); - arealabel.className = "arealabel"; - arealabel.odi_div = div; - arealabel.innerHTML = "0x0"; - img.arealabel = arealabel; - - var load_button = document.createElement("button"); - load_button.className = "load_button"; - load_button.odi_div = div; - load_button.innerHTML = basename; - load_button.onclick = function() - { - this.parentElement.removeChild(this); - lazy_load_one(this.odi_div); - }; - - div.image = img; - div.anchor = a; - a.appendChild(img); - a.appendChild(arealabel); - div.appendChild(a); - div.appendChild(load_button); - image_divs.push(div); - } - else - { - if (paramless_url.match(AUDIO_TYPES)) - { - var mediatype = "audio"; - } - else if (paramless_url.match(VIDEO_TYPES)) - { - var mediatype = "video"; - } - else - { - continue; - } - - var div = document.createElement("div"); - div.id = generate_id(32); - div.className = "odi_media_div"; - div.odi_type = "media"; - - var center = document.createElement("center"); - center.odi_div = div; - - var a = document.createElement("a"); - a.odi_div = div; - a.innerHTML = get_basename(url); - a.target = "_blank"; - a.style.display = "block"; - a.href = url; - - var media = document.createElement(mediatype); - media.odi_div = div; - media.controls = true; - media.preload = "none"; - - sources = get_alternate_sources(url); - for (var sourceindex = 0; sourceindex < sources.length; sourceindex += 1) - { - source = document.createElement("source"); - source.src = sources[sourceindex]; - source.odi_div = div; - media.appendChild(source); - } - - div.media = media; - div.anchor = a; - center.appendChild(a); - div.appendChild(center); - div.appendChild(media); - media_divs.push(div); - } - - if (div == null) + var odi_div = create_odi_div(url); + if (odi_div == null) { continue; } - - div.url = url; - div.basename = basename; - - button = document.createElement("button"); - button.className = "delete_button"; - button.odi_div = div; - button.innerHTML = "X"; - button.onclick = function() + if (odi_div.odi_type == "image") { - delete_odi_div(this); - }; - div.appendChild(button); - /*console.log("built " + div);*/ + image_divs.push(odi_div); + } + else + { + media_divs.push(odi_div); + } } - odi_divs = []; array_extend(odi_divs, image_divs); - odi_divs.push(document.createElement("br")); array_extend(odi_divs, media_divs); return odi_divs; } @@ -281,17 +299,28 @@ function create_workspace() var widthfilter = create_command_box_button("widthfilter", "min width", filter_width); var sorter = create_command_button("sort size", sort_size); var dumper = create_command_button("dump urls", dump_urls); + var ingest_box = document.createElement("textarea"); + var ingest_button = create_command_button("ingest", ingest); var start_button = create_command_button("load all", function(){start(); this.parentElement.removeChild(this);}); start_button.style.display = "block"; - control_panel.id = "CONTROL_PANEL"; control_panel.className = "control_panel"; workspace.id = "WORKSPACE"; workspace.className = "workspace"; + var ingest_div = document.createElement("div"); + ingest_div.id = "INGEST"; + ingest_div.className = "ingest"; + ingest_box.id = "ingestbox"; + ingest_box.className = "ingestbox"; + ingest_div.appendChild(ingest_box); + ingest_div.appendChild(ingest_button); + ingest_div.appendChild(ingest_box); + ingest_div.appendChild(ingest_button); + document.body.appendChild(control_panel); control_panel.appendChild(resizer); control_panel.appendChild(refilter); @@ -300,6 +329,7 @@ function create_workspace() control_panel.appendChild(widthfilter); control_panel.appendChild(sorter); control_panel.appendChild(dumper); + control_panel.appendChild(ingest_div); control_panel.appendChild(start_button); document.body.appendChild(workspace); } @@ -328,10 +358,7 @@ function dump_urls() if (textbox == null) { textbox = document.createElement("textarea"); - textbox.id = "urldumpbox"; - textbox.style.overflowY = "scroll"; - textbox.style.height = "300px"; - textbox.style.width = "90%"; + textbox.className = "urldumpbox"; workspace = document.getElementById("WORKSPACE"); workspace.appendChild(textbox); } @@ -344,7 +371,6 @@ function dump_urls() function fill_workspace(divs) { - clear_workspace(); console.log("filling workspace"); workspace = document.getElementById("WORKSPACE"); @@ -394,7 +420,7 @@ function filter_re(pattern, do_delete) for (var index = 0; index < odi_divs.length; index += 1) { div = odi_divs[index]; - match = div.basename.match(pattern); + match = div.url.match(pattern); if ((match && do_delete) || (!match && do_keep)) { delete_odi_div(div); @@ -444,8 +470,8 @@ function get_all_urls() {console.log("Rejecting reddit thumb"); continue;} if (url.indexOf("pixel.reddit") != -1 || url.indexOf("reddit.com/static/pixel") != -1) {console.log("Rejecting reddit pixel"); continue} - if (url.indexOf("/thumb/") != -1) - {console.log("Rejecting /thumb/"); continue;} + /*if (url.indexOf("/thumb/") != -1) + {console.log("Rejecting /thumb/"); continue;}*/ if (url.indexOf("/loaders/") != -1) {console.log("Rejecting loader"); continue;} if (url.indexOf("memegen") != -1) @@ -465,7 +491,6 @@ function get_all_urls() urls.push(sub_url); seen_urls.add(sub_url); - all_urls.push(sub_url); } seen_urls.add(url); } @@ -567,6 +592,34 @@ function generate_id(length) return "odi_" + text.join(""); } +function ingest() +{ + /* Take the text from the INGEST box, and make odi divs from it */ + var odi_divs = get_odi_divs(); + var ingestbox = document.getElementById("ingestbox"); + var text = ingestbox.value; + var urls = text.split("\n"); + for (var index = 0; index < urls.length; index += 1) + { + url = urls[index].trim(); + sub_urls = normalize_url(url); + if (sub_urls == null) + {continue;} + + for (var url_index = 0; url_index < sub_urls.length; url_index += 1) + { + sub_url = sub_urls[url_index]; + var odi_div = create_odi_div(sub_url); + if (odi_div == null) + {continue;} + odi_divs.push(odi_div); + } + } + ingestbox.value = ""; + clear_workspace(); + fill_workspace(odi_divs); +} + function lazy_load_all() { lazies = get_lazy_divs(); @@ -611,7 +664,6 @@ function lazy_load_one(element, comeback) }; image.onerror = function() { - array_remove(all_urls, this.odi_div); delete_odi_div(this); if (comeback){lazy_load_all()}; }; @@ -711,15 +763,18 @@ function normalize_url(url) function resize_images(height) { odi_divs = get_odi_divs(); - image_height = height; + height = height.toString() + "px"; for (var index = 0; index < odi_divs.length; index += 1) { var div = odi_divs[index]; - if (div.image == undefined) + if (div.image) { - continue; + div.image.style.height = height; + } + else if (div.media && div.mediatype == "video") + { + div.media.style.height = height; } - div.image.height = height; } } @@ -761,7 +816,7 @@ function start() function main() { - all_urls = get_all_urls(); + var all_urls = get_all_urls(); var divs = create_odi_divs(all_urls); create_workspace(); fill_workspace(divs); diff --git a/Javascript/tab_renamer.js b/Javascript/tab_renamer.js index 997a0ff..d951d10 100644 --- a/Javascript/tab_renamer.js +++ b/Javascript/tab_renamer.js @@ -1,7 +1,11 @@ javascript: function rename() { - document.title = prompt("New page title:"); + var new_title = prompt("New page title:"); + if (new_title !== null) + { + document.title = new_title; + } } rename(); \ No newline at end of file diff --git a/OpenDirDL/README.md b/OpenDirDL/README.md index 3e6896f..f5b45a8 100644 --- a/OpenDirDL/README.md +++ b/OpenDirDL/README.md @@ -7,6 +7,9 @@ Requires `pip install beautifulsoup4` See inside opendirdl.py for usage instructions. +- 2016 08 01 + - Made the digest work even if you forget the http:// + - 2016 07 29 - Moved some nested function definitions out to the top level, and made the construction of the file tree its own function. These functions really don't need to be used on their own, but they were cluttering the logic of the `tree` command. - Renamed `Tree.listnodes` to `Tree.list_children` and the `customsort` now expects to operate on Node objects rather than `(identifier, Node)` tuples. Nodes already have their identifier so the tuple was unecessary. diff --git a/OpenDirDL/opendirdl.py b/OpenDirDL/opendirdl.py index 2c64b34..0586dcb 100644 --- a/OpenDirDL/opendirdl.py +++ b/OpenDirDL/opendirdl.py @@ -274,6 +274,8 @@ class Walker: def __init__(self, walkurl, databasename=None, fullscan=False): if not walkurl.endswith('/'): walkurl += '/' + if '://' not in walkurl.split('.')[0]: + walkurl = 'http://' + walkurl self.walkurl = walkurl if databasename in (None, ''): @@ -761,7 +763,6 @@ def smart_insert(sql, cur, url=None, head=None, commit=True): if is_new: cur.execute('INSERT INTO urls VALUES(?, ?, ?, ?, ?)', data) else: - print(url) command = ''' UPDATE urls SET content_length = coalesce(?, content_length), @@ -777,6 +778,8 @@ def smart_insert(sql, cur, url=None, head=None, commit=True): def url_to_filepath(text): text = urllib.parse.unquote(text) parts = urllib.parse.urlsplit(text) + if any(part == '' for part in [parts.scheme, parts.netloc]): + raise ValueError('Not a valid URL') scheme = parts.scheme root = parts.netloc (folder, filename) = os.path.split(parts.path) @@ -861,7 +864,8 @@ def download( if outputdir in (None, ''): # This assumes that all URLs in the database are from the same domain. - # If they aren't, it's the user's fault because Walkers don't leave the given site. + # If they aren't, it's the user's fault because Walkers don't leave the given site + # on their own. cur.execute('SELECT url FROM urls LIMIT 1') url = cur.fetchone()[0] outputdir = url_to_filepath(url)['root'] @@ -877,19 +881,24 @@ def download( folder = os.path.join(outputdir, url_filepath['folder']) os.makedirs(folder, exist_ok=True) - fullname = os.path.join(folder, url_filepath['filename']) + final_fullname = os.path.join(folder, url_filepath['filename']) temporary_basename = hashit(url, 16) + '.oddltemporary' temporary_fullname = os.path.join(folder, temporary_basename) - if os.path.isfile(fullname): + # Because we use .oddltemporary files, the behavior of `overwrite` here + # is different than the behavior of `overwrite` in downloady. + # The overwrite used in the following block refers to the finalized file. + # The overwrite passed to downloady refers to the oddltemporary which + # may be resumed. + if os.path.isfile(final_fullname): if overwrite: - os.remove(fullname) + os.remove(final_fullname) else: - write('Skipping "%s". Use `--overwrite`' % fullname) + write('Skipping "%s". Use `--overwrite`' % final_fullname) continue overwrite = overwrite or None - write('Downloading "%s" as "%s"' % (fullname, temporary_basename)) + write('Downloading "%s" as "%s"' % (final_fullname, temporary_basename)) downloady.download_file( url, localname=temporary_fullname, @@ -897,7 +906,7 @@ def download( callback_progress=downloady.progress2, overwrite=overwrite ) - os.rename(temporary_fullname, fullname) + os.rename(temporary_fullname, final_fullname) def download_argparse(args): return download( diff --git a/SpinalTap/spinal.py b/SpinalTap/spinal.py index 4e9c89e..c03a20b 100644 --- a/SpinalTap/spinal.py +++ b/SpinalTap/spinal.py @@ -718,7 +718,7 @@ def walk_generator( callback_exclusion(absolute_name, 'file') continue if normalize(absolute_name) in exclude_filenames: - callback_exclusion(absolute_filename, 'file') + callback_exclusion(absolute_name, 'file') continue yield(str_to_fp(absolute_name))