-'''.replace('\n', '')
-HTML_FORMAT_FILE = '
{name} ({size})'
-
DB_INIT = '''
CREATE TABLE IF NOT EXISTS urls(
url TEXT,
@@ -435,92 +386,6 @@ class Walker:
## WALKER ##########################################################################################
-## OTHER CLASSES ###################################################################################
-## ##
-class TreeExistingChild(Exception):
- pass
-
-class TreeInvalidIdentifier(Exception):
- pass
-
-class TreeNode:
- def __init__(self, identifier, data=None):
- assert isinstance(identifier, str)
- assert '\\' not in identifier
- self.identifier = identifier
- self.data = data
- self.parent = None
- self.children = {}
-
- def __eq__(self, other):
- return isinstance(other, TreeNode) and self.abspath() == other.abspath()
-
- def __getitem__(self, key):
- return self.children[key]
-
- def __hash__(self):
- return hash(self.abspath())
-
- def __repr__(self):
- return 'TreeNode %s' % self.abspath()
-
- def abspath(self):
- node = self
- nodes = [node]
- while node.parent is not None:
- node = node.parent
- nodes.append(node)
- nodes.reverse()
- nodes = [node.identifier for node in nodes]
- return '\\'.join(nodes)
-
- def add_child(self, other_node, overwrite_parent=False):
- self.check_child_availability(other_node.identifier)
- if other_node.parent is not None and not overwrite_parent:
- raise ValueError('That node already has a parent. Try `overwrite_parent=True`')
-
- other_node.parent = self
- self.children[other_node.identifier] = other_node
- return other_node
-
- def check_child_availability(self, identifier):
- if identifier in self.children:
- raise TreeExistingChild('Node %s already has child %s' % (self.identifier, identifier))
-
- def detach(self):
- del self.parent.children[self.identifier]
- self.parent = None
-
- def list_children(self, customsort=None):
- children = list(self.children.values())
- if customsort is None:
- children.sort(key=lambda node: node.identifier.lower())
- else:
- children.sort(key=customsort)
- return children
-
- def merge_other(self, othertree, otherroot=None):
- newroot = None
- if ':' in othertree.identifier:
- if otherroot is None:
- raise Exception('Must specify a new name for the other tree\'s root')
- else:
- newroot = otherroot
- else:
- newroot = othertree.identifier
- othertree.identifier = newroot
- othertree.parent = self
- self.check_child_availability(newroot)
- self.children[newroot] = othertree
-
- def walk(self, customsort=None):
- yield self
- for child in self.list_children(customsort=customsort):
- yield from child.walk(customsort=customsort)
-## ##
-## OTHER CLASSES ###################################################################################
-
-
## GENERAL FUNCTIONS ###############################################################################
## ##
def build_file_tree(databasename):
@@ -533,67 +398,22 @@ def build_file_tree(databasename):
if len(fetch_all) == 0:
return
- path_form = '{domain}\\{folder}\\{filename}'
- all_items = []
+ path_datas = []
+ # :|| is my temporary (probably not temporary) hack for including the URL
+ # scheme without causing the pathtree processor to think there's a top
+ # level directory called 'http'.
+ # It will be replaced with :// in the calling `tree` function.
+ path_form = '{scheme}:||{domain}\\{folder}\\{filename}'
for item in fetch_all:
url = item[SQL_URL]
size = item[SQL_CONTENT_LENGTH]
path_parts = url_split(item[SQL_URL])
- path_parts = path_form.format(**path_parts)
- #path_parts = urllib.parse.unquote(path_parts)
- path_parts = path_parts.split('\\')
- item = {'url': url, 'size': size, 'path_parts': path_parts}
- all_items.append(item)
+ path = path_form.format(**path_parts)
+ path = urllib.parse.unquote(path)
+ path_data = {'path': path, 'size': size, 'data': url}
+ path_datas.append(path_data)
- all_items.sort(key=lambda x: x['url'])
-
- root_data = {
- 'item_type': 'directory',
- 'name': databasename,
- }
- scheme = url_split(all_items[0]['url'])['scheme']
- tree_root = TreeNode(databasename, data=root_data)
- tree_root.unsorted_children = all_items
- node_queue = set()
- node_queue.add(tree_root)
-
- # In this process, URLs are divided up into their nodes one directory layer at a time.
- # The root receives all URLs, and creates nodes for each of the top-level
- # directories. Those nodes receive all subdirectories, and repeat.
- while len(node_queue) > 0:
- node = node_queue.pop()
- for new_child_data in node.unsorted_children:
- path_parts = new_child_data['path_parts']
- # Create a new node for the directory, path_parts[0]
- # path_parts[1:] is assigned to that node to be divided next.
- child_identifier = path_parts.pop(0)
- #child_identifier = child_identifier.replace(':', '#')
-
- child = node.children.get(child_identifier, None)
- if not child:
- child = TreeNode(child_identifier, data={})
- child.unsorted_children = []
- node.add_child(child)
-
- child.data['name'] = child_identifier
- if len(path_parts) > 0:
- child.data['item_type'] = 'directory'
- child.unsorted_children.append(new_child_data)
- node_queue.add(child)
- else:
- child.data['item_type'] = 'file'
- child.data['size'] = new_child_data['size']
- child.data['url'] = new_child_data['url']
- if node.parent is None:
- continue
- elif node.parent == tree_root:
- node.data['url'] = scheme + '://' + node.identifier
- else:
- node.data['url'] = node.parent.data['url'] + '/' + node.identifier
-
- del node.unsorted_children
-
- return tree_root
+ return pathtree.from_paths(path_datas, root_name=databasename)
def db_init(sql, cur):
lines = DB_INIT.split(';')
@@ -652,83 +472,15 @@ def int_none(x):
return x
return int(x)
-def recursive_get_size(node):
- '''
- Calculate the size of the Directory nodes by summing the sizes of all children.
- Modifies the nodes in-place.
- '''
- return_value = {
- 'size': 0,
- 'unmeasured': 0,
- }
- if node.data['item_type'] == 'file':
- if node.data['size'] is None:
- return_value['unmeasured'] = 1
- return_value['size'] = node.data['size']
-
- else:
- for child in node.list_children():
- child_details = recursive_get_size(child)
- return_value['size'] += child_details['size'] or 0
- return_value['unmeasured'] += child_details['unmeasured']
- node.data['size'] = return_value['size']
-
- return return_value
-
-def recursive_print_node(node, depth=0, use_html=False, output_file=None):
- '''
- Given a tree node (presumably the root), print it and all of its children.
- '''
- size = node.data['size']
- if size is None:
- size = UNKNOWN_SIZE_STRING
- else:
- size = bytestring.bytestring(size)
-
- if use_html:
- css_class = 'directory_even' if depth % 2 == 0 else 'directory_odd'
- if node.data['item_type'] == 'directory':
- directory_url = node.data.get('url')
- directory_anchor = '
►' if directory_url else ''
- directory_anchor = directory_anchor.format(url=directory_url)
- line = HTML_FORMAT_DIRECTORY.format(
- css=css_class,
- directory_anchor=directory_anchor,
- name=node.data['name'],
- size=size,
- )
- else:
- line = HTML_FORMAT_FILE.format(
- name=node.data['name'],
- size=size,
- url=node.data['url'],
- )
- else:
- line = '{space}{bar}{name} : ({size})'
- line = line.format(
- space='| ' * (depth-1),
- bar='|---' if depth > 0 else '',
- name=node.data['name'],
- size=size
- )
- write(line, output_file)
-
- # Sort by type (directories first) then subsort by lowercase path
- customsort = lambda node: (
- node.data['item_type'] == 'file',
- node.data['url'].lower(),
- )
-
- for child in node.list_children(customsort=customsort):
- recursive_print_node(child, depth=depth+1, use_html=use_html, output_file=output_file)
-
- if node.data['item_type'] == 'directory':
- if use_html:
- # Close the directory div
- write('
+'''.replace('\n', '')
+
+HTML_FORMAT_FILE = '
{name} ({size})'
+
+
+class PathTree(treeclass.Tree):
+ def __init__(
+ self,
+ path,
+ display_name=None,
+ item_type=None,
+ size=None,
+ **kwargs,
+ ):
+ self.path = normalize_slash(path)
+ if display_name is None:
+ self.display_name = self.path.split(os.sep)[-1]
+ else:
+ self.display_name = display_name
+ kwargs['identifier'] = self.display_name
+ super(PathTree, self).__init__(**kwargs)
+ self.size = size
+ self.item_type = item_type
+
+def normalize_slash(path):
+ path = path.replace('/', '\\')
+ path = path.rstrip(os.sep)
+ return path
+
+def from_paths(path_datas, root_name):
+ all_datas = []
+ for data in path_datas:
+ if isinstance(data, str):
+ data = {'path': data}
+ elif isinstance(data, dict):
+ pass
+ else:
+ raise TypeError(data)
+ data['parts'] = data['path'].split(os.sep)
+ all_datas.append(data)
+
+ #path_parts = path_parts.split('\\')
+ #item = {'url': url, 'size': size, 'path_parts': path_parts}
+ #all_items.append(item)
+ #scheme = url_split(all_items[0]['url'])['scheme']
+
+ all_datas.sort(key=lambda x: x['path'])
+
+ tree_root = PathTree(root_name, item_type='directory')
+ tree_root.unsorted_children = all_datas
+ node_queue = set()
+ node_queue.add(tree_root)
+
+ # In this process, URLs are divided up into their nodes one directory layer at a time.
+ # The root has all URLs as its `unsorted_children` attribute, and creates
+ # nodes for each of the top-level directories.
+ # Those nodes receive all subdirectories, and repeat.
+ while len(node_queue) > 0:
+ node = node_queue.pop()
+ for new_child_data in node.unsorted_children:
+ # Create a new node for the subdirectory, which is path_parts[0]
+ # The rest of the child path is assigned to that node to be further divided.
+ # By popping, we modify the path_parts in place so that the next cycle
+ # only deals with the remaining subpath.
+ path_parts = new_child_data['parts']
+ child_identifier = path_parts.pop(0)
+
+ child = node.children.get(child_identifier)
+ if not child:
+ child = PathTree(child_identifier)
+ child.unsorted_children = []
+ node.add_child(child)
+
+ if len(path_parts) > 0:
+ child.item_type = 'directory'
+ child.unsorted_children.append(new_child_data)
+ else:
+ child.item_type = 'file'
+ child.size = new_child_data.get('size')
+ child.data = new_child_data.get('data')
+ node_queue.add(child)
+
+ if node.parent is not None and node.parent != tree_root:
+ node.path = node.parent.path + os.sep + node.path
+
+ del node.unsorted_children
+ return tree_root
+
+def recursive_get_size(node):
+ '''
+ Calculate the size of the Directory nodes by summing the sizes of all children.
+ Modifies the nodes in-place.
+ '''
+ return_value = {
+ 'size': 0,
+ 'unmeasured': 0,
+ }
+ if node.item_type == 'file':
+ if node.size is None:
+ return_value['unmeasured'] = 1
+ # = instead of += because if the node.size is None, we want to propogate
+ # that to the caller, rather than normalizing it to 0.
+ return_value['size'] = node.size
+
+ else:
+ for child in node.list_children():
+ child_details = recursive_get_size(child)
+ return_value['size'] += child_details['size'] or 0
+ return_value['unmeasured'] += child_details['unmeasured']
+ node.size = return_value['size']
+
+ return return_value
+
+def recursive_print_node(node, depth=0, use_html=False, header=None, footer=None):
+ '''
+ Given a tree node (presumably the root), print it and all of its children.
+
+ use_html:
+ Generate a neat HTML page instead of plain text.
+ header:
+ This text goes at the top of the file, or just below the tag.
+ footer:
+ This text goes at the end of the file, or just above the tag.
+ '''
+ if depth == 0:
+ if use_html:
+ yield '\n'
+ yield HTML_TREE_HEAD
+ yield ''
+ if header is not None:
+ yield header
+
+ size = node.size
+ if size is None:
+ size = '???'
+ else:
+ size = bytestring.bytestring(size)
+
+ if use_html:
+ css_class = 'directory_even' if depth % 2 == 0 else 'directory_odd'
+ if node.item_type == 'directory':
+ directory_url = node.path
+ directory_anchor = '
→' if directory_url else ''
+ directory_anchor = directory_anchor.format(url=directory_url)
+ line = HTML_FORMAT_DIRECTORY.format(
+ css=css_class,
+ directory_anchor=directory_anchor,
+ name=node.display_name,
+ size=size,
+ )
+ else:
+ line = HTML_FORMAT_FILE.format(
+ name=node.display_name,
+ size=size,
+ url=node.path,
+ )
+ else:
+ line = '{space}{bar}{name} : ({size})'
+ line = line.format(
+ space='| ' * (depth-1),
+ bar='|---' if depth > 0 else '',
+ name=node.display_name,
+ size=size
+ )
+ yield line
+
+ # Sort by type (directories first) then subsort by lowercase path
+ customsort = lambda node: (
+ node.item_type == 'file',
+ node.path.lower(),
+ )
+
+ for child in node.list_children(customsort=customsort):
+ yield from recursive_print_node(child, depth=depth+1, use_html=use_html)
+
+ if node.item_type == 'directory':
+ if use_html:
+ # Close the directory div
+ yield '