From 7e58c95f15d833600e8515bfc64a5f99bf8b7765 Mon Sep 17 00:00:00 2001 From: Ethan Dalool Date: Tue, 29 Sep 2020 16:51:59 -0700 Subject: [PATCH] Mostly failed experiment: tag_autocomplete indexeddb. The current system has bad performance when you've got 100,000+ tags. I discovered that when the server returns 304, the browser gives the ajax a 200 with the full response, and it's not clear to me if js can know it got a 304. So, the tag set is being fully re-parsed from the response on every page load. I was thinking that I should store that in IndexedDB to avoid the parsing step, but... since the JSON.parse is done by my common.get before it hits this function, it's meaningless. Not to mention I still have to rebuild the datalist on every page since of course that state isn't shared between tabs. Not worth the DB stuff. We'll see what happens next. --- .../backend/endpoints/tag_endpoints.py | 3 +- frontends/etiquette_flask/static/js/api.js | 7 + .../static/js/tag_autocomplete.js | 158 ++++++++++++++---- .../etiquette_flask/templates/clipboard.html | 1 - .../etiquette_flask/templates/photo.html | 2 - .../etiquette_flask/templates/search.html | 1 - 6 files changed, 135 insertions(+), 37 deletions(-) diff --git a/frontends/etiquette_flask/backend/endpoints/tag_endpoints.py b/frontends/etiquette_flask/backend/endpoints/tag_endpoints.py index 25eac6c..a59eff8 100644 --- a/frontends/etiquette_flask/backend/endpoints/tag_endpoints.py +++ b/frontends/etiquette_flask/backend/endpoints/tag_endpoints.py @@ -1,4 +1,5 @@ import flask; from flask import request +import time import etiquette @@ -68,7 +69,7 @@ def post_tag_remove_child(tagname): def get_all_tag_names(): all_tags = list(common.P.get_all_tag_names()) all_synonyms = common.P.get_all_synonyms() - response = {'tags': all_tags, 'synonyms': all_synonyms} + response = {'updated': int(time.time()), 'tags': all_tags, 'synonyms': all_synonyms} return jsonify.make_json_response(response) @site.route('/tag/') diff --git a/frontends/etiquette_flask/static/js/api.js b/frontends/etiquette_flask/static/js/api.js index 16eb827..1a795c0 100644 --- a/frontends/etiquette_flask/static/js/api.js +++ b/frontends/etiquette_flask/static/js/api.js @@ -347,6 +347,13 @@ function edit(tag_name, name, description, callback) common.post(url, data, callback); } +api.tags.get_all_tags = +function get_all_tags(callback) +{ + const url = "/all_tags.json"; + common.get(url, callback); +} + api.tags.remove_child = function remove_child(tag_name, child_name, callback) { diff --git a/frontends/etiquette_flask/static/js/tag_autocomplete.js b/frontends/etiquette_flask/static/js/tag_autocomplete.js index 951294e..968730f 100644 --- a/frontends/etiquette_flask/static/js/tag_autocomplete.js +++ b/frontends/etiquette_flask/static/js/tag_autocomplete.js @@ -1,34 +1,64 @@ const tag_autocomplete = {}; -tag_autocomplete.tagset = {"tags": [], "synonyms": {}}; +tag_autocomplete.tags = new Set(); +tag_autocomplete.synonyms = {}; tag_autocomplete.DATALIST_ID = "tag_autocomplete_datalist"; +// { +// const db_name = "tag_autocomplete"; +// const db_version = 1; +// const open_request = window.indexedDB.open(db_name, db_version); +// open_request.onsuccess = function(event) +// { +// const db = event.target.result; +// tag_autocomplete.db = db; +// console.log("Initialized db."); +// } +// open_request.onupgradeneeded = function(event) +// { +// const db = event.target.result; +// const tag_store = db.createObjectStore("tags", {"keyPath": "name"}); +// const synonym_store = db.createObjectStore("synonyms", {"keyPath": "name"}); +// const meta_store = db.createObjectStore("meta", {"keyPath": "key"}); +// tag_store.createIndex("name", "name", {unique: true}); +// synonym_store.createIndex("name", "name", {unique: true}); +// console.log("Installed db schema."); +// } +// } + +//////////////////////////////////////////////////////////////////////////////////////////////////// + tag_autocomplete.init_datalist = function init_datalist() { + console.log("Init datalist."); let datalist; datalist = document.getElementById(tag_autocomplete.DATALIST_ID); - if (!datalist) + if (datalist) { - datalist = document.createElement("datalist"); - datalist.id = tag_autocomplete.DATALIST_ID; - document.body.appendChild(datalist); + return; } + datalist = document.createElement("datalist"); + datalist.id = tag_autocomplete.DATALIST_ID; + document.body.appendChild(datalist); + + const fragment = document.createDocumentFragment(); common.delete_all_children(datalist); - for (const tag_name of tag_autocomplete.tagset["tags"]) + for (const tag_name of tag_autocomplete.tags) { const option = document.createElement("option"); option.value = tag_name; - datalist.appendChild(option); + fragment.appendChild(option); } - for (const synonym in tag_autocomplete.tagset["synonyms"]) + for (const synonym in tag_autocomplete.synonyms) { const option = document.createElement("option"); - option.value = tag_autocomplete.tagset["synonyms"][synonym] + "+" + synonym; - datalist.appendChild(option); + option.value = tag_autocomplete.synonyms[synonym] + "+" + synonym; + fragment.appendChild(option); } + datalist.appendChild(fragment); } tag_autocomplete.normalize_tagname = @@ -79,49 +109,113 @@ tag_autocomplete.resolve = function resolve(tagname) { tagname = tag_autocomplete.normalize_tagname(tagname); - if (tag_autocomplete.tagset["tags"].indexOf(tagname) != -1) + if (tag_autocomplete.tags.has(tagname)) { return tagname; } - if (tagname in tag_autocomplete.tagset["synonyms"]) + if (tagname in tag_autocomplete.synonyms) { - return tag_autocomplete.tagset["synonyms"][tagname]; + return tag_autocomplete.synonyms[tagname]; } return null; } -tag_autocomplete.update_tagset_callback = -function update_tagset_callback(response) +// function update_stored_tags(data) +// { +// console.log("Updating db tags."); +// const updated = data.updated; +// const version_transaction = tag_autocomplete.db.transaction(["meta"], "readwrite"); +// const meta_store = version_transaction.objectStore("meta"); +// meta_store.add({"key": "updated", "val": updated}); +// const tags_transaction = tag_autocomplete.db.transaction(["tags"], "readwrite"); +// const tags_store = tags_transaction.objectStore("tags"); +// for (const name of data.tags) +// { +// tags_store.add({"name": name}); +// tag_autocomplete.tags.add(name); +// } +// const synonyms_transaction = tag_autocomplete.db.transaction(["synonyms"], "readwrite"); +// const synonyms_store = synonyms_transaction.objectStore("synonyms"); +// for (const [name, mastertag] of Object.entries(data.synonyms)) +// { +// synonyms_store.add({"name": name, "mastertag": mastertag}); +// } +// tag_autocomplete.synonyms = data.synonyms; +// count = data.tags.length + Object.keys(data.synonyms).length; +// console.log(`Updated db tags with ${count} items.`); +// if (document.getElementById(tag_autocomplete.DATALIST_ID)) +// { +// setTimeout(() => tag_autocomplete.init_datalist(), 0); +// } +// } + +// function load_stored_tags() +// { +// console.log("Loading stored db tags."); +// const load_transaction = tag_autocomplete.db.transaction(["tags", "synonyms"]); +// const tags_store = load_transaction.objectStore("tags"); +// const tags_request = tags_store.getAll(); +// tags_request.onsuccess = function(event) +// { +// for (row of event.target.result) +// { +// tag_autocomplete.tags.add(row["name"]); +// } +// } +// // const synonyms_transaction = tag_autocomplete.db.transaction(["synonyms"]); +// const synonyms_store = load_transaction.objectStore("synonyms"); +// const synonyms_request = synonyms_store.getAll(); +// synonyms_request.onsuccess = function(event) +// { +// for (row of event.target.result) +// { +// tag_autocomplete.synonyms[row["name"]] = row["mastertag"]; +// } +// if (document.getElementById(tag_autocomplete.DATALIST_ID)) +// { +// setTimeout(() => tag_autocomplete.init_datalist(), 0); +// } +// } +// } + +tag_autocomplete.get_all_tags_callback = +function get_all_tags_callback(response) { if (response["meta"]["status"] == 304) { return; } - if (response["meta"]["status"] == 200) + if (response["meta"]["status"] != 200) { - tag_autocomplete.tagset = response["data"]; - if (document.getElementById(tag_autocomplete.DATALIST_ID)) - { - tag_autocomplete.init_datalist(); - } - console.log(`Updated tagset contains ${tag_autocomplete.tagset.tags.length}.`); - return tag_autocomplete.tagset; + console.error(response); + return; } - console.error(response); -} -tag_autocomplete.update_tagset = -function update_tagset() -{ - console.log("Updating known tagset."); - const url = "/all_tags.json"; - common.get(url, tag_autocomplete.update_tagset_callback); + // const server_updated = response.data.updated; + // const transaction = tag_autocomplete.db.transaction(["meta"]); + // const meta_store = transaction.objectStore("meta"); + // const request = meta_store.get("updated"); + // request.onsuccess = function(event) + // { + // if (event.target.result === undefined || event.target.result < server_updated) + // { + // update_stored_tags(response.data); + // } + // else + // { + // load_stored_tags(); + // } + // } + tag_autocomplete.tags = new Set(response.data.tags); + tag_autocomplete.synonyms = response.data.synonyms; + setTimeout(() => tag_autocomplete.init_datalist(), 0); + return tag_autocomplete.tagset; } tag_autocomplete.on_pageload = function on_pageload() { - tag_autocomplete.update_tagset(); + setTimeout(() => api.tags.get_all_tags(tag_autocomplete.get_all_tags_callback), 0); tag_autocomplete.init_entry_with_tagname_replacements(); } document.addEventListener("DOMContentLoaded", tag_autocomplete.on_pageload); diff --git a/frontends/etiquette_flask/templates/clipboard.html b/frontends/etiquette_flask/templates/clipboard.html index db59fba..3523533 100644 --- a/frontends/etiquette_flask/templates/clipboard.html +++ b/frontends/etiquette_flask/templates/clipboard.html @@ -225,7 +225,6 @@ function my_clipboard_load_save_hook() refresh_divs(); } -tag_autocomplete.init_datalist(); photo_clipboard.on_load_hooks.push(my_clipboard_load_save_hook); photo_clipboard.on_save_hooks.push(my_clipboard_load_save_hook); diff --git a/frontends/etiquette_flask/templates/photo.html b/frontends/etiquette_flask/templates/photo.html index 662e921..7d3d78b 100644 --- a/frontends/etiquette_flask/templates/photo.html +++ b/frontends/etiquette_flask/templates/photo.html @@ -541,8 +541,6 @@ function move_hoverzoom(event) photo_viewer.style.backgroundPosition=(-x)+"px "+(-y)+"px"; } -tag_autocomplete.init_datalist(); - function autofocus_add_tag_box() { /* diff --git a/frontends/etiquette_flask/templates/search.html b/frontends/etiquette_flask/templates/search.html index 5f2f0b3..c2cc15f 100644 --- a/frontends/etiquette_flask/templates/search.html +++ b/frontends/etiquette_flask/templates/search.html @@ -655,7 +655,6 @@ function tag_input_hook_forbids() tag_input_hook(this, inputted_forbids, "search_builder_forbids_inputted"); } -tag_autocomplete.init_datalist(); const input_musts = document.getElementById("search_builder_musts_input"); const ul_musts = input_musts.parentElement.parentElement; const input_mays = document.getElementById("search_builder_mays_input");