From 2a8af94fe39e5ab5dd6f6a80a6dd74f9d5ab4f1e Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Sat, 2 Nov 2024 17:29:11 +0800 Subject: [PATCH 01/43] feat: Seperated user database Add scripts for migration and management --- .gitignore | 1 + migrator.py | 55 ++++++++++++++++++++ package.json | 3 +- requirements.txt | 3 +- user_utils.py | 131 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 191 insertions(+), 2 deletions(-) create mode 100644 migrator.py create mode 100644 user_utils.py diff --git a/.gitignore b/.gitignore index c6a07bc..ab92bde 100644 --- a/.gitignore +++ b/.gitignore @@ -55,5 +55,6 @@ mercury_annotations.json .venv .idea mercury.sqlite +users.sqlite test.jsonl diff --git a/migrator.py b/migrator.py new file mode 100644 index 0000000..e36edd5 --- /dev/null +++ b/migrator.py @@ -0,0 +1,55 @@ +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="Migrate data from source to output") + parser.add_argument("--source", type=str, required=True, help="Path to the source SQLite file") + parser.add_argument("--output", type=str, required=True, help="Path to the output SQLite file") + args = parser.parse_args() + + import sqlite3 + + conn_source = sqlite3.connect(args.source) + cursor_source = conn_source.cursor() + + cursor_source.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='users'") + if cursor_source.fetchone() is None: + print("Table `user` does not exist in the source database.") + exit(1) + + conn_output = sqlite3.connect(args.output) + cursor_output = conn_output.cursor() + cursor_output.execute(""" + CREATE TABLE IF NOT EXISTS users ( + user_id TEXT PRIMARY KEY, + user_name TEXT NOT NULL, + email TEXT NOT NULL UNIQUE, + hashed_password TEXT NOT NULL + ) + """) + + import argon2 + import random + import string + + ph = argon2.PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) + + + def generate_random_string(length=16): + characters = string.ascii_letters + string.digits + string.punctuation + return ''.join(random.choice(characters) for _ in range(length)) + + + def generate_random_example_email(): + return f"{generate_random_string(8)}@changeme.com" + + + cursor_source.execute("SELECT user_id, user_name FROM users") + for row in cursor_source.fetchall(): + user_id, user_name = row + password = generate_random_string() + hashed_password = ph.hash(password) + email = generate_random_example_email() + cursor_output.execute("INSERT INTO users (user_id, user_name, email, hashed_password) VALUES (?, ?, ?, ?)", + (user_id, user_name, email, hashed_password)) + + conn_output.commit() diff --git a/package.json b/package.json index 432d92d..0ffb9e7 100644 --- a/package.json +++ b/package.json @@ -26,5 +26,6 @@ "@types/react": "18.2.33", "@types/react-dom": "18.2.14", "typescript": "^5.2.2" - } + }, + "packageManager": "pnpm@9.12.3+sha512.cce0f9de9c5a7c95bef944169cc5dfe8741abfb145078c0d508b868056848a87c81e626246cb60967cbd7fd29a6c062ef73ff840d96b3c86c40ac92cf4a813ee" } diff --git a/requirements.txt b/requirements.txt index 1f3b288..46bf46a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,5 @@ tqdm sqlite-vec sentence-transformers spacy -openai \ No newline at end of file +openai +argon2-cffi \ No newline at end of file diff --git a/user_utils.py b/user_utils.py new file mode 100644 index 0000000..f218052 --- /dev/null +++ b/user_utils.py @@ -0,0 +1,131 @@ +import sqlite3 +import uuid +import argon2 +import random +import string +import argparse + + +def generate_random_string(length=16): + characters = string.ascii_letters + string.digits + string.punctuation + return ''.join(random.choice(characters) for _ in range(length)) + + +class DatabaseUtils: + def __init__(self, db_path): + self.conn = sqlite3.connect(db_path) + self.cursor = self.conn.cursor() + self.ph = argon2.PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) + + def reset_user_password(self, user_id, new_password=generate_random_string()): + hashed_password = self.ph.hash(new_password) + self.cursor.execute("UPDATE users SET hashed_password = ? WHERE user_id = ?", (hashed_password, user_id)) + self.conn.commit() + return new_password + + def change_user_email(self, user_id, new_email): + self.cursor.execute("UPDATE users SET email = ? WHERE user_id = ?", (new_email, user_id)) + self.conn.commit() + + def change_user_name(self, user_id, new_username): + self.cursor.execute("UPDATE users SET user_name = ? WHERE user_id = ?", (new_username, user_id)) + self.conn.commit() + + def get_user_by_email(self, email): + self.cursor.execute("SELECT * FROM users WHERE email = ?", (email,)) + return self.cursor.fetchone() + + def get_user_by_id(self, user_id): + self.cursor.execute("SELECT * FROM users WHERE user_id = ?", (user_id,)) + return self.cursor.fetchone() + + def new_user(self, user_name, email, password=generate_random_string()): + hashed_password = self.ph.hash(password) + user_id = uuid.uuid4().hex + self.cursor.execute("INSERT INTO users (user_id, user_name, email, hashed_password) VALUES (?, ?, ?, ?)", + (user_id, user_name, email, hashed_password)) + self.conn.commit() + return password + + def delete_user(self, user_id): + self.cursor.execute("DELETE FROM users WHERE user_id = ?", (user_id,)) + self.conn.commit() + + def close(self): + self.conn.close() + + +def main(): + main_parser = argparse.ArgumentParser(description="Manage users") + main_parser.add_argument("--sqlite_path", type=str, required=True, help="Path to the user SQLite database") + user_commands_parser = main_parser.add_subparsers(dest="command", required=True) + + new_user_parser = user_commands_parser.add_parser("new", help="Create a new user") + new_user_parser.add_argument("--user_name", type=str, required=True, help="Username of the new user") + new_user_parser.add_argument("--email", type=str, required=True, help="Email of the new user") + new_user_parser.add_argument("--password", type=str, help="Password of the new user") + + delete_user_parser = user_commands_parser.add_parser("delete", help="Delete a user") + delete_user_parser.add_argument("--user_id", type=str, required=True, help="User ID to delete") + + reset_password_parser = user_commands_parser.add_parser("reset_password", help="Reset a user's password") + reset_password_parser.add_argument("--user_id", type=str, required=True, help="User ID to reset the password") + reset_password_parser.add_argument("--new_password", type=str, help="New password for the user") + + change_email_parser = user_commands_parser.add_parser("change_email", help="Change a user's email") + change_email_parser.add_argument("--user_id", type=str, required=True, help="User ID to change the email") + change_email_parser.add_argument("--new_email", type=str, required=True, help="New email for the user") + + change_username_parser = user_commands_parser.add_parser("change_username", help="Change a user's username") + change_username_parser.add_argument("--user_id", type=str, required=True, help="User ID to change the username") + change_username_parser.add_argument("--new_username", type=str, required=True, help="New username for the user") + + get_user_parser = user_commands_parser.add_parser("get", help="Get a user") + get_user_parser.add_argument("--user_id", type=str, help="User ID to get") + get_user_parser.add_argument("--email", type=str, help="Email to get") + + args = main_parser.parse_args() + + db_utils = DatabaseUtils(args.sqlite_path) + + match args.command: + case "new": + password = db_utils.new_user(args.user_name, args.email, args.password) + print(f"New user created with password: {password}") + case "delete": + db_utils.delete_user(args.user_id) + print("User deleted") + case "reset_password": + if args.new_password: + new_password = db_utils.reset_user_password(args.user_id, args.new_password) + else: + new_password = db_utils.reset_user_password(args.user_id) + print(f"Password reset to: {new_password}") + case "change_email": + db_utils.change_user_email(args.user_id, args.new_email) + print("Email changed") + case "change_username": + db_utils.change_user_name(args.user_id, args.new_username) + print("Username changed") + case "get": + if args.user_id: + user = db_utils.get_user_by_id(args.user_id) + elif args.email: + user = db_utils.get_user_by_email(args.email) + else: + user = None + + if user: + print(f"User ID: {user[0]}") + print(f"Username: {user[1]}") + print(f"Email: {user[2]}") + else: + print("User not found") + case _: + print("Invalid command") + + db_utils.close() + + +if __name__ == "__main__": + main() From f2ba6d1aa30ab553d7acd498b8ce96d82dba96f3 Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Sat, 2 Nov 2024 22:28:18 +0800 Subject: [PATCH 02/43] feat(database): auth_user --- database.py | 249 +++++++++++++++++++++++++++-------------------- requirements.txt | 3 +- 2 files changed, 143 insertions(+), 109 deletions(-) diff --git a/database.py b/database.py index 9a469f1..89d4c2c 100644 --- a/database.py +++ b/database.py @@ -4,37 +4,43 @@ import pandas as pd import threading import re +import argon2 import sqlite3 import sqlite_vec from dotenv import load_dotenv + class OldLabelData(TypedDict): # readable by frontend - record_id: str # a unit name for the annotation - sample_id: str # traditionally, mercury_{\d+} where \d is the sample number, e.g., 15 + record_id: str # a unit name for the annotation + sample_id: str # traditionally, mercury_{\d+} where \d is the sample number, e.g., 15 summary_start: int summary_end: int source_start: int source_end: int - consistent: str # used to be boolean - task_index: int # traditionally, \d+ where \d is the sample number, e.g., 15 + consistent: str # used to be boolean + task_index: int # traditionally, \d+ where \d is the sample number, e.g., 15 user_id: str note: str -class AnnotSpan(TypedDict): # In future expansion, the fields can be any user-defined fields - source: tuple[int, int] # optional + +class AnnotSpan(TypedDict): # In future expansion, the fields can be any user-defined fields + source: tuple[int, int] # optional summary: tuple[int, int] + class LabelData(TypedDict): # human annotation on a sample - annot_id: int - sample_id: int + annot_id: int + sample_id: int annot_spans: AnnotSpan annotator: str label: str # json string note: str -def convert_LabelData(lb: LabelData | OldLabelData, direction: Literal["new2old", "old2new"] ) -> LabelData | OldLabelData: + +def convert_LabelData(lb: LabelData | OldLabelData, + direction: Literal["new2old", "old2new"]) -> LabelData | OldLabelData: if direction == "old2new": return { "annot_id": lb["record_id"], @@ -63,11 +69,13 @@ def convert_LabelData(lb: LabelData | OldLabelData, direction: Literal["new2old" "note": lb["note"] } + class AnnotationLabelItem(TypedDict): text: str start: int end: int + class AnnotationItem(TypedDict): source: AnnotationLabelItem summary: AnnotationLabelItem @@ -108,35 +116,35 @@ class AnnotationData(TypedDict): # def fetch_annotations(sqlite_db_path: str) -> List[LabelData]: # db = sqlite3.connect(sqlite_db_path) - # cmd = "SELECT annot_id, doc_id, annot_spans, annotator, label FROM annotations" - # annotations = db.execute(cmd).fetchall() - # db.close() - - # label_data = [] - # for annot_id, doc_id, annot_spans, annotator, label in annotations: - # annot_spans = json.loads(text_spans) - # label_data.append({ - # "record_id": annot_id, - # "sample_id": doc_id, # traditionally, this is mercury_{\d+} where \d is the sample number, e.g., 15 - # "summary_start": text_spans["summary"][0], - # "summary_end": text_spans["summary"][1], - # "source_start": text_spans["source"][0], - # "source_end": text_spans["source"][1], - # "consistent": label, - # "task_index": doc_id, # traditionally, this is \d+ where \d is the sample number, e.g., 15 - # "user_id": annotator - # }) - - # annotations = pd.DataFrame.from_records( - # label_data, - # columns=["record_id", "sample_id", "summary_start", "summary_end", "source_start", - # "source_end", "consistent", "task_index", "user_id"]) - # return annotations +# cmd = "SELECT annot_id, doc_id, annot_spans, annotator, label FROM annotations" +# annotations = db.execute(cmd).fetchall() +# db.close() + +# label_data = [] +# for annot_id, doc_id, annot_spans, annotator, label in annotations: +# annot_spans = json.loads(text_spans) +# label_data.append({ +# "record_id": annot_id, +# "sample_id": doc_id, # traditionally, this is mercury_{\d+} where \d is the sample number, e.g., 15 +# "summary_start": text_spans["summary"][0], +# "summary_end": text_spans["summary"][1], +# "source_start": text_spans["source"][0], +# "source_end": text_spans["source"][1], +# "consistent": label, +# "task_index": doc_id, # traditionally, this is \d+ where \d is the sample number, e.g., 15 +# "user_id": annotator +# }) + +# annotations = pd.DataFrame.from_records( +# label_data, +# columns=["record_id", "sample_id", "summary_start", "summary_end", "source_start", +# "source_end", "consistent", "task_index", "user_id"]) +# return annotations class Database: -# class Annotate: + # class Annotate: # def __init__(self, annotation_corpus_id: int, vectara_client: Vectara = Vectara()): - def __init__(self, sqlite_db_path: str): + def __init__(self, mercury_db_path: str, user_db_path: str): self.lock = threading.Lock() # self.vectara_client = vectara_client # self.annotation_corpus_id = annotation_corpus_id @@ -150,25 +158,34 @@ def __init__(self, sqlite_db_path: str): # self.annotations = fetch_annotations(sqlite_db_path) # prepare the database - db = sqlite3.connect(sqlite_db_path) - print ("Open db at ", sqlite_db_path) - db.execute("CREATE TABLE IF NOT EXISTS annotations (\ + mercury_db = sqlite3.connect(mercury_db_path) + print("Open db at ", mercury_db_path) + mercury_db.execute("CREATE TABLE IF NOT EXISTS annotations (\ annot_id INTEGER PRIMARY KEY AUTOINCREMENT, \ sample_id INTEGER, \ annot_spans TEXT, \ annotator TEXT, \ label TEXT, \ note TEXT)") - db.execute( - "CREATE TABLE IF NOT EXISTS users (user_id TEXT PRIMARY KEY, user_name TEXT)" - ) - db.enable_load_extension(True) - sqlite_vec.load(db) - db.enable_load_extension(False) - db.commit() - self.db = db # Forrst is unsure whether it is a good idea to keep the db connection open - - @staticmethod # Forrest: Seems no need to update this function after Vectara-to-SQLite migration + # mercury_db.execute( + # "CREATE TABLE IF NOT EXISTS users (user_id TEXT PRIMARY KEY, user_name TEXT)" + # ) + mercury_db.enable_load_extension(True) + sqlite_vec.load(mercury_db) + mercury_db.enable_load_extension(False) + mercury_db.commit() + user_db = sqlite3.connect(user_db_path) + user_db.execute("""CREATE TABLE IF NOT EXISTS users ( + user_id TEXT PRIMARY KEY, + user_name TEXT NOT NULL, + email TEXT NOT NULL UNIQUE, + hashed_password TEXT NOT NULL)""") + user_db.commit() + self.mercury_db = mercury_db # Forrst is unsure whether it is a good idea to keep the db connection open + self.user_db = user_db + self.ph = argon2.PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) + + @staticmethod # Forrest: Seems no need to update this function after Vectara-to-SQLite migration def database_lock(): def decorator(func): def wrapper(self, *args, **kwargs): @@ -176,16 +193,18 @@ def wrapper(self, *args, **kwargs): result = func(self, *args, **kwargs) self.lock.release() return result + return wrapper + return decorator def fetch_data_for_labeling(self): """Fetch the source-summary pairs for labeling from the database.""" data_for_labeling = {} - sectioned_chunks = {} + sectioned_chunks = {} # db = sqlite3.connect(sqlite_db_path) - db = self.db + db = self.mercury_db texts = db.execute("SELECT text, text_type, sample_id, chunk_offset FROM chunks").fetchall() """ texts = [('The quick brown fox.', 'source', 1, 0), @@ -230,7 +249,7 @@ def fetch_data_for_labeling(self): data_for_labeling = [ { - "_id": str(sample_id), + "_id": str(sample_id), "source": " ".join(sectioned_chunks[sample_id]["source"].values()), "summary": " ".join(sectioned_chunks[sample_id]["summary"].values()) } @@ -256,10 +275,10 @@ def fetch_data_for_labeling(self): data_for_labeling.sort(key=lambda x: int(x["_id"])) return data_for_labeling - + def fetch_configs(self): # db = sqlite3.connect(sqlite_db_path) - db = self.db + db = self.mercury_db configs = db.execute("SELECT key, value FROM config").fetchall() return {key: value for key, value in configs} @@ -278,7 +297,7 @@ def push_annotation(self, label_data: OldLabelData): # return sql_cmd = "SELECT * FROM annotations WHERE sample_id = ? AND annot_spans = ? AND annotator = ? AND label = ? AND note = ?" - res = self.db.execute(sql_cmd, ( + res = self.mercury_db.execute(sql_cmd, ( label_data["sample_id"], json.dumps(label_data["annot_spans"]), label_data["annotator"], @@ -291,7 +310,7 @@ def push_annotation(self, label_data: OldLabelData): # record_id = uuid.uuid4().hex # No need for this line in SQLite because it auto-increments # label_data["record_id"] = record_id # self.annotations.loc[len(self.annotations.index)] = ( - # label_data["record_id"], + # label_data["record_id"], # label_data["sample_id"], # label_data["summary_start"], # label_data["summary_end"], @@ -311,14 +330,14 @@ def push_annotation(self, label_data: OldLabelData): # label_data = convert_LabelData(label_data, "old2new") sql_cmd = "INSERT INTO annotations (sample_id, annot_spans, annotator, label, note) VALUES (?, ?, ?, ?, ?)" - self.db.execute(sql_cmd, ( + self.mercury_db.execute(sql_cmd, ( label_data["sample_id"], json.dumps(label_data["annot_spans"]), label_data["annotator"], label_data["label"], label_data["note"] )) - self.db.commit() + self.mercury_db.commit() @database_lock() # def delete_annotation(self, record_id: str, user_id: str): @@ -332,33 +351,30 @@ def delete_annotation(self, record_id: str, annotator: str): # self.annotations.drop(record_index, inplace=True) # self.vectara_client.delete_document(self.annotation_corpus_id, record_id) sql_cmd = "DELETE FROM annotations WHERE annot_id = ? AND annotator = ?" - self.db.execute(sql_cmd, (int(record_id), annotator)) - self.db.commit() - + self.mercury_db.execute(sql_cmd, (int(record_id), annotator)) + self.mercury_db.commit() + @database_lock() - def add_user(self, user_id: str, user_name: str): + def add_user(self, user_id: str, user_name: str): #TODO: remove this method since now only admin can add user sql_cmd = "INSERT INTO users (user_id, user_name) VALUES (?, ?)" - self.db.execute(sql_cmd, (user_id, user_name)) - self.db.commit() - + self.mercury_db.execute(sql_cmd, (user_id, user_name)) + self.mercury_db.commit() + @database_lock() def change_user_name(self, user_id: str, user_name: str): - sql_cmd = "UPDATE users SET user_name = ? WHERE user_id = ?" - self.db.execute(sql_cmd, (user_name, user_id)) - self.db.commit() - + self.user_db.execute("UPDATE users SET user_name = ? WHERE user_id = ?", (user_name, user_id)) + self.mercury_db.commit() + @database_lock() def get_user_name(self, user_id: str) -> str: - sql_cmd = "SELECT user_name FROM users WHERE user_id = ?" - res = self.db.execute(sql_cmd, (user_id,)) + res = self.user_db.execute("SELECT user_name FROM users WHERE user_id = ?", (user_id,)) user_name = res.fetchone() if user_name is None: return None return user_name[0] - + def get_user_name_without_lock(self, user_id: str) -> str: - sql_cmd = "SELECT user_name FROM users WHERE user_id = ?" - res = self.db.execute(sql_cmd, (user_id,)) + res = self.user_db.execute("SELECT user_name FROM users WHERE user_id = ?", (user_id,)) user_name = res.fetchone() if user_name is None: return None @@ -366,19 +382,19 @@ def get_user_name_without_lock(self, user_id: str) -> str: @database_lock() # def export_user_data(self, user_id: str) -> list[LabelData]: - def export_user_data(self, annotator: str) -> list[LabelData]: + def export_user_data(self, annotator_uuid: str) -> list[LabelData]: # return self.annotations[self.annotations["user_id"] == user_id].to_dict(orient="records") sql_cmd = "SELECT * FROM annotations WHERE annotator = ?" - res = self.db.execute(sql_cmd, (annotator,)) + res = self.mercury_db.execute(sql_cmd, (annotator_uuid,)) annotations = res.fetchall() - label_data = [] # in OldLabelData format - for annot_id, sample_id, annot_spans, annotator, label, note in annotations: + label_data = [] # in OldLabelData format + for annot_id, sample_id, annot_spans, annotator_uuid, label, note in annotations: annot_spans = json.loads(annot_spans) label_data.append(convert_LabelData({ "annot_id": annot_id, - "sample_id": sample_id, + "sample_id": sample_id, "annot_spans": annot_spans, - "annotator": annotator, + "annotator": annotator_uuid, "label": json.loads(label), "note": note }, "new2old")) @@ -392,25 +408,25 @@ def export_task_history(self, sample_id: int, annotator: str) -> list[LabelData] # (self.annotations["task_index"] == task_index) # ].to_dict(orient="records") sql_cmd = "SELECT * FROM annotations WHERE annotator = ? AND sample_id = ?" - res = self.db.execute(sql_cmd, (annotator, sample_id)) + res = self.mercury_db.execute(sql_cmd, (annotator, sample_id)) annotations = res.fetchall() label_data = [] for annot_id, sample_id, annot_spans, annotator, label, note in annotations: annot_spans = json.loads(annot_spans) label_data.append(convert_LabelData({ "annot_id": annot_id, - "sample_id": sample_id, + "sample_id": sample_id, "annot_spans": annot_spans, "annotator": annotator, "label": json.loads(label), "note": note }, "new2old")) return label_data - + @database_lock() def dump_annotator_labels(self, annotator: str): sql_cmd = "SELECT * FROM annotations WHERE annotator = ?" - res = self.db.execute(sql_cmd, (annotator,)) + res = self.mercury_db.execute(sql_cmd, (annotator,)) annotations = res.fetchall() results = [] results_dict = {} @@ -419,12 +435,14 @@ def dump_annotator_labels(self, annotator: str): full_texts = {} for text_type in ["source", "summary"]: sql_cmd = "SELECT text FROM chunks WHERE sample_id = ? AND text_type = ? ORDER BY chunk_offset" - res = self.db.execute(sql_cmd, (sample_id, text_type)) - text = res.fetchall() # text = [('The quick brown fox.',), ('Jumps over a lazy dog.',)] + res = self.mercury_db.execute(sql_cmd, (sample_id, text_type)) + text = res.fetchall() # text = [('The quick brown fox.',), ('Jumps over a lazy dog.',)] text = [t[0] for t in text] full_texts[text_type] = " ".join(text) - - result_local = {"annot_id": annot_id, "sample_id": sample_id, "annotator": annotator, "label": json.loads(label), "note": note, "annotator_name": self.get_user_name_without_lock(annotator)} + + result_local = {"annot_id": annot_id, "sample_id": sample_id, "annotator": annotator, + "label": json.loads(label), "note": note, + "annotator_name": self.get_user_name_without_lock(annotator)} # annot_spans example: {'source': (1, 10), 'summary': (7, 10)} annot_spans = json.loads(annot_spans) for text_type, (start, end) in annot_spans.items(): @@ -435,18 +453,20 @@ def dump_annotator_labels(self, annotator: str): results.append(result_local) - results_dict.setdefault(sample_id, {"source": full_texts["source"], "summary": full_texts["summary"], "annotations": []}) + results_dict.setdefault(sample_id, {"source": full_texts["source"], "summary": full_texts["summary"], + "annotations": []}) results_dict[sample_id]["annotations"].append(result_local) results_nested = [{"sample_id": key, **value} for key, value in results_dict.items()] # TODO: copy and paste from dump_annotation is too ugly. Please turn common code to a function - sql_cmd = "SELECT * from sample_meta" # get the metadata - res = self.db.execute(sql_cmd) + sql_cmd = "SELECT * from sample_meta" # get the metadata + res = self.mercury_db.execute(sql_cmd) sample_meta = res.fetchall() sample_meta_dict = {sample_id: json.loads(json_meta) for sample_id, json_meta in sample_meta} - sample_meta_dict = {sample_id: {f"meta_{k}": v for k, v in meta.items()} for sample_id, meta in sample_meta_dict.items()} + sample_meta_dict = {sample_id: {f"meta_{k}": v for k, v in meta.items()} for sample_id, meta in + sample_meta_dict.items()} # add metadata to each dict in results_nested new_results_nested = [] @@ -467,23 +487,25 @@ def dump_annotation( ): sql_cmd = "SELECT * FROM annotations" - res = self.db.execute(sql_cmd) + res = self.mercury_db.execute(sql_cmd) annotations = res.fetchall() # match annotations with chunks by doc_id results = [] - results_dict = {} # keys are sample_id, values are source text, summary text, and each pair of spans and labels and annotators + results_dict = {} # keys are sample_id, values are source text, summary text, and each pair of spans and labels and annotators for annot_id, sample_id, annot_spans, annotator, label, note in annotations: # find the source and summary text by doc_id full_texts = {} for text_type in ["source", "summary"]: sql_cmd = "SELECT text FROM chunks WHERE sample_id = ? AND text_type = ? ORDER BY chunk_offset" - res = self.db.execute(sql_cmd, (sample_id, text_type)) - text = res.fetchall() # text = [('The quick brown fox.',), ('Jumps over a lazy dog.',)] + res = self.mercury_db.execute(sql_cmd, (sample_id, text_type)) + text = res.fetchall() # text = [('The quick brown fox.',), ('Jumps over a lazy dog.',)] text = [t[0] for t in text] full_texts[text_type] = " ".join(text) - - result_local = {"annot_id": annot_id, "sample_id": sample_id, "annotator": annotator, "label": json.loads(label), "note": note, "annotator_name": self.get_user_name_without_lock(annotator)} + + result_local = {"annot_id": annot_id, "sample_id": sample_id, "annotator": annotator, + "label": json.loads(label), "note": note, + "annotator_name": self.get_user_name_without_lock(annotator)} # annot_spans example: {'source': (1, 10), 'summary': (7, 10)} annot_spans = json.loads(annot_spans) for text_type, (start, end) in annot_spans.items(): @@ -493,16 +515,18 @@ def dump_annotation( results.append(result_local) - results_dict.setdefault(sample_id, {"source": full_texts["source"], "summary": full_texts["summary"], "annotations": []}) + results_dict.setdefault(sample_id, {"source": full_texts["source"], "summary": full_texts["summary"], + "annotations": []}) results_dict[sample_id]["annotations"].append(result_local) results_nested = [{"sample_id": key, **value} for key, value in results_dict.items()] - sql_cmd = "SELECT * from sample_meta" # get the metadata - res = self.db.execute(sql_cmd) + sql_cmd = "SELECT * from sample_meta" # get the metadata + res = self.mercury_db.execute(sql_cmd) sample_meta = res.fetchall() sample_meta_dict = {sample_id: json.loads(json_meta) for sample_id, json_meta in sample_meta} - sample_meta_dict = {sample_id: {f"meta_{k}": v for k, v in meta.items()} for sample_id, meta in sample_meta_dict.items()} + sample_meta_dict = {sample_id: {f"meta_{k}": v for k, v in meta.items()} for sample_id, meta in + sample_meta_dict.items()} # add metadata to each dict in results_nested new_results_nested = [] @@ -516,22 +540,31 @@ def dump_annotation( full_texts = {} for text_type in ["source", "summary"]: sql_cmd = "SELECT text FROM chunks WHERE sample_id = ? AND text_type = ? ORDER BY chunk_offset" - res = self.db.execute(sql_cmd, (sample_id, text_type)) - text = res.fetchall() # text = [('The quick brown fox.',), ('Jumps over a lazy dog.',)] + res = self.mercury_db.execute(sql_cmd, (sample_id, text_type)) + text = res.fetchall() # text = [('The quick brown fox.',), ('Jumps over a lazy dog.',)] text = [t[0] for t in text] full_texts[text_type] = " ".join(text) - sample_dict = {"sample_id": sample_id, "source": full_texts["source"], "summary": full_texts["summary"], "annotations": []} + sample_dict = {"sample_id": sample_id, "source": full_texts["source"], "summary": full_texts["summary"], + "annotations": []} sample_dict.update(sample_meta_dict[sample_id]) new_results_nested.append(sample_dict) - + results_nested = new_results_nested results_nested = sorted(results_nested, key=lambda d: d['sample_id']) - + if dump_file is None: return results_nested with open(dump_file, "w") as f: json.dump(results_nested, f, indent=2, ensure_ascii=False) - #TODO add JSONL support. Automatically detect file format based on filename extension + # TODO add JSONL support. Automatically detect file format based on filename extension + + @database_lock() + def auth_user(self, email: str, password: str): + res = self.user_db.execute("SELECT hashed_password FROM users WHERE email = ?", (email,)) + hashed_password = res.fetchone() + if hashed_password is None: + return False + return self.ph.verify(hashed_password[0], password) if __name__ == "__main__": @@ -549,7 +582,7 @@ def get_env_id_value(env_name: str) -> int | None: parser = argparse.ArgumentParser( - description="Dump all annotations from a Vectara corpus to a JSON file.", + description="Dump all annotations from a Vectara corpus to a JSON file.", formatter_class=argparse.ArgumentDefaultsHelpFormatter ) parser.add_argument("sqlite_db_path", type=str, help="Path to the SQLite database") diff --git a/requirements.txt b/requirements.txt index 46bf46a..1792c6d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,5 @@ sqlite-vec sentence-transformers spacy openai -argon2-cffi \ No newline at end of file +argon2-cffi +python-multipart \ No newline at end of file From 9d15e3592255ac3d699436b1677ec851236fc198 Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Sat, 2 Nov 2024 23:04:17 +0800 Subject: [PATCH 03/43] feat: Authentication --- app/login/page.tsx | 36 +++++++ database.py | 1 - requirements.txt | 3 +- server.py | 109 +++++++++++++++------ user_utils.py | 5 +- utils/request.ts | 231 +++++++++++++++++++++++++-------------------- 6 files changed, 252 insertions(+), 133 deletions(-) create mode 100644 app/login/page.tsx diff --git a/app/login/page.tsx b/app/login/page.tsx new file mode 100644 index 0000000..a661c25 --- /dev/null +++ b/app/login/page.tsx @@ -0,0 +1,36 @@ +"use client" + +import { Button, Field, Input, makeResetStyles, Title1, tokens } from "@fluentui/react-components" +import { login } from "../../utils/request" + +const useStackClassName = makeResetStyles({ + display: "flex", + flexDirection: "column", + maxWidth: "350px", + rowGap: tokens.spacingVerticalL, +}) + +export default function Login() { + async function formAction(formData) { + if (formData.get("email") && formData.get("password")) { + await login(formData.get("email"), formData.get("password")) + } + } + + return ( + <> + Login +
+ + + + + + + +
+ + ) +} diff --git a/database.py b/database.py index 89d4c2c..a6282a6 100644 --- a/database.py +++ b/database.py @@ -566,7 +566,6 @@ def auth_user(self, email: str, password: str): return False return self.ph.verify(hashed_password[0], password) - if __name__ == "__main__": import argparse import os diff --git a/requirements.txt b/requirements.txt index 1792c6d..299a3ad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,5 @@ sentence-transformers spacy openai argon2-cffi -python-multipart \ No newline at end of file +python-multipart +pyjwt \ No newline at end of file diff --git a/server.py b/server.py index 5c549f3..1e8f8ba 100644 --- a/server.py +++ b/server.py @@ -7,8 +7,9 @@ import uvicorn from dotenv import load_dotenv -from fastapi import FastAPI, Header +from fastapi import Depends, FastAPI, Header, HTTPException, status from fastapi.staticfiles import StaticFiles +from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm from pydantic import BaseModel from typing import List @@ -25,6 +26,10 @@ from ingester import Embedder from database import Database +import jwt +from jwt.exceptions import InvalidTokenError +from datetime import datetime, timedelta, timezone + app = FastAPI() app.add_middleware( CORSMiddleware, @@ -33,12 +38,15 @@ allow_methods=["*"], allow_headers=["*"], ) + + # vectara_client = Vectara() def serialize_f32(vector: List[float]) -> bytes: """serializes a list of floats into a compact "raw bytes" format""" return struct.pack("%sf" % len(vector), *vector) + class Label(BaseModel): summary_start: int summary_end: int @@ -47,27 +55,61 @@ class Label(BaseModel): consistent: list[str] note: str + class Selection(BaseModel): start: int end: int from_summary: bool + class Name(BaseModel): name: str -@app.get("/candidate_labels") -async def get_labels() -> list: # get all candidate labels for human annotators to choose from + +class Token(BaseModel): + access_token: str + token_type: str + + +SECRET_KEY = "" +ALGORITHM = "HS256" +ACCESS_TOKEN_EXPIRE_MINUTES = 10080 + +oauth2_scheme = OAuth2PasswordBearer(tokenUrl="login") + + +def create_access_token(email: str): + to_encode = {"email": email} + expire = datetime.now(timezone.utc) + timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES) + to_encode.update({"exp": expire}) + encoded_jwt = jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM) + return encoded_jwt + + +@app.post("/login") +async def login(form_data: Annotated[OAuth2PasswordRequestForm, Depends()]) -> Token: + if not database.auth_user(form_data.username, form_data.password): + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, + detail="Incorrect username or password", + headers={"WWW-Authenticate": "Bearer"}) + access_token = create_access_token(form_data.username) + return Token(access_token=access_token, token_type="bearer") + +@app.get("/candidate_labels") +async def get_labels() -> list: # get all candidate labels for human annotators to choose from with open("labels.yaml") as f: labels = yaml.safe_load(f) return labels -@app.get("/user/new") # please update the route name to be more meaningful, e.g., /user/new_user + +@app.get("/user/new") # please update the route name to be more meaningful, e.g., /user/new_user async def create_new_user(): user_id = uuid.uuid4().hex user_name = "New User" database.add_user(user_id, user_name) return {"key": user_id, "name": user_name} + @app.post("/user/name") async def update_user_name(name: Name, user_key: Annotated[str, Header()]): if user_key.startswith('"') and user_key.endswith('"'): @@ -75,6 +117,7 @@ async def update_user_name(name: Name, user_key: Annotated[str, Header()]): database.change_user_name(user_key, name.name) return {"message": "success"} + @app.get("/user/me") async def get_user_name(user_key: Annotated[str, Header()]): if user_key.startswith('"') and user_key.endswith('"'): @@ -85,7 +128,8 @@ async def get_user_name(user_key: Annotated[str, Header()]): else: return {"name": username} -@app.get("/user/export") # please update the route name to be more meaningful, e.g., /user/export_user_data + +@app.get("/user/export") # please update the route name to be more meaningful, e.g., /user/export_user_data async def export_user_data(user_key: Annotated[str, Header()]): if user_key.startswith('"') and user_key.endswith('"'): user_key = user_key[1:-1] @@ -128,29 +172,30 @@ async def post_task(task_index: int, label: Label, user_key: Annotated[str, Head # task_index=task_index, # user_id=user_key, # ) - + sample_id = task_index annot_spans = {} if label.summary_start != -1: annot_spans["summary"] = (label.summary_start, label.summary_end) if label.source_start != -1: annot_spans["source"] = (label.source_start, label.source_end) - + annotator = user_key - + label_string = json.dumps(label.consistent) - + database.push_annotation({ "sample_id": sample_id, "annotator": annotator, "label": label_string, "annot_spans": annot_spans, "note": label.note - }) # the label_data is in databse.OldLabelData format + }) # the label_data is in databse.OldLabelData format return {"message": "success"} -@app.post("/task/{task_index}/select") # TODO: to be updated by Forrest using openAI's API or local model to embed text on the fly +@app.post( + "/task/{task_index}/select") # TODO: to be updated by Forrest using openAI's API or local model to embed text on the fly async def post_selections(task_index: int, selection: Selection): if task_index >= len(tasks): return {"error": "Invalid task index"} @@ -158,9 +203,9 @@ async def post_selections(task_index: int, selection: Selection): return {"error": "Invalid task index"} # use_id = source_corpus_id if selection.from_summary else summary_corpus_id query = ( - tasks[task_index]["source"][selection.start : selection.end] + tasks[task_index]["source"][selection.start: selection.end] if not selection.from_summary - else tasks[task_index]["summary"][selection.start : selection.end] + else tasks[task_index]["summary"][selection.start: selection.end] ) id_ = tasks[task_index]["_id"] @@ -176,7 +221,7 @@ async def post_selections(task_index: int, selection: Selection): # first embedd query embedding = embedder.embed([query], embedding_dimension=configs["embedding_dimension"])[0] - + # Then get the chunk_id's from the opposite document sql_cmd = "SELECT chunk_id, text FROM chunks WHERE text_type = ? AND sample_id = ?" if selection.from_summary: @@ -186,9 +231,9 @@ async def post_selections(task_index: int, selection: Selection): chunk_id_and_text = database.db.execute(sql_cmd, [text_type, task_index]).fetchall() search_chunk_ids = [row[0] for row in chunk_id_and_text] - vecter_db_row_ids = [str(x+1) for x in search_chunk_ids] # rowid starts from 1 while chunk_id starts from 0 + vecter_db_row_ids = [str(x + 1) for x in search_chunk_ids] # rowid starts from 1 while chunk_id starts from 0 - if len(search_chunk_ids) == 1: # no need for vector search + if len(search_chunk_ids) == 1: # no need for vector search selections = [{ "score": 1.0, "offset": 0, @@ -204,13 +249,13 @@ async def post_selections(task_index: int, selection: Selection): SELECT \ rowid, \ distance \ - FROM embeddings " \ - " WHERE rowid IN ({0})" \ - "AND embedding MATCH '{1}' \ - ORDER BY distance \ - LIMIT 5;".format(', '.join(vecter_db_row_ids), embedding) + FROM embeddings " \ + " WHERE rowid IN ({0})" \ + "AND embedding MATCH '{1}' \ + ORDER BY distance \ + LIMIT 5;".format(', '.join(vecter_db_row_ids), embedding) # print ("SQL_CMD", sql_cmd) - + # vector_search_result = database.db.execute(sql_cmd, [*search_chunk_ids, serialize_f32(embedding)]).fetchall() vector_search_result = database.db.execute(sql_cmd).fetchall() # [(2, 0.20000001788139343), (1, 0.40000003576278687)] @@ -219,7 +264,8 @@ async def post_selections(task_index: int, selection: Selection): chunk_ids_of_top_k = [row[0] for row in vector_search_result] # get the char_offset and len from the chunks table based on the chunk_ids - sql_cmd = "SELECT chunk_id, text, char_offset FROM chunks WHERE chunk_id in ({0});".format(', '.join('?' for _ in chunk_ids_of_top_k)) + sql_cmd = "SELECT chunk_id, text, char_offset FROM chunks WHERE chunk_id in ({0});".format( + ', '.join('?' for _ in chunk_ids_of_top_k)) search_chunk_ids = [row[0] for row in vector_search_result] response = database.db.execute(sql_cmd, search_chunk_ids).fetchall() # [(1, 'This is a test.', 0, 14), (2, 'This is a test.', 15, 14)] @@ -233,7 +279,7 @@ async def post_selections(task_index: int, selection: Selection): text = i[1] selections.append( { - "score": 1 - score, # semantic similarity is 1 - distance + "score": 1 - score, # semantic similarity is 1 - distance "offset": offset, "len": len(text), "to_doc": selection.from_summary, @@ -269,32 +315,39 @@ async def delete_annotation(record_id: str, user_key: Annotated[str, Header()]): database.delete_annotation(record_id, user_key) return {"message": f"delete anntation {record_id} success"} + @app.get("/labels") async def get_labels(): return database.dump_annotation(dump_file=None) + @app.get("/history") # redirect route to history.html async def history(): return FileResponse("dist/history.html") + @app.get("/viewer") async def viewer(): return FileResponse("dist/viewer.html") -if __name__ == "__main__": +if __name__ == "__main__": app.mount("/", StaticFiles(directory="dist", html=True), name="dist") import argparse parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument("--sqlite_db", type=str, default="./mercury.sqlite") + parser.add_argument("--mercury_db", type=str, required=True, default="./mercury.sqlite") + parser.add_argument("--user_db", type=str, required=True, default="./user.sqlite") parser.add_argument("--port", type=int, default=8000) + parser.add_argument("--jwt_secret", type=str, required=True) args = parser.parse_args() - print ("Using sqlite db: ", args.sqlite_db) + print("Using Mercury SQLite db: ", args.mercury_db) + print("Using User SQLite db: ", args.user_db) + SECRET_KEY = args.jwt_secret - database = Database(args.sqlite_db) + database = Database(args.mercury_db, args.user_db) # TODO: the name 'tasks' can be misleading. It should be changed to something more descriptive. tasks = database.fetch_data_for_labeling() diff --git a/user_utils.py b/user_utils.py index f218052..fa95fd2 100644 --- a/user_utils.py +++ b/user_utils.py @@ -90,7 +90,10 @@ def main(): match args.command: case "new": - password = db_utils.new_user(args.user_name, args.email, args.password) + if args.password: + password = db_utils.new_user(args.user_name, args.email, args.password) + else: + password = db_utils.new_user(args.user_name, args.email) print(f"New user created with password: {password}") case "delete": db_utils.delete_user(args.user_id) diff --git a/utils/request.ts b/utils/request.ts index f9a0cec..3e8cd72 100644 --- a/utils/request.ts +++ b/utils/request.ts @@ -1,139 +1,166 @@ import type { AllTasksLength, LabelData, LabelRequest, Normal, SectionResponse, SelectionRequest, Task } from "./types" -const backend = process.env.NEXT_PUBLIC_BACKEND || ""; +const backend = process.env.NEXT_PUBLIC_BACKEND || "" const getKey = async (): Promise => { - const hasUserMe = await checkUserMe(); - - const key = localStorage.getItem("key") - if (key === "" || key === null) { - const response = await fetch(`${backend}/user/new`); - const data = await response.json(); - localStorage.setItem("key", data.key); - if (hasUserMe) { - localStorage.setItem("name", data.name); - } - return data.key; - } - + const hasUserMe = await checkUserMe() + + const key = localStorage.getItem("key") + if (key === "" || key === null) { + const response = await fetch(`${backend}/user/new`) + const data = await response.json() + localStorage.setItem("key", data.key) if (hasUserMe) { - const nameResponse = await fetch(`${backend}/user/me`, { - headers: { - "User-Key": key, - }, - }); - - const data = await nameResponse.json(); - if ("error" in data) { - localStorage.removeItem("key"); - localStorage.removeItem("name"); - return getKey(); - } - localStorage.setItem("name", data.name); - return Promise.resolve(key) - } + localStorage.setItem("name", data.name) + } + return data.key + } + + if (hasUserMe) { + const nameResponse = await fetch(`${backend}/user/me`, { + headers: { + "User-Key": key, + }, + }) + + const data = await nameResponse.json() + if ("error" in data) { + localStorage.removeItem("key") + localStorage.removeItem("name") + return getKey() + } + localStorage.setItem("name", data.name) + return Promise.resolve(key) + } } const checkUserMe = async (): Promise => { - const response = await fetch(`${backend}/user/me`, { - headers: { - "User-Key": "OK_CHECK", - }, - }); - return response.ok; + const response = await fetch(`${backend}/user/me`, { + headers: { + "User-Key": "OK_CHECK", + }, + }) + return response.ok } const changeName = async (name: string): Promise => { - const key = await getKey(); - const response = await fetch(`${backend}/user/name`, { - method: "POST", - headers: { - "Content-Type": "application/json", - "User-Key": key, - }, - body: JSON.stringify({ name }), - }); - const data = await response.json(); - localStorage.setItem("name", name); - return data as Normal; + const key = await getKey() + const response = await fetch(`${backend}/user/name`, { + method: "POST", + headers: { + "Content-Type": "application/json", + "User-Key": key, + }, + body: JSON.stringify({ name }), + }) + const data = await response.json() + localStorage.setItem("name", name) + return data as Normal } const getAllLabels = async (): Promise<(string | object)[]> => { - const response = await fetch(`${backend}/candidate_labels`); - const data = await response.json(); - return data as string[]; + const response = await fetch(`${backend}/candidate_labels`) + const data = await response.json() + return data as string[] } const getAllTasksLength = async (): Promise => { - const response = await fetch(`${backend}/task`); - const data = await response.json(); - return data as AllTasksLength; + const response = await fetch(`${backend}/task`) + const data = await response.json() + return data as AllTasksLength } const getSingleTask = async (taskIndex: number): Promise => { - const response = await fetch(`${backend}/task/${taskIndex}`); - const data = await response.json(); - return data as Task | Error; + const response = await fetch(`${backend}/task/${taskIndex}`) + const data = await response.json() + return data as Task | Error } const selectText = async (taskIndex: number, req: SelectionRequest): Promise => { - const response = await fetch(`${backend}/task/${taskIndex}/select`, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(req), - }); - const data = await response.json(); - return data as SectionResponse | Error; + const response = await fetch(`${backend}/task/${taskIndex}/select`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(req), + }) + const data = await response.json() + return data as SectionResponse | Error } const labelText = async (taskIndex: number, req: LabelRequest): Promise => { - const key = await getKey(); - const response = await fetch(`${backend}/task/${taskIndex}/label`, { - method: "POST", - headers: { - "Content-Type": "application/json", - "User-Key": key, - }, - body: JSON.stringify(req), - }); - const data = await response.json(); - return data as Normal; + const key = await getKey() + const response = await fetch(`${backend}/task/${taskIndex}/label`, { + method: "POST", + headers: { + "Content-Type": "application/json", + "User-Key": key, + }, + body: JSON.stringify(req), + }) + const data = await response.json() + return data as Normal } const exportLabel = async (): Promise => { - const key = await getKey(); - const response = await fetch(`${backend}/user/export`, { - headers: { - "User-Key": key, - }, - }); - const data = await response.json(); - return data as LabelData[]; + const key = await getKey() + const response = await fetch(`${backend}/user/export`, { + headers: { + "User-Key": key, + }, + }) + const data = await response.json() + return data as LabelData[] } const getTaskHistory = async (taskIndex: number): Promise => { - const key = await getKey(); - const response = await fetch(`${backend}/task/${taskIndex}/history`, { - headers: { - "User-Key": key, - }, - }); - const data = await response.json(); - return data as LabelData[]; + const key = await getKey() + const response = await fetch(`${backend}/task/${taskIndex}/history`, { + headers: { + "User-Key": key, + }, + }) + const data = await response.json() + return data as LabelData[] } const deleteRecord = async (recordId: string): Promise => { - const key = await getKey(); - const response = await fetch(`${backend}/record/${recordId}`, { - method: "DELETE", - headers: { - "User-Key": key, - }, - }); - const data = await response.json(); - return data as Normal; + const key = await getKey() + const response = await fetch(`${backend}/record/${recordId}`, { + method: "DELETE", + headers: { + "User-Key": key, + }, + }) + const data = await response.json() + return data as Normal +} + +const login = async (email: string, password: string): Promise => { +const response = await fetch(`${backend}/login`, { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded", + }, + body: new URLSearchParams({ "username": email, "password": password }), +}) + const data = await response.json() + if ("access_token" in data) { + localStorage.setItem("access_token", data.access_token) + } + return data as Normal } -export { getAllTasksLength, getSingleTask, selectText, labelText, exportLabel, getTaskHistory, deleteRecord, getAllLabels, changeName, checkUserMe } +export { + getAllTasksLength, + getSingleTask, + selectText, + labelText, + exportLabel, + getTaskHistory, + deleteRecord, + getAllLabels, + changeName, + checkUserMe, + login, +} From 47d7ee239e0cd8fe6fd4059dab8b4e114f32acef Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Sun, 3 Nov 2024 11:50:21 +0800 Subject: [PATCH 04/43] feat: Change authentication method to access_token --- database.py | 20 ++++---- server.py | 85 +++++++++++++++++++-------------- utils/request.ts | 122 +++++++++++++++++++++++++++++------------------ utils/types.ts | 6 +++ 4 files changed, 138 insertions(+), 95 deletions(-) diff --git a/database.py b/database.py index a6282a6..37b2422 100644 --- a/database.py +++ b/database.py @@ -363,15 +363,13 @@ def add_user(self, user_id: str, user_name: str): #TODO: remove this method sinc @database_lock() def change_user_name(self, user_id: str, user_name: str): self.user_db.execute("UPDATE users SET user_name = ? WHERE user_id = ?", (user_name, user_id)) - self.mercury_db.commit() + self.user_db.commit() @database_lock() - def get_user_name(self, user_id: str) -> str: - res = self.user_db.execute("SELECT user_name FROM users WHERE user_id = ?", (user_id,)) - user_name = res.fetchone() - if user_name is None: - return None - return user_name[0] + def get_user_by_id(self, user_id: str): + res = self.user_db.execute("SELECT * FROM users WHERE user_id = ?", (user_id,)) + user = res.fetchone() + return user def get_user_name_without_lock(self, user_id: str) -> str: res = self.user_db.execute("SELECT user_name FROM users WHERE user_id = ?", (user_id,)) @@ -560,11 +558,11 @@ def dump_annotation( @database_lock() def auth_user(self, email: str, password: str): - res = self.user_db.execute("SELECT hashed_password FROM users WHERE email = ?", (email,)) - hashed_password = res.fetchone() - if hashed_password is None: + res = self.user_db.execute("SELECT * FROM users WHERE email = ?", (email,)) + user = res.fetchone() + if user is None: return False - return self.ph.verify(hashed_password[0], password) + return self.ph.verify(user[3], password), user[0] if __name__ == "__main__": import argparse diff --git a/server.py b/server.py index 1e8f8ba..c46b498 100644 --- a/server.py +++ b/server.py @@ -71,6 +71,12 @@ class Token(BaseModel): token_type: str +class User(BaseModel): + id: str + name: str + email: str + + SECRET_KEY = "" ALGORITHM = "HS256" ACCESS_TOKEN_EXPIRE_MINUTES = 10080 @@ -78,9 +84,12 @@ class Token(BaseModel): oauth2_scheme = OAuth2PasswordBearer(tokenUrl="login") -def create_access_token(email: str): - to_encode = {"email": email} - expire = datetime.now(timezone.utc) + timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES) +def create_access_token(data: dict, expires_delta: timedelta | None = None): + to_encode = data.copy() + if expires_delta: + expire = datetime.now(timezone.utc) + expires_delta + else: + expire = datetime.now(timezone.utc) + timedelta(minutes=15) to_encode.update({"exp": expire}) encoded_jwt = jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM) return encoded_jwt @@ -88,13 +97,17 @@ def create_access_token(email: str): @app.post("/login") async def login(form_data: Annotated[OAuth2PasswordRequestForm, Depends()]) -> Token: - if not database.auth_user(form_data.username, form_data.password): + auth_success, user_id = database.auth_user(form_data.username, + form_data.password) + if not auth_success: # username here is actually email, since OAuth2 requires key be username raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Incorrect username or password", headers={"WWW-Authenticate": "Bearer"}) - access_token = create_access_token(form_data.username) + access_token_expires = timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES) + access_token = create_access_token({"user_id": user_id}, access_token_expires) return Token(access_token=access_token, token_type="bearer") + @app.get("/candidate_labels") async def get_labels() -> list: # get all candidate labels for human annotators to choose from with open("labels.yaml") as f: @@ -110,30 +123,35 @@ async def create_new_user(): return {"key": user_id, "name": user_name} -@app.post("/user/name") -async def update_user_name(name: Name, user_key: Annotated[str, Header()]): - if user_key.startswith('"') and user_key.endswith('"'): - user_key = user_key[1:-1] - database.change_user_name(user_key, name.name) - return {"message": "success"} +@app.get("/user/me") +async def get_user(token: Annotated[str, Depends(oauth2_scheme)]) -> User: + credentials_exception = HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Could not validate credentials", + headers={"WWW-Authenticate": "Bearer"}, + ) + try: + payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM], verify=True) + user_id: str = payload.get("user_id") + if user_id is None: + raise credentials_exception + except InvalidTokenError: + raise credentials_exception + user = database.get_user_by_id(user_id) + if user is None: + raise credentials_exception + return User(id=user[0], name=user[1], email=user[2]) -@app.get("/user/me") -async def get_user_name(user_key: Annotated[str, Header()]): - if user_key.startswith('"') and user_key.endswith('"'): - user_key = user_key[1:-1] - username = database.get_user_name(user_key) - if username is None: - return {"error": "User not found"} - else: - return {"name": username} +@app.post("/user/name") +async def update_user_name(name: Name, user: Annotated[User, Depends(get_user)]): + database.change_user_name(user.id, name.name) + return {"message": "success"} @app.get("/user/export") # please update the route name to be more meaningful, e.g., /user/export_user_data -async def export_user_data(user_key: Annotated[str, Header()]): - if user_key.startswith('"') and user_key.endswith('"'): - user_key = user_key[1:-1] - return database.dump_annotator_labels(user_key) +async def export_user_data(user: Annotated[User, Depends(get_user)]): + return database.dump_annotator_labels(user.id) @app.get("/task") @@ -150,17 +168,12 @@ async def get_task(task_index: int = 0): @app.get("/task/{task_index}/history") -async def get_task_history(task_index: int, user_key: Annotated[str, Header()]): - if user_key.startswith('"') and user_key.endswith('"'): - user_key = user_key[1:-1] - return database.export_task_history(task_index, user_key) +async def get_task_history(task_index: int, user: Annotated[User, Depends(get_user)]): + return database.export_task_history(task_index, user.id) @app.post("/task/{task_index}/label") -async def post_task(task_index: int, label: Label, user_key: Annotated[str, Header()]): - if user_key.startswith('"') and user_key.endswith('"'): - user_key = user_key[1:-1] - +async def post_task(task_index: int, label: Label, user: Annotated[User, Depends(get_user)]): # label_data = LabelData( # record_id="not assigned", # sample_id=tasks[task_index]["_id"], @@ -180,7 +193,7 @@ async def post_task(task_index: int, label: Label, user_key: Annotated[str, Head if label.source_start != -1: annot_spans["source"] = (label.source_start, label.source_end) - annotator = user_key + annotator = user.id label_string = json.dumps(label.consistent) @@ -309,10 +322,8 @@ async def post_selections(task_index: int, selection: Selection): @app.delete("/record/{record_id}") -async def delete_annotation(record_id: str, user_key: Annotated[str, Header()]): - if user_key.startswith('"') and user_key.endswith('"'): - user_key = user_key[1:-1] - database.delete_annotation(record_id, user_key) +async def delete_annotation(record_id: str, user: Annotated[User, Depends(get_user)]): + database.delete_annotation(record_id, user.id) return {"message": f"delete anntation {record_id} success"} diff --git a/utils/request.ts b/utils/request.ts index 3e8cd72..9a60e07 100644 --- a/utils/request.ts +++ b/utils/request.ts @@ -1,55 +1,82 @@ -import type { AllTasksLength, LabelData, LabelRequest, Normal, SectionResponse, SelectionRequest, Task } from "./types" +import type { + AllTasksLength, + LabelData, + LabelRequest, + Normal, + SectionResponse, + SelectionRequest, + Task, + User, +} from "./types" const backend = process.env.NEXT_PUBLIC_BACKEND || "" -const getKey = async (): Promise => { - const hasUserMe = await checkUserMe() - - const key = localStorage.getItem("key") - if (key === "" || key === null) { - const response = await fetch(`${backend}/user/new`) - const data = await response.json() - localStorage.setItem("key", data.key) - if (hasUserMe) { - localStorage.setItem("name", data.name) - } - return data.key +// const getKey = async (): Promise => { +// const hasUserMe = await checkUserMe() +// +// const key = localStorage.getItem("key") +// if (key === "" || key === null) { +// const response = await fetch(`${backend}/user/new`) +// const data = await response.json() +// localStorage.setItem("key", data.key) +// if (hasUserMe) { +// localStorage.setItem("name", data.name) +// } +// return data.key +// } +// +// if (hasUserMe) { +// const nameResponse = await fetch(`${backend}/user/me`, { +// headers: { +// "User-Key": key, +// }, +// }) +// +// const data = await nameResponse.json() +// if ("error" in data) { +// localStorage.removeItem("key") +// localStorage.removeItem("name") +// return getKey() +// } +// localStorage.setItem("name", data.name) +// return Promise.resolve(key) +// } +// } +const getAccessToken = (): string => { + const accessToken = localStorage.getItem("access_token") + if (accessToken === "" || accessToken === null) { + console.log("Please login") } + return accessToken +} - if (hasUserMe) { - const nameResponse = await fetch(`${backend}/user/me`, { - headers: { - "User-Key": key, - }, - }) - - const data = await nameResponse.json() - if ("error" in data) { - localStorage.removeItem("key") - localStorage.removeItem("name") - return getKey() - } - localStorage.setItem("name", data.name) - return Promise.resolve(key) - } +const getUserMe = async (): Promise => { + const access_token = getAccessToken() + const response = await fetch(`${backend}/user/me`, { + headers: { + "Authorization": `Bearer ${access_token}`, + }, + }) + return response.json() } const checkUserMe = async (): Promise => { + const access_token = getAccessToken() const response = await fetch(`${backend}/user/me`, { headers: { - "User-Key": "OK_CHECK", + "Authorization": `Bearer ${access_token}`, }, }) return response.ok } const changeName = async (name: string): Promise => { - const key = await getKey() + const access_token = getAccessToken() const response = await fetch(`${backend}/user/name`, { method: "POST", headers: { "Content-Type": "application/json", - "User-Key": key, + "Authorization": `Bearer ${access_token}`, }, body: JSON.stringify({ name }), }) @@ -89,12 +116,12 @@ const selectText = async (taskIndex: number, req: SelectionRequest): Promise => { - const key = await getKey() + const access_token = getAccessToken() const response = await fetch(`${backend}/task/${taskIndex}/label`, { method: "POST", headers: { "Content-Type": "application/json", - "User-Key": key, + "Authorization": `Bearer ${access_token}`, }, body: JSON.stringify(req), }) @@ -103,10 +130,10 @@ const labelText = async (taskIndex: number, req: LabelRequest): Promise } const exportLabel = async (): Promise => { - const key = await getKey() + const access_token = getAccessToken() const response = await fetch(`${backend}/user/export`, { headers: { - "User-Key": key, + "Authorization": `Bearer ${access_token}`, }, }) const data = await response.json() @@ -114,10 +141,10 @@ const exportLabel = async (): Promise => { } const getTaskHistory = async (taskIndex: number): Promise => { - const key = await getKey() + const access_token = getAccessToken() const response = await fetch(`${backend}/task/${taskIndex}/history`, { headers: { - "User-Key": key, + "Authorization": `Bearer ${access_token}`, }, }) const data = await response.json() @@ -125,11 +152,11 @@ const getTaskHistory = async (taskIndex: number): Promise => { } const deleteRecord = async (recordId: string): Promise => { - const key = await getKey() + const access_token = getAccessToken() const response = await fetch(`${backend}/record/${recordId}`, { method: "DELETE", headers: { - "User-Key": key, + "Authorization": `Bearer ${access_token}`, }, }) const data = await response.json() @@ -137,13 +164,13 @@ const deleteRecord = async (recordId: string): Promise => { } const login = async (email: string, password: string): Promise => { -const response = await fetch(`${backend}/login`, { - method: "POST", - headers: { - "Content-Type": "application/x-www-form-urlencoded", - }, - body: new URLSearchParams({ "username": email, "password": password }), -}) + const response = await fetch(`${backend}/login`, { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded", + }, + body: new URLSearchParams({ "username": email, "password": password }), + }) const data = await response.json() if ("access_token" in data) { localStorage.setItem("access_token", data.access_token) @@ -162,5 +189,6 @@ export { getAllLabels, changeName, checkUserMe, + getUserMe, login, } diff --git a/utils/types.ts b/utils/types.ts index bf36aa8..b323726 100644 --- a/utils/types.ts +++ b/utils/types.ts @@ -60,3 +60,9 @@ export type LabelData = { user_id: string note: string } + +export type User = { + id: string + name: string + email: string +} From b9dcbf4e67ecd6b30347fa68a327f21f50811434 Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Sun, 3 Nov 2024 13:44:12 +0800 Subject: [PATCH 05/43] feat(frontend)!: simple login --- app/login/page.tsx | 5 +- app/page.tsx | 869 +++++++++++++++++++++++---------------------- server.py | 4 + utils/request.ts | 15 +- 4 files changed, 469 insertions(+), 424 deletions(-) diff --git a/app/login/page.tsx b/app/login/page.tsx index a661c25..a8102d4 100644 --- a/app/login/page.tsx +++ b/app/login/page.tsx @@ -13,7 +13,10 @@ const useStackClassName = makeResetStyles({ export default function Login() { async function formAction(formData) { if (formData.get("email") && formData.get("password")) { - await login(formData.get("email"), formData.get("password")) + const success = await login(formData.get("email"), formData.get("password")) + if (success) { + window.location.href = "/" + } } } diff --git a/app/page.tsx b/app/page.tsx index 9a06ba3..3290340 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -13,7 +13,7 @@ import { ProgressBar, Table, TableBody, TableCell, TableHeader, TableHeaderCell, TableRow, Text, - Title1 + Title1, Toast, ToastBody, Toaster, ToastFooter, ToastTitle, useId, useToastController, } from "@fluentui/react-components" import { useAtom } from "jotai" import { atomWithStorage } from "jotai/utils" @@ -31,8 +31,7 @@ import { selectText, deleteRecord, getAllLabels, - changeName, - checkUserMe + changeName, getUserMe, checkUserMe, } from "../utils/request" import { type LabelData, type SectionResponse, type Task, userSectionResponse } from "../utils/types" import { @@ -44,11 +43,11 @@ import { EyeRegular, HandRightRegular, IosChevronRightRegular, - ShareRegular -} from "@fluentui/react-icons"; + ShareRegular, +} from "@fluentui/react-icons" import { Allotment } from "allotment" import "allotment/dist/style.css" -import ColumnResize from "react-table-column-resizer"; +import ColumnResize from "react-table-column-resizer" import "./page.css" const labelIndexAtom = atomWithStorage("labelIndex", 0) @@ -88,24 +87,24 @@ const getColor = (score: number) => { // Function to determine if a color is light or dark const isLightColor = (color: string) => { // Remove the hash if present - let newcolor = color.replace('#', ''); + let newcolor = color.replace("#", "") // Convert 3-digit hex to 6-digit hex if (newcolor.length === 3) { - newcolor = newcolor.split('').map(char => char + char).join(''); + newcolor = newcolor.split("").map(char => char + char).join("") } // Convert hex to RGB - const r = Number.parseInt(newcolor.substring(0, 2), 16); - const g = Number.parseInt(newcolor.substring(2, 4), 16); - const b = Number.parseInt(newcolor.substring(4, 6), 16); + const r = Number.parseInt(newcolor.substring(0, 2), 16) + const g = Number.parseInt(newcolor.substring(2, 4), 16) + const b = Number.parseInt(newcolor.substring(4, 6), 16) // Calculate luminance - const luminance = 0.2126 * r + 0.7152 * g + 0.0722 * b; + const luminance = 0.2126 * r + 0.7152 * g + 0.0722 * b // Return true if luminance is greater than 128 (light color) - return luminance > 200; -}; + return luminance > 200 +} const exportJSON = () => { exportLabel().then(data => { @@ -135,54 +134,82 @@ export default function Index() { const [labels, setLabels] = useState<(string | object)[]>([]) const [userName, setUserName] = useState("No Name") const [tempUserName, setTempUserName] = useState(userName) - const [hideName, setHideName] = useState(true) + + const toasterId = useId("toaster") + const { dispatchToast } = useToastController(toasterId) const historyColumns = [ { columnKey: "summary", label: "Summary" }, { columnKey: "source", label: "Source" }, { columnKey: "consistent", label: "Consistent" }, - { columnKey: "actions", label: "Actions" } + { columnKey: "actions", label: "Actions" }, ] + useEffect(() => { + const access_token = localStorage.getItem("access_token") + if (access_token == "" || access_token == null) { + dispatchToast( + + Not logged in + , + { intent: "error" }, + ) + window.location.href = "/login" + return + } + checkUserMe(access_token).then(valid => { + if (!valid) { + localStorage.removeItem("access_token") + dispatchToast( + + Session expired + , + { intent: "error" }, + ) + } + return + }) + }, []) + useEffect(() => { if (getLock.current) return setUserName(localStorage.getItem("name") || "No Name") Promise.all([ - getAllTasksLength(), + getAllTasksLength(), getAllLabels(), - checkUserMe(), + getUserMe(), ]) - .then(([tasks, labels, result]) => { - setMaxIndex(tasks.all) - setLabels(labels) - setHideName(!result) - getLock.current = true - }) - .then(() => { - // get query of url - const url = new URL(window.location.href) - const index = url.searchParams.get("sample") - if (index !== null) { - washHand() - const indexNumber = Number.parseInt(index) - if (!Number.isNaN(indexNumber) && indexNumber >= 0 && indexNumber <= maxIndex) { - setLabelIndex(indexNumber) + .then(([tasks, labels, user]) => { + setMaxIndex(tasks.all) + setLabels(labels) + setUserName(user.name) + getLock.current = true + }) + .then(() => { + // get query of url + const url = new URL(window.location.href) + const index = url.searchParams.get("sample") + if (index !== null) { + washHand() + const indexNumber = Number.parseInt(index) + if (!Number.isNaN(indexNumber) && indexNumber >= 0 && indexNumber <= maxIndex) { + setLabelIndex(indexNumber) + } } - } - }) + }) }, []) useEffect(() => { getSingleTask(labelIndex) - .then(task => { - if ("doc" in task) { - setCurrentTask(task) - } - }) - .catch(error => { - setCurrentTask(null) - console.error(error) - }) + .then(task => { + if ("doc" in task) { + setCurrentTask(task) + } + }) + .catch(error => { + setCurrentTask(null) + console.error(error) + }) }, [labelIndex]) const updateHistory = () => { @@ -202,7 +229,7 @@ export default function Index() { useEffect(() => { if (viewingRecord === null || currentTask === null) { washHand() - return + return } setFirstRange([viewingRecord.source_start, viewingRecord.source_end]) setRangeId("doc") @@ -213,28 +240,29 @@ export default function Index() { const func = (event) => { const selection = window.getSelection() const target = event.target as HTMLElement - + const mercuryElements = document.querySelectorAll("[data-mercury-disable-selection]") - + // console.log(mercuryElements) - + for (const element of mercuryElements) { if (element.contains(target)) { return } } - + if (target.id.startsWith("label-")) { return } - + if ( - !selection.containsNode(document.getElementById("summary"), true) && - !selection.containsNode(document.getElementById("doc"), true) + !selection.containsNode(document.getElementById("summary"), true) && + !selection.containsNode(document.getElementById("doc"), true) ) { if (userSelection !== null) { setUserSelection(null) - } else { + } + else { washHand() } return @@ -246,15 +274,18 @@ export default function Index() { if (span.parentElement?.id === "summary" || span.parentElement?.id === "doc") { return } - } else if (target.tagName === "P") { + } + else if (target.tagName === "P") { const p = target as HTMLParagraphElement if (p.id === "summary" || p.id === "doc") { return } - } else { + } + else { if (userSelection !== null) { setUserSelection(null) - } else { + } + else { washHand() } return @@ -280,21 +311,21 @@ export default function Index() { end: firstRange[1], from_summary: rangeId === "summary", }) - .then(response => { - setWaiting(null) - if ("error" in response) { - console.error(response.error) - return - } - if (firstRange === null || rangeId === null) { - setServerSelection(null) - return - } - setServerSelection(response as SectionResponse) - }) - .catch(error => { - console.error(error) - }) + .then(response => { + setWaiting(null) + if ("error" in response) { + console.error(response.error) + return + } + if (firstRange === null || rangeId === null) { + setServerSelection(null) + return + } + setServerSelection(response as SectionResponse) + }) + .catch(error => { + console.error(error) + }) }, 100)() }, [firstRange, rangeId, labelIndex]) @@ -310,108 +341,109 @@ export default function Index() { const JustSliceText = (props: { text: string; startAndEndOffset: [number, number] }) => { const fakeResponse = userSectionResponse( - props.startAndEndOffset[0], - props.startAndEndOffset[1], - rangeId === "summary", + props.startAndEndOffset[0], + props.startAndEndOffset[1], + rangeId === "summary", ) const sliceArray = updateSliceArray(props.text, [fakeResponse]) return sliceArray.map(slice => { return slice[3] === 2 ? ( - { - if (firstRange === null || rangeId === null) { - return Promise.resolve() - } - await labelText(labelIndex, { - source_start: rangeId === "summary" ? -1 : firstRange[0], - source_end: rangeId === "summary" ? -1 + 1 : firstRange[1], - summary_start: rangeId === "summary" ? firstRange[0] : -1, - summary_end: rangeId === "summary" ? firstRange[1] : -1, - consistent: label, - note: note, - }) - updateHistory() - }} - message="Check all types that apply below." - /> - ) : ( - - {props.text.slice(slice[0], slice[1] + 1)} - - ) - }) - } - - const SliceText = (props: { text: string; slices: SectionResponse; user: [number, number] | null }) => { - const newSlices = - props.user === null - ? props.slices - : [userSectionResponse(props.user[0], props.user[1], rangeId === "summary")] - const sliceArray = updateSliceArray(props.text, newSlices) - const allScore = [] - for (const slice of newSlices) { - allScore.push(slice.score) - } - const normalColor = normalizationColor(allScore) - return ( - <> - {sliceArray.map(slice => { - const isBackendSlice = slice[2] - const score = slice[3] - const bg_color = isBackendSlice ? score === 2 ? "#85e834" : getColor(normalColor[slice[4]]) : "#ffffff" - const textColor = isLightColor(bg_color) ? 'black' : 'white' - // const textColor= 'red' - return isBackendSlice && viewingRecord == null ? ( - { if (firstRange === null || rangeId === null) { return Promise.resolve() } await labelText(labelIndex, { - source_start: rangeId === "summary" ? slice[0] : firstRange[0], - source_end: rangeId === "summary" ? slice[1] + 1 : firstRange[1], - summary_start: rangeId === "summary" ? firstRange[0] : slice[0], - summary_end: rangeId === "summary" ? firstRange[1] : slice[1] + 1, + source_start: rangeId === "summary" ? -1 : firstRange[0], + source_end: rangeId === "summary" ? -1 + 1 : firstRange[1], + summary_start: rangeId === "summary" ? firstRange[0] : -1, + summary_end: rangeId === "summary" ? firstRange[1] : -1, consistent: label, note: note, }) updateHistory() }} - message="Select the type(s) of hallucinatin below." - /> - ) : ( - + ) : ( + - {props.text.slice(slice[0], slice[1] + 1)} - - ) - })} - + > + {props.text.slice(slice[0], slice[1] + 1)} + + ) + }) + } + + const SliceText = (props: { text: string; slices: SectionResponse; user: [number, number] | null }) => { + const newSlices = + props.user === null + ? props.slices + : [userSectionResponse(props.user[0], props.user[1], rangeId === "summary")] + const sliceArray = updateSliceArray(props.text, newSlices) + const allScore = [] + for (const slice of newSlices) { + allScore.push(slice.score) + } + const normalColor = normalizationColor(allScore) + return ( + <> + + {sliceArray.map(slice => { + const isBackendSlice = slice[2] + const score = slice[3] + const bg_color = isBackendSlice ? score === 2 ? "#85e834" : getColor(normalColor[slice[4]]) : "#ffffff" + const textColor = isLightColor(bg_color) ? "black" : "white" + // const textColor= 'red' + return isBackendSlice && viewingRecord == null ? ( + { + if (firstRange === null || rangeId === null) { + return Promise.resolve() + } + await labelText(labelIndex, { + source_start: rangeId === "summary" ? slice[0] : firstRange[0], + source_end: rangeId === "summary" ? slice[1] + 1 : firstRange[1], + summary_start: rangeId === "summary" ? firstRange[0] : slice[0], + summary_end: rangeId === "summary" ? firstRange[1] : slice[1] + 1, + consistent: label, + note: note, + }) + updateHistory() + }} + message="Select the type(s) of hallucinatin below." + /> + ) : ( + + {props.text.slice(slice[0], slice[1] + 1)} + + ) + })} + ) } @@ -424,10 +456,10 @@ export default function Index() { switch (stage) { case Stage.None: { if ( - range.intersectsNode(element) && - range.startContainer === range.endContainer && - range.startContainer === element.firstChild && - range.startOffset !== range.endOffset + range.intersectsNode(element) && + range.startContainer === range.endContainer && + range.startContainer === element.firstChild && + range.startOffset !== range.endOffset ) { setFirstRange([range.startOffset, range.endOffset]) setUserSelection(null) @@ -445,7 +477,8 @@ export default function Index() { if (element.id === rangeId || element.parentElement?.id === rangeId) { setFirstRange(getRangeTextHandleableRange(range)) setUserSelection(null) - } else { + } + else { setUserSelection(getRangeTextHandleableRange(range)) } break @@ -454,68 +487,67 @@ export default function Index() { } return ( - <> - Mercury Label -
-
-
- {JSON.stringify(firstRange) === "[-1,-1]" || viewingRecord != null ? ( - + ) : ( + + )} + - ) : ( - - )} - - - {!hideName && ( - +
- + > Change Name { - setTempUserName(event.target.value) - }} + type="text" + value={tempUserName} + onChange={event => { + setTempUserName(event.target.value) + }} /> @@ -523,13 +555,12 @@ export default function Index() {
- )} - - - {/* + + + {/* */} - {/* */} -
-
-
- - - - - -
-
- {currentTask === null ? ( -

Loading...

- ) : ( + +
- - -
} + iconPosition="before" + onClick={() => { + washHand() + setLabelIndex(labelIndex - 1) + }} + > + Previous + + + + + +
+
+ {currentTask === null ? ( +

Loading...

+ ) : ( +
- - - Source - - } - /> - { - checkSelection(event.target as HTMLSpanElement) - }} - > - {serverSelection !== null && serverSelection.length > 0 && rangeId === "summary" ? ( - - ) : rangeId === "doc" ? ( - - ) : ( - currentTask.doc - )} - - -
- - - -
- + + +
- - Summary - - } - /> - checkSelection(event.target as HTMLSpanElement)} + - {serverSelection !== null && rangeId === "doc" ? ( - - ) : rangeId === "summary" ? ( - - ) : ( - currentTask.sum - )} - - -
-
- - - Existing annotations - - - } - /> - {history === null ? ( -

Loading...

- ) : ( - - - - Source - {/* @ts-ignore */} - - Summary - {/* @ts-ignore */} - - Label(s) - {/* @ts-ignore */} - - Note - {/* @ts-ignore */} - - {/* TODO: Display the resizer. Now they are invisible. */} - Actions - - - - {history - .sort((a, b) => { - let c = a.source_start - b.source_start - if (c === 0) c = a.summary_start - b.summary_start - return c - }) - .map((record, index) => ( - - {currentTask.doc.slice(record.source_start, record.source_end)} - - {currentTask.sum.slice(record.summary_start, record.summary_end)} - - {record.consistent.join(", ")} - - {record.note} - - - {viewingRecord != null && viewingRecord.record_id === record.record_id ? ( - - ) : ( - - )} - - - - - ))} - -
- )} - -
+ + Source + + } + /> + { + checkSelection(event.target as HTMLSpanElement) + }} + > + {serverSelection !== null && serverSelection.length > 0 && rangeId === "summary" ? ( + + ) : rangeId === "doc" ? ( + + ) : ( + currentTask.doc + )} + + +
+ + + +
+ + + Summary + + } + /> + checkSelection(event.target as HTMLSpanElement)} + > + {serverSelection !== null && rangeId === "doc" ? ( + + ) : rangeId === "summary" ? ( + + ) : ( + currentTask.sum + )} + + +
+
+ + + Existing annotations + + + } + /> + {history === null ? ( +

Loading...

+ ) : ( + + + + Source + {/* @ts-ignore */} + + Summary + {/* @ts-ignore */} + + Label(s) + {/* @ts-ignore */} + + Note + {/* @ts-ignore */} + + {/* TODO: Display the resizer. Now they are invisible. */} + Actions + + + + {history + .sort((a, b) => { + let c = a.source_start - b.source_start + if (c === 0) c = a.summary_start - b.summary_start + return c + }) + .map((record, index) => ( + + {currentTask.doc.slice(record.source_start, record.source_end)} + + {currentTask.sum.slice(record.summary_start, record.summary_end)} + + {record.consistent.join(", ")} + + {record.note} + + + {viewingRecord != null && viewingRecord.record_id === record.record_id ? ( + + ) : ( + + )} + + + + + ))} + +
+ )} + +
+ + - - -
- )} - + + )} + ) } diff --git a/server.py b/server.py index c46b498..76b4c78 100644 --- a/server.py +++ b/server.py @@ -341,6 +341,10 @@ async def history(): async def viewer(): return FileResponse("dist/viewer.html") +@app.get("/login") +async def login(): + return FileResponse("dist/login.html") + if __name__ == "__main__": app.mount("/", StaticFiles(directory="dist", html=True), name="dist") diff --git a/utils/request.ts b/utils/request.ts index 9a60e07..e644cfe 100644 --- a/utils/request.ts +++ b/utils/request.ts @@ -60,8 +60,7 @@ const getUserMe = async (): Promise => { return response.json() } -const checkUserMe = async (): Promise => { - const access_token = getAccessToken() +const checkUserMe = async (access_token: string): Promise => { const response = await fetch(`${backend}/user/me`, { headers: { "Authorization": `Bearer ${access_token}`, @@ -148,7 +147,11 @@ const getTaskHistory = async (taskIndex: number): Promise => { }, }) const data = await response.json() - return data as LabelData[] + if (Array.isArray(data)) { + return data as LabelData[] + } else { + return [] + } } const deleteRecord = async (recordId: string): Promise => { @@ -163,7 +166,7 @@ const deleteRecord = async (recordId: string): Promise => { return data as Normal } -const login = async (email: string, password: string): Promise => { +const login = async (email: string, password: string): Promise => { const response = await fetch(`${backend}/login`, { method: "POST", headers: { @@ -174,8 +177,10 @@ const login = async (email: string, password: string): Promise => { const data = await response.json() if ("access_token" in data) { localStorage.setItem("access_token", data.access_token) + return true + } else { + return false } - return data as Normal } export { From 0564ea652a06b7e1c9d595da7ea3f342d0325169 Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Tue, 5 Nov 2024 21:15:38 +0800 Subject: [PATCH 06/43] docs(README): Add instructions for migration, update technical details --- README.md | 252 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 154 insertions(+), 98 deletions(-) diff --git a/README.md b/README.md index 61f0ced..7330df5 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,13 @@ Mercury is a semantic-assisted, cross-text text labeling tool. -1. semantic-assisted: when you select a text span, semantically related text segments will be highlighted -- so you don't have to eyebal through lengthy texts. +1. semantic-assisted: when you select a text span, semantically related text segments will be highlighted -- so you + don't have to eyebal through lengthy texts. 2. cross-text: you are labeling text spans from two different texts. -Therefore, Mercury is very efficient for the labeling of NLP tasks that involve comparing texts between two documents which are also lengthy, such as hallucination detection or factual consistency/faithfulness in RAG systems. Semantic assistance not only saves time and reduces fatigues but also avoids mistakes. +Therefore, Mercury is very efficient for the labeling of NLP tasks that involve comparing texts between two documents +which are also lengthy, such as hallucination detection or factual consistency/faithfulness in RAG systems. Semantic +assistance not only saves time and reduces fatigues but also avoids mistakes. Currently, Mercury only supports labeling inconsistencies between the source and summary for summarization in RAG. @@ -16,83 +19,102 @@ Currently, Mercury only supports labeling inconsistencies between the source and > [!NOTE] > You need Python and Node.js. -Mercury uses [`sqlite-vec`](https://github.com/asg017/sqlite-vec) to store and search embeddings. +Mercury uses [`sqlite-vec`](https://github.com/asg017/sqlite-vec) to store and search embeddings. 1. `pip3 install -r requirements.txt && python3 -m spacy download en_core_web_sm` -2. If you don't have `pnpm` installed: `npm install -g pnpm`, you may need sudo. If you don't have `npm`, try `sudo apt install npm`. - -3. To use `sqlite-vec` via Python's built-in `sqlite3` module, you must have SQLite>3.41 (otherwise `LIMIT` or `k=?` will not work properly with `rowid IN (?)` for vector search) installed and set Python's built-in `sqlite3` module to use it. Python's built-in `sqlite3` module uses its own binary library that is independent of the OS's SQLite. So upgrading the OS's SQLite will not affect Python's `sqlite3` module. You need to follow the steps below: - * Download and compile SQLite>3.41.0 from source - ```bash - wget https://www.sqlite.org/2024/sqlite-autoconf-3460100.tar.gz - tar -xvf sqlite-autoconf-3460100.tar.gz - cd sqlite-autoconf-3460100 - ./configure - make - ``` +2. If you don't have `pnpm` installed: `npm install -g pnpm`, you may need sudo. If you don't have `npm`, try + `sudo apt install npm`. + +3. To use `sqlite-vec` via Python's built-in `sqlite3` module, you must have SQLite>3.41 (otherwise `LIMIT` or `k=?` + will not work properly with `rowid IN (?)` for vector search) installed and set Python's built-in `sqlite3` module to + use it. Python's built-in `sqlite3` module uses its own binary library that is independent of the OS's SQLite. So + upgrading the OS's SQLite will not affect Python's `sqlite3` module. You need to follow the steps below: + * Download and compile SQLite>3.41.0 from source + ```bash + wget https://www.sqlite.org/2024/sqlite-autoconf-3460100.tar.gz + tar -xvf sqlite-autoconf-3460100.tar.gz + cd sqlite-autoconf-3460100 + ./configure + make + ``` * Set Python's built-in `sqlite3` module to use the compiled SQLite. - Suppose you are currently at path `$SQLITE_Compile`. Then set this environment variable (feel free to replace `$SQLITE_Compile` with the actual absolute/relative path): + Suppose you are currently at path `$SQLITE_Compile`. Then set this environment variable (feel free to replace + `$SQLITE_Compile` with the actual absolute/relative path): ```bash export LD_PRELAOD=$SQLITE_Compile/.libs/libsqlite3.so ``` - You may add the above line to `~.bashrc` to make it permanent. + You may add the above line to `~.bashrc` to make it permanent. * Verify that Python's `sqlite3` module is using the correct SQLite, run this Python code: - ```python + ```shell python3 -c "import sqlite3; print(sqlite3.sqlite_version)" ``` If the output is the version of SQLite you just compiled, you are good to go. - * If you are using Mac and run into troubles, please follow SQLite-vec's [instructions](https://alexgarcia.xyz/sqlite-vec/python.html#updated-sqlite). + * If you are using Mac and run into troubles, please follow + SQLite-vec's [instructions](https://alexgarcia.xyz/sqlite-vec/python.html#updated-sqlite). -4. To use `sqlite-vec` directly in `sqlite` prompt, simply [compile `sqlite-vec` from source](https://alexgarcia.xyz/sqlite-vec/compiling.html) and load the compiled `vec0.o`. The usage can be found in the [README](https://github.com/asg017/sqlite-vec?tab=readme-ov-file#sample-usage) of SQLite-vec. +4. To use `sqlite-vec` directly in `sqlite` prompt, simply [compile + `sqlite-vec` from source](https://alexgarcia.xyz/sqlite-vec/compiling.html) and load the compiled `vec0.o`. The usage + can be found in the [README](https://github.com/asg017/sqlite-vec?tab=readme-ov-file#sample-usage) of SQLite-vec. -## Usage +## Usage 1. Ingest data for labeling Run `python3 ingester.py -h` to see the options. - The ingester takes a CSV, JSON, or JSONL file and loads texts from two text columns (configurable via option `ingest_column_1` and `ingest_column_2` which default to `source` and `summary`) of the file. Mercury uses three Vectara corpora to store the sources, the summaries, and the human annotations. You can provide the corpus IDs to overwrite or append data to existing corpora. + The ingester takes a CSV, JSON, or JSONL file and loads texts from two text columns (configurable via option + `ingest_column_1` and `ingest_column_2` which default to `source` and `summary`) of the file. Mercury uses three + Vectara corpora to store the sources, the summaries, and the human annotations. You can provide the corpus IDs to + overwrite or append data to existing corpora. 2. `pnpm install && pnpm build` (You need to recompile the frontend each time the UI code changes.) -4. Manually set the labels for annotators to choose from in the `labels.yaml` file. Mercury supports hierarchical labels. -3. `python3 server.py`. Be sure to set the candidate labels to choose from in the `server.py` file. +3. Manually set the labels for annotators to choose from in the `labels.yaml` file. Mercury supports hierarchical + labels. +4. Migrate existing user data if you have used old version of mercury: `python3 migrate.py`. + This script will generate random email and password for each user. You can change them later with `user_utils.py`. +5. Generate a jwt secret key: `openssl rand -base64 32`. +6. `python3 server.py`. Be sure to set the candidate labels to choose from in the `server.py` file. -The annotations are stored in the `annotations` table in a SQLite database (hardcoded name `mercury.sqlite`). See the section [`annotations` table](#annotations-table-the-human-annotations) for the schema. +The annotations are stored in the `annotations` table in a SQLite database (hardcoded name `mercury.sqlite`). See the +section [`annotations` table](#annotations-table-the-human-annotations) for the schema. The dumped human annotations are stored in a JSON format like this: +class Any: +pass + ```python [ - {# first sample + { # first sample 'sample_id': int, - 'source': str, + 'source': str, 'summary': str, - 'annotations': [ # a list of annotations from many human annotators + 'annotations': [ # a list of annotations from many human annotators { 'annot_id': int, - 'sample_id': int, # relative to the ingestion file + 'sample_id': int, # relative to the ingestion file 'annotator': str, # the annotator unique id - 'annotator_name': str, # the annotator name + 'annotator_name': str, # the annotator name 'label': list[str], 'note': str, - 'summary_span': str, # the text span in the summary + 'summary_span': str, # the text span in the summary 'summary_start': int, 'summary_end': int, - 'source_span': str, # the text span in the source + 'source_span': str, # the text span in the source 'source_start': int, 'source_end': int, } - ], - 'meta_field_1': Any, # whatever meta info about the sample - 'meta_field_2': Any, + ], + 'meta_field_1': Any, # whatever meta info about the sample + 'meta_field_2': Any, ... - }, - {# second sample + }, + { # second sample ... - }, + }, ... ] ``` @@ -102,19 +124,28 @@ You can view exported data in `http://[your_host]/viewer` ## Technical details Terminology: + * A **sample** is a pair of source and summary. * A **document** is either a source or a summary. * A **chunk** is a sentence in a document. -> [!NOTE] SQLite uses 1-indexed for `autoincrement` columns while the rest of the code uses 0-indexed. +> [!NOTE] SQLite uses 1-indexed for `autoincrement` columns while the rest of the code uses 0-indexed. + +### Tables +Mercury uses two database. Seperated user database can be used across projects. +#### Mercury main database: -### Tables +`chunks`, `embeddings`, `annotations`, `config`. -Three tables: `chunks`, `embeddings`, `annotations`, `users` and `leaderboard`. All powered by SQLite. In particular, `embeddings` is powered by `sqlite-vec`. +#### User database: + +`users`. + +All powered by SQLite. In particular, `embeddings` is powered by `sqlite-vec`. #### `chunks` table: chunks and metadata -Each row is a chunk. +Each row is a chunk. A JSONL file like this: @@ -126,76 +157,89 @@ A JSONL file like this: will be ingested into the `chunks` table as below: -| chunk_id | text | text_type | sample _id | char _offset | chunk _offset| -|----------|----------------------------|-----------|------------|--------------|--------------| -| 0 | "The quick brown fox." | source | 0 | 0 | 0 | -| 1 | "Jumps over the lazy dog." | source | 0 | 21 | 1 | -| 2 | "We the people." | source | 1 | 0 | 0 | -| 3 | "Of the U.S.A." | source | 1 | 15 | 1 | -| 4 | "26 letters." | summary | 0 | 0 | 0 | -| 5 | "The U.S. Constitution." | summary | 1 | 0 | 0 | -| 6 | "It is great." | summary | 1 | 23 | 1 | - -Meaning of select columns: -* `char_offset` is the offset of a chunk in its parent document measured by the starting character of the chunk. It allows us to find the chunk in the document. +| chunk_id | text | text_type | sample _id | char _offset | chunk _offset | +|----------|----------------------------|-----------|------------|--------------|---------------| +| 0 | "The quick brown fox." | source | 0 | 0 | 0 | +| 1 | "Jumps over the lazy dog." | source | 0 | 21 | 1 | +| 2 | "We the people." | source | 1 | 0 | 0 | +| 3 | "Of the U.S.A." | source | 1 | 15 | 1 | +| 4 | "26 letters." | summary | 0 | 0 | 0 | +| 5 | "The U.S. Constitution." | summary | 1 | 0 | 0 | +| 6 | "It is great." | summary | 1 | 23 | 1 | + +Meaning of select columns: + +* `char_offset` is the offset of a chunk in its parent document measured by the starting character of the chunk. It + allows us to find the chunk in the document. * `chunk_offset_local` is the index of a chunk in its parent document. It is used to find the chunk in the document. * `text_type` is takes value from the ingestion file. `source` and `summary` for now. -* All columns are 0-indexed. -* The `sample_id` is the index of the sample in the ingestion file. Because the ingestion file could be randomly sampled from a bigger dataset, the `sample_id` is not necessarily global. +* All columns are 0-indexed. +* The `sample_id` is the index of the sample in the ingestion file. Because the ingestion file could be randomly sampled + from a bigger dataset, the `sample_id` is not necessarily global. #### `embeddings` table: the embeddings of chunks -| rowid | embedding | -|----------|-----------| -| 1 | [0.1, 0.2, ..., 0.9] | -| 2 | [0.2, 0.3, ..., 0.8] | +| rowid | embedding | +|-------|----------------------| +| 1 | [0.1, 0.2, ..., 0.9] | +| 2 | [0.2, 0.3, ..., 0.8] | -* `rowid` here and `chunk_id` in the `chunks` table have one-to-one correspondence. `rowid` is 1-indexed due to `sqlite-vec`. We cannot do anything about it. So when aligning the tables `chunks` and `embeddings`, remember to subtract 1 from `rowid` to get `chunk_id`. +* `rowid` here and `chunk_id` in the `chunks` table have one-to-one correspondence. `rowid` is 1-indexed due to + `sqlite-vec`. We cannot do anything about it. So when aligning the tables `chunks` and `embeddings`, remember to + subtract 1 from `rowid` to get `chunk_id`. #### `annotations` table: the human annotations -| annot_id | sample _id | annot_spans | annotator | label | note | -|----------|------------|-----------------------------------------|-----------|------------|------| -| 1 | 1 | {'source': [1, 10], 'summary': [7, 10]} | 2fe9bb69 | ["ambivalent"] | "I am not sure." | +| annot_id | sample _id | annot_spans | annotator | label | note | +|----------|------------|-----------------------------------------|-----------|----------------|--------------------------------| +| 1 | 1 | {'source': [1, 10], 'summary': [7, 10]} | 2fe9bb69 | ["ambivalent"] | "I am not sure." | | 2 | 1 | {'summary': [2, 8]} | a24cb15c | ["extrinsic"] | "No connection to the source." | * `sample_id` are the `id`'s of chunks in the `chunks` table. -* `text_spans` is a JSON text field that stores the text spans selected by the annotator. Each entry is a dictionary where keys must be those in the `text_type` column in the `chunks` table (hardcoded to `source` and `summary` now) and the values are lists of two integers: the start and end indices of the text span in the chunk. For extrinsic hallucinations (no connection to the source at all), only `summary`-key items. The reason we use JSON here is that SQLite does not support array types. +* `text_spans` is a JSON text field that stores the text spans selected by the annotator. Each entry is a dictionary + where keys must be those in the `text_type` column in the `chunks` table (hardcoded to `source` and `summary` now) + and the values are lists of two integers: the start and end indices of the text span in the chunk. For extrinsic + hallucinations (no connection to the source at all), only `summary`-key items. The reason we use JSON here is that + SQLite does not support array types. #### `config` table: the configuration -For example: +For example: -| key | value | -|----------|-------| -| embdding_model | "openai/text-embedding-3-small" | -| embdding_dimension | 4 | +| key | value | +|--------------------|---------------------------------| +| embdding_model | "openai/text-embedding-3-small" | +| embdding_dimension | 4 | #### `sample_meta` table: the sample metadata -| sample_id | json_meta | -|-----------|-----------| -| 0 | {"model":"meta-llama\/Meta-Llama-3.1-70B-Instruct","HHEMv1":0.43335,"HHEM-2.1":0.39717,"HHEM-2.1-English":0.90258,"trueteacher":1,"true_nli":0.0,"gpt-3.5-turbo":1,"gpt-4-turbo":1,"gpt-4o":1, "sample_id":727} | -| 1 | {"model":"openai\/GPT-3.5-Turbo","HHEMv1":0.43003,"HHEM-2.1":0.97216,"HHEM-2.1-English":0.92742,"trueteacher":1,"true_nli":1.0,"gpt-3.5-turbo":1,"gpt-4-turbo":1,"gpt-4o":1, "sample_id": 1018} | +| sample_id | json_meta | +|-----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0 | {"model":"meta-llama\/Meta-Llama-3.1-70B-Instruct","HHEMv1":0.43335,"HHEM-2.1":0.39717,"HHEM-2.1-English":0.90258,"trueteacher":1,"true_nli":0.0,"gpt-3.5-turbo":1,"gpt-4-turbo":1,"gpt-4o":1, "sample_id":727} | +| 1 | {"model":"openai\/GPT-3.5-Turbo","HHEMv1":0.43003,"HHEM-2.1":0.97216,"HHEM-2.1-English":0.92742,"trueteacher":1,"true_nli":1.0,"gpt-3.5-turbo":1,"gpt-4-turbo":1,"gpt-4o":1, "sample_id": 1018} | -0-indexed, the `sample_id` column is the `sample_id` in the `chunks` table. It is local to the ingestion file. The `json_meta` is whatever info other than ingestion columns (source and summary) in the ingestion file. +0-indexed, the `sample_id` column is the `sample_id` in the `chunks` table. It is local to the ingestion file. The +`json_meta` is whatever info other than ingestion columns (source and summary) in the ingestion file. #### `users` table: the annotators -| user_id | user_name | -|----------------------------------|-----------| -| add93a266ab7484abdc623ddc3bf6441 | Alice | -| 68d41e465458473c8ca1959614093da7 | Bob | +| user_id | user_name | email | password | +|----------------------------------|-----------|---------------|-------------| +| add93a266ab7484abdc623ddc3bf6441 | Alice | a@example.com | super_safe | +| 68d41e465458473c8ca1959614093da7 | Bob | b@example.com | my_password | ### How to do vector search -SQLite-vec uses Euclidean distance for vector search. So all embeddings much be normalized to unit length. Fortunately, OpenAI and Sentence-Bert's embeddings are already normalized. +SQLite-vec uses Euclidean distance for vector search. So all embeddings much be normalized to unit length. Fortunately, +OpenAI and Sentence-Bert's embeddings are already normalized. -1. Suppose the user selects a text span in chunk of global chunk ID `x`. Assume that the text span selection cannot cross sentence boundaries. -2. Get `x`'s `doc_id` from the `chunks` table. +1. Suppose the user selects a text span in chunk of global chunk ID `x`. Assume that the text span selection cannot + cross sentence boundaries. +2. Get `x`'s `doc_id` from the `chunks` table. 3. Get `x`'s embedding from the `embeddings` table by `where rowid = {chunk_id}`. Denote it as `x_embedding`. -4. Get the `chunk_id`s of all chunks in the opposite document (source if `x` is in summary, and vice versa) by `where doc_id = {doc_id} and text_type={text_type}`. Denote such chunk IDs as `y1, y2, ..., yn`. -5. Send a query to SQLite like this: +4. Get the `chunk_id`s of all chunks in the opposite document (source if `x` is in summary, and vice versa) by + `where doc_id = {doc_id} and text_type={text_type}`. Denote such chunk IDs as `y1, y2, ..., yn`. +5. Send a query to SQLite like this: ```sql SELECT rowid, @@ -206,27 +250,33 @@ SQLite-vec uses Euclidean distance for vector search. So all embeddings much be ORDER BY distance LIMIT 5 ``` - This will find the 5 most similar chunks to `x` in the opposite document. It limits vector search within the opposite document by `rowid in (y1, y2, ..., yn)`. Note that `rowid`, `embedding`, and `distance` are predefined by `sqlite-vec`. + This will find the 5 most similar chunks to `x` in the opposite document. It limits vector search within the opposite + document by `rowid in (y1, y2, ..., yn)`. Note that `rowid`, `embedding`, and `distance` are predefined by + `sqlite-vec`. -Here is a running example (using the data [above](#chunks-table-chunks-and-metadata)): +Here is a running example (using the data [above](#chunks-table-chunks-and-metadata)): -1. Suppose the data has been ingested. The embedder is `openai/`text-embedding-3-small` and the embedding dimension is 4. -2. Suppose the user selects `sample_id = 1` and `chunk_id = 5`: "The U.S. Constitution." The `text_type` of `chunk_id = 5` is `summary` -- the opposite document is the source. -3. Let's get the chunk IDs of the source document: +1. Suppose the data has been ingested. The embedder is `openai/`text-embedding-3-small` and the embedding dimension is + 4. +2. Suppose the user selects `sample_id = 1` and `chunk_id = 5`: "The U.S. Constitution." The `text_type` of + `chunk_id = 5` is `summary` -- the opposite document is the source. +3. Let's get the chunk IDs of the source document: ```sql SELECT chunk_id FROM chunks WHERE sample_id = 1 and text_type = 'source' ``` - The return is `2, 3`. -4. The embedding of "The U.S. Constitution" can be obtained from the `embeddings` table by `where rowid = 6`. Note that because SQLite uses 1-indexed, so we need to add 1 from `chunk_id` to get `rowid`. + The return is `2, 3`. +4. The embedding of "The U.S. Constitution" can be obtained from the `embeddings` table by `where rowid = 6`. Note that + because SQLite uses 1-indexed, so we need to add 1 from `chunk_id` to get `rowid`. ```sql SELECT embedding FROM embeddings WHERE rowid = 6 ``` - The return is `[0.08553484082221985, 0.21519172191619873, 0.46908700466156006, 0.8522521257400513]`. -5. Now We search for its nearest neighbors in its corresponding source chunks of `rowid` 4 and 5 -- again, obtained by adding 1 from `chunk_id` 2 and 3 obtained in step 3. + The return is `[0.08553484082221985, 0.21519172191619873, 0.46908700466156006, 0.8522521257400513]`. +5. Now We search for its nearest neighbors in its corresponding source chunks of `rowid` 4 and 5 -- again, obtained by + adding 1 from `chunk_id` 2 and 3 obtained in step 3. ```sql SELECT rowid, @@ -236,13 +286,19 @@ Here is a running example (using the data [above](#chunks-table-chunks-and-metad and rowid in (4, 5) ORDER BY distance ``` - The return is `[(4, 0.3506483733654022), (5, 1.1732779741287231)]`. -6. Translate the `rowid` back to `chunk_id` by subtracting 4 and 5 to get 2 and 3. The closest source chunk is "We the people" (`rowid=3` while `chunk_id`=2) which is the most famous three words in the US Constitution. + The return is `[(4, 0.3506483733654022), (5, 1.1732779741287231)]`. +6. Translate the `rowid` back to `chunk_id` by subtracting 4 and 5 to get 2 and 3. The closest source chunk is "We the + people" (`rowid=3` while `chunk_id`=2) which is the most famous three words in the US Constitution. ### Limitations -1. OpenAI's embedding endpoint can only embed up to 8192 tokens in each call. -2. `embdding_dimension` is only useful for OpenAI models. Most other models do not support changing the embedding dimension. + +1. OpenAI's embedding endpoint can only embed up to 8192 tokens in each call. +2. `embdding_dimension` is only useful for OpenAI models. Most other models do not support changing the embedding + dimension. ### Embedding speed and/or embedding dimension -1. `multi-qa-mpnet-base-dot-v1` takes about 0.219 second on a x86 CPU to embed one sentence when batch_size is 1. The embedding dimension is 768. -2. `BAAI/bge-small-en-v1.5` takes also about 0.202 second on a x86 CPU to embed one sentence when batch_size is 1. The embedding dimension is 384. + +1. `multi-qa-mpnet-base-dot-v1` takes about 0.219 second on a x86 CPU to embed one sentence when batch_size is 1. The + embedding dimension is 768. +2. `BAAI/bge-small-en-v1.5` takes also about 0.202 second on a x86 CPU to embed one sentence when batch_size is 1. The + embedding dimension is 384. From 04a192c80a4b8f6d852989198190fbb3e45c4264 Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Thu, 14 Nov 2024 19:09:48 +0800 Subject: [PATCH 07/43] fix(backend): Fix type error --- database.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/database.py b/database.py index 37b2422..8f8e012 100644 --- a/database.py +++ b/database.py @@ -561,8 +561,12 @@ def auth_user(self, email: str, password: str): res = self.user_db.execute("SELECT * FROM users WHERE email = ?", (email,)) user = res.fetchone() if user is None: - return False - return self.ph.verify(user[3], password), user[0] + return False, None + try: + success = self.ph.verify(user[3], password) + except argon2.exceptions.VerifyMismatchError: + success = False + return success, user[0] if __name__ == "__main__": import argparse From 3a195845e452445c9e1440345447dcdba0f0907e Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Thu, 14 Nov 2024 21:56:03 +0800 Subject: [PATCH 08/43] fix(frontend): Add error handling --- app/login/page.tsx | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/app/login/page.tsx b/app/login/page.tsx index a8102d4..1b784c5 100644 --- a/app/login/page.tsx +++ b/app/login/page.tsx @@ -1,6 +1,15 @@ "use client" -import { Button, Field, Input, makeResetStyles, Title1, tokens } from "@fluentui/react-components" +import { + Button, + Field, + Input, + makeResetStyles, + Title1, Toast, Toaster, ToastTitle, + tokens, + useId, + useToastController, +} from "@fluentui/react-components" import { login } from "../../utils/request" const useStackClassName = makeResetStyles({ @@ -11,24 +20,35 @@ const useStackClassName = makeResetStyles({ }) export default function Login() { + const toasterId = useId("toaster") + const { dispatchToast } = useToastController(toasterId) + async function formAction(formData) { if (formData.get("email") && formData.get("password")) { const success = await login(formData.get("email"), formData.get("password")) if (success) { window.location.href = "/" + } else { + dispatchToast( + + User does not exist or mismatched email and password + , + { position: "bottom-start", intent: "error" }, + ) } } } return ( <> + Login
- + - + - - - - - - -
- - - Change Name - - { - setTempUserName(event.target.value) - }} - /> - - -
- - + {/* @@ -760,7 +706,7 @@ export default function Index() { if (c === 0) c = a.summary_start - b.summary_start return c }) - .map((record, index) => ( + .map((record, _) => ( {currentTask.doc.slice(record.source_start, record.source_end)} diff --git a/components/userPopover.tsx b/components/userPopover.tsx new file mode 100644 index 0000000..f1c4c83 --- /dev/null +++ b/components/userPopover.tsx @@ -0,0 +1,59 @@ +"use client" + +import { + Avatar, + Body1, + Button, + Field, + Input, + Popover, PopoverProps, + PopoverSurface, + PopoverTrigger, +} from "@fluentui/react-components" +import { changeName } from "../utils/request" +import { useTrackedUserStore } from "../store/useUserStore" +import { useEffect, useState } from "react" + +export default function UserPopover() { + const userState = useTrackedUserStore() + const [open, setOpen] = useState(false) + const handleOpenChange: PopoverProps["onOpenChange"] = (e, data) => setOpen(data.open || false) + + useEffect(() => { + userState.fetch().then(() => {}) + }, []) + + async function formSetName(formData) { + changeName(formData.get("newName")).then(() => { + userState.setName(formData.get("newName")) + setOpen(false) + }) + } + + return ( + + + + + + +
+ + + + + + +
+ + + ) +} diff --git a/package.json b/package.json index 0ffb9e7..39bc85d 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "@floating-ui/react": "^0.26.12", "@fluentui/react-components": "^9.47.3", "@fluentui/react-icons": "^2.0.250", + "allotment": "^1.20.2", "jotai": "^2.7.2", "js-sha512": "^0.9.0", "lodash": "^4.17.21", @@ -17,7 +18,8 @@ "react": "^18.2.0", "react-dom": "^18.2.0", "react-table-column-resizer": "^1.2.3", - "allotment": "^1.20.2" + "react-tracked": "^2.0.1", + "zustand": "^5.0.1" }, "devDependencies": { "@biomejs/biome": "1.6.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f642247..5d2dc7e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -41,6 +41,12 @@ importers: react-table-column-resizer: specifier: ^1.2.3 version: 1.2.4(react@18.2.0) + react-tracked: + specifier: ^2.0.1 + version: 2.0.1(react@18.2.0)(scheduler@0.23.0) + zustand: + specifier: ^5.0.1 + version: 5.0.1(@types/react@18.2.33)(react@18.2.0) devDependencies: '@biomejs/biome': specifier: 1.6.4 @@ -787,6 +793,9 @@ packages: prop-types@15.8.1: resolution: {integrity: sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==} + proxy-compare@3.0.0: + resolution: {integrity: sha512-y44MCkgtZUCT9tZGuE278fB7PWVf7fRYy0vbRXAts2o5F0EfC4fIQrvQQGBJo1WJbFcVLXzApOscyJuZqHQc1w==} + react-dom@18.2.0: resolution: {integrity: sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==} peerDependencies: @@ -803,6 +812,12 @@ packages: peerDependencies: react: ^18.2.0 + react-tracked@2.0.1: + resolution: {integrity: sha512-qjbmtkO2IcW+rB2cFskRWDTjKs/w9poxvNnduacjQA04LWxOoLy9J8WfIEq1ahifQ/tVJQECrQPBm+UEzKRDtg==} + peerDependencies: + react: '>=18.0.0' + scheduler: '>=0.19.0' + react-transition-group@4.4.5: resolution: {integrity: sha512-pZcd1MCJoiKiBR2NRxeCRg13uCXbydPnmB4EOeRrY7480qNWO8IIgQG6zlDkm6uRMsURXPuKq0GWtiM59a5Q6g==} peerDependencies: @@ -863,6 +878,12 @@ packages: undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + use-context-selector@2.0.0: + resolution: {integrity: sha512-owfuSmUNd3eNp3J9CdDl0kMgfidV+MkDvHPpvthN5ThqM+ibMccNE0k+Iq7TWC6JPFvGZqanqiGCuQx6DyV24g==} + peerDependencies: + react: '>=18.0.0' + scheduler: '>=0.19.0' + use-disposable@1.0.2: resolution: {integrity: sha512-UMaXVlV77dWOu4GqAFNjRzHzowYKUKbJBQfCexvahrYeIz4OkUYUjna4Tjjdf92NH8Nm8J7wEfFRgTIwYjO5jg==} peerDependencies: @@ -877,6 +898,24 @@ packages: react: 16.8.0 - 18 react-dom: 16.8.0 - 18 + zustand@5.0.1: + resolution: {integrity: sha512-pRET7Lao2z+n5R/HduXMio35TncTlSW68WsYBq2Lg1ASspsNGjpwLAsij3RpouyV6+kHMwwwzP0bZPD70/Jx/w==} + engines: {node: '>=12.20.0'} + peerDependencies: + '@types/react': '>=18.0.0' + immer: '>=9.0.6' + react: '>=18.0.0' + use-sync-external-store: '>=1.2.0' + peerDependenciesMeta: + '@types/react': + optional: true + immer: + optional: true + react: + optional: true + use-sync-external-store: + optional: true + snapshots: '@babel/runtime@7.24.1': @@ -2069,6 +2108,8 @@ snapshots: object-assign: 4.1.1 react-is: 16.13.1 + proxy-compare@3.0.0: {} + react-dom@18.2.0(react@18.2.0): dependencies: loose-envify: 1.4.0 @@ -2083,6 +2124,13 @@ snapshots: dependencies: react: 18.2.0 + react-tracked@2.0.1(react@18.2.0)(scheduler@0.23.0): + dependencies: + proxy-compare: 3.0.0 + react: 18.2.0 + scheduler: 0.23.0 + use-context-selector: 2.0.0(react@18.2.0)(scheduler@0.23.0) + react-transition-group@4.4.5(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@babel/runtime': 7.24.1 @@ -2130,6 +2178,11 @@ snapshots: undici-types@5.26.5: {} + use-context-selector@2.0.0(react@18.2.0)(scheduler@0.23.0): + dependencies: + react: 18.2.0 + scheduler: 0.23.0 + use-disposable@1.0.2(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@types/react': 18.2.33 @@ -2142,3 +2195,8 @@ snapshots: '@juggle/resize-observer': 3.4.0 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) + + zustand@5.0.1(@types/react@18.2.33)(react@18.2.0): + optionalDependencies: + '@types/react': 18.2.33 + react: 18.2.0 diff --git a/store/useUserStore.ts b/store/useUserStore.ts new file mode 100644 index 0000000..1d6a4b7 --- /dev/null +++ b/store/useUserStore.ts @@ -0,0 +1,21 @@ +import { create } from "zustand" +import { User } from "../utils/types" +import { getUserMe } from "../utils/request" +import { createTrackedSelector } from "react-tracked" + +interface UserStore { + user: User + fetch: () => Promise + setName: (name: string) => void +} + +export const useUserStore = create()((set) => ({ + user: {} as User, + fetch: async () => { + const user = await getUserMe() + set({ user }) + }, + setName: (name: string) => set((state) => ({ user: { ...state.user, name } })), +})) + +export const useTrackedUserStore = createTrackedSelector(useUserStore) \ No newline at end of file From db168e474758e5593e8dbbc3f85b5f07630aba9f Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Sat, 16 Nov 2024 13:25:41 +0800 Subject: [PATCH 13/43] refactor(frontend): Separate LabelPagination component --- app/page.tsx | 113 ++++++++++----------------------- components/labelPagination.tsx | 54 ++++++++++++++++ store/useIndexStore.tsx | 26 ++++++++ 3 files changed, 114 insertions(+), 79 deletions(-) create mode 100644 components/labelPagination.tsx create mode 100644 store/useIndexStore.tsx diff --git a/app/page.tsx b/app/page.tsx index 3335b9c..9abf6e0 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -5,14 +5,11 @@ import { Button, Card, CardHeader, - Field, - ProgressBar, Table, TableBody, TableCell, + Table, TableBody, TableCell, TableHeader, TableHeaderCell, TableRow, Text, Title1, Toast, Toaster, ToastTitle, useId, useToastController, } from "@fluentui/react-components" -import { useAtom } from "jotai" -import { atomWithStorage } from "jotai/utils" import _ from "lodash" import { useEffect, useLayoutEffect, useRef, useState } from "react" import Tooltip from "../components/tooltip" @@ -21,7 +18,6 @@ import getRangeTextHandleableRange from "../utils/rangeTextNodes" import { exportLabel, getTaskHistory, - getAllTasksLength, getSingleTask, labelText, selectText, @@ -33,12 +29,10 @@ import { type LabelData, type SectionResponse, type Task, userSectionResponse } import { ArrowExportRegular, ArrowSyncRegular, - ChevronLeftRegular, DeleteRegular, EyeOffRegular, EyeRegular, HandRightRegular, - IosChevronRightRegular, ShareRegular, } from "@fluentui/react-icons" import { Allotment } from "allotment" @@ -46,8 +40,8 @@ import "allotment/dist/style.css" import ColumnResize from "react-table-column-resizer" import "./page.css" import UserPopover from "../components/userPopover" - -const labelIndexAtom = atomWithStorage("labelIndex", 0) +import LabelPagination from "../components/labelPagination" +import { useTrackedIndexStore } from "../store/useIndexStore" enum Stage { None = 0, @@ -116,8 +110,7 @@ const exportJSON = () => { } export default function Index() { - const [labelIndex, setLabelIndex] = useAtom(labelIndexAtom) - const [maxIndex, setMaxIndex] = useState(1) + const indexStore = useTrackedIndexStore() const [currentTask, setCurrentTask] = useState(null) const getLock = useRef(false) const [firstRange, setFirstRange] = useState<[number, number] | null>(null) @@ -132,6 +125,7 @@ export default function Index() { const toasterId = useId("toaster") const { dispatchToast } = useToastController(toasterId) + useEffect(() => { const access_token = localStorage.getItem("access_token") if (access_token == "" || access_token == null) { @@ -159,32 +153,30 @@ export default function Index() { }, []) useEffect(() => { - if (getLock.current) return - Promise.all([ - getAllTasksLength(), - getAllLabels(), - ]) - .then(([tasks, labels]) => { - setMaxIndex(tasks.all) - setLabels(labels) - getLock.current = true - }) - .then(() => { - // get query of url - const url = new URL(window.location.href) - const index = url.searchParams.get("sample") - if (index !== null) { - washHand() - const indexNumber = Number.parseInt(index) - if (!Number.isNaN(indexNumber) && indexNumber >= 0 && indexNumber <= maxIndex) { - setLabelIndex(indexNumber) - } - } - }) + indexStore.fetchMaxIndex().then(() => {}) + }, []) + + useEffect(() => { + const url = new URL(window.location.href) + const index = url.searchParams.get("sample") + if (index !== null) { + washHand() + const indexNumber = Number.parseInt(index) + if (!Number.isNaN(indexNumber) && indexNumber >= 0 && indexNumber <= indexStore.max) { + indexStore.setIndex(indexNumber) + } + } + }, [indexStore.max]) + + useEffect(() => { + getAllLabels().then((labels) => { + setLabels(labels) + getLock.current = true + }) }, []) useEffect(() => { - getSingleTask(labelIndex) + getSingleTask(indexStore.index) .then(task => { if ("doc" in task) { setCurrentTask(task) @@ -194,10 +186,10 @@ export default function Index() { setCurrentTask(null) console.error(error) }) - }, [labelIndex]) + }, [indexStore.index]) const updateHistory = () => { - getTaskHistory(labelIndex) + getTaskHistory(indexStore.index) .then(data => { setHistory(data) setViewingRecord(null) @@ -290,7 +282,7 @@ export default function Index() { _.debounce(() => { if (DISABLE_QUERY || viewingRecord != null) return setWaiting(rangeId === "summary" ? "doc" : "summary") - selectText(labelIndex, { + selectText(indexStore.index, { start: firstRange[0], end: firstRange[1], from_summary: rangeId === "summary", @@ -311,7 +303,7 @@ export default function Index() { console.error(error) }) }, 100)() - }, [firstRange, rangeId, labelIndex]) + }, [firstRange, rangeId, indexStore.index]) const washHand = () => { setFirstRange(null) @@ -344,7 +336,7 @@ export default function Index() { if (firstRange === null || rangeId === null) { return Promise.resolve() } - await labelText(labelIndex, { + await labelText(indexStore.index, { source_start: rangeId === "summary" ? -1 : firstRange[0], source_end: rangeId === "summary" ? -1 + 1 : firstRange[1], summary_start: rangeId === "summary" ? firstRange[0] : -1, @@ -403,7 +395,7 @@ export default function Index() { if (firstRange === null || rangeId === null) { return Promise.resolve() } - await labelText(labelIndex, { + await labelText(indexStore.index, { source_start: rangeId === "summary" ? slice[0] : firstRange[0], source_end: rangeId === "summary" ? slice[1] + 1 : firstRange[1], summary_start: rangeId === "summary" ? firstRange[0] : slice[0], @@ -495,7 +487,7 @@ export default function Index() { */}
-
- - - - - -
+
{currentTask === null ? (

Loading...

diff --git a/components/labelPagination.tsx b/components/labelPagination.tsx new file mode 100644 index 0000000..44cce5e --- /dev/null +++ b/components/labelPagination.tsx @@ -0,0 +1,54 @@ +"use client" + +import { Button, Field, ProgressBar } from "@fluentui/react-components" +import { ChevronLeftRegular, IosChevronRightRegular } from "@fluentui/react-icons" +import { useTrackedIndexStore } from "../store/useIndexStore" + +type Props = { + beforeChangeIndex?: Function, +} + +export default function LabelPagination({ beforeChangeIndex = () => {} }: Props) { + const indexStore = useTrackedIndexStore() + + return ( +
+ + + + + +
+ ) +} diff --git a/store/useIndexStore.tsx b/store/useIndexStore.tsx new file mode 100644 index 0000000..356742a --- /dev/null +++ b/store/useIndexStore.tsx @@ -0,0 +1,26 @@ +import { create } from "zustand" +import { createTrackedSelector } from "react-tracked" +import { getAllTasksLength } from "../utils/request" + +interface IndexStore { + index: number + max: number + fetchMaxIndex: () => Promise + previous: () => void + next: () => void + setIndex: (index: number) => void +} + +export const useIndexStore = create()((set) => ({ + index: 0, + max: 0, + fetchMaxIndex: async () => { + const tasks = await getAllTasksLength() + set({ max: tasks.all - 1 }) + }, + previous: () => set((state) => ({ index: state.index - 1})), + next: () => set((state) => ({ index: state.index + 1})), + setIndex: (index: number) => set({ index }) +})) + +export const useTrackedIndexStore = createTrackedSelector(useIndexStore) From 94d90ea6cc08696b69b72360ae0a96c5e01e1d05 Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Sat, 16 Nov 2024 14:35:55 +0800 Subject: [PATCH 14/43] refactor(frontend): Separate ExistingPane component --- app/page.tsx | 174 ++++++--------------------------- components/editor/existing.tsx | 119 ++++++++++++++++++++++ store/useHistoryStore.ts | 30 ++++++ store/useTaskStore.ts | 26 +++++ 4 files changed, 204 insertions(+), 145 deletions(-) create mode 100644 components/editor/existing.tsx create mode 100644 store/useHistoryStore.ts create mode 100644 store/useTaskStore.ts diff --git a/app/page.tsx b/app/page.tsx index 9abf6e0..9d3741f 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -5,8 +5,6 @@ import { Button, Card, CardHeader, - Table, TableBody, TableCell, - TableHeader, TableHeaderCell, TableRow, Text, Title1, Toast, Toaster, ToastTitle, useId, useToastController, } from "@fluentui/react-components" @@ -17,31 +15,26 @@ import { updateSliceArray } from "../utils/mergeArray" import getRangeTextHandleableRange from "../utils/rangeTextNodes" import { exportLabel, - getTaskHistory, - getSingleTask, labelText, selectText, - deleteRecord, getAllLabels, checkUserMe, } from "../utils/request" -import { type LabelData, type SectionResponse, type Task, userSectionResponse } from "../utils/types" +import { type SectionResponse, userSectionResponse } from "../utils/types" import { ArrowExportRegular, - ArrowSyncRegular, - DeleteRegular, - EyeOffRegular, - EyeRegular, HandRightRegular, ShareRegular, } from "@fluentui/react-icons" import { Allotment } from "allotment" import "allotment/dist/style.css" -import ColumnResize from "react-table-column-resizer" import "./page.css" import UserPopover from "../components/userPopover" import LabelPagination from "../components/labelPagination" import { useTrackedIndexStore } from "../store/useIndexStore" +import ExistingPane from "../components/editor/existing" +import { useHistoryStore } from "../store/useHistoryStore" +import { useTaskStore } from "../store/useTaskStore" enum Stage { None = 0, @@ -111,7 +104,8 @@ const exportJSON = () => { export default function Index() { const indexStore = useTrackedIndexStore() - const [currentTask, setCurrentTask] = useState(null) + const historyStore = useHistoryStore() + const taskStore = useTaskStore() const getLock = useRef(false) const [firstRange, setFirstRange] = useState<[number, number] | null>(null) const [rangeId, setRangeId] = useState(null) @@ -119,8 +113,6 @@ export default function Index() { const [userSelection, setUserSelection] = useState<[number, number] | null>(null) const [waiting, setWaiting] = useState(null) const [stage, setStage] = useState(Stage.None) - const [history, setHistory] = useState(null) - const [viewingRecord, setViewingRecord] = useState(null) const [labels, setLabels] = useState<(string | object)[]>([]) const toasterId = useId("toaster") @@ -176,41 +168,22 @@ export default function Index() { }, []) useEffect(() => { - getSingleTask(indexStore.index) - .then(task => { - if ("doc" in task) { - setCurrentTask(task) - } - }) - .catch(error => { - setCurrentTask(null) - console.error(error) - }) + taskStore.fetch(indexStore.index).then(() => {}) }, [indexStore.index]) - const updateHistory = () => { - getTaskHistory(indexStore.index) - .then(data => { - setHistory(data) - setViewingRecord(null) - }).catch(error => { - setHistory(null) - setViewingRecord(null) - console.error(error) - }) - } - - useEffect(updateHistory, [currentTask]) + useEffect(() => { + historyStore.updateHistory(indexStore.index).then(() => {}) + }, [taskStore.current]) useEffect(() => { - if (viewingRecord === null || currentTask === null) { + if (historyStore.viewingRecord === null || taskStore.current === null) { washHand() return } - setFirstRange([viewingRecord.source_start, viewingRecord.source_end]) + setFirstRange([historyStore.viewingRecord.source_start, historyStore.viewingRecord.source_end]) setRangeId("doc") - setServerSelection([userSectionResponse(viewingRecord.summary_start, viewingRecord.summary_end, true)]) - }, [viewingRecord]) + setServerSelection([userSectionResponse(historyStore.viewingRecord.summary_start, historyStore.viewingRecord.summary_end, true)]) + }, [historyStore.viewingRecord]) useLayoutEffect(() => { const func = (event) => { @@ -280,7 +253,7 @@ export default function Index() { return } _.debounce(() => { - if (DISABLE_QUERY || viewingRecord != null) return + if (DISABLE_QUERY || historyStore.viewingRecord != null) return setWaiting(rangeId === "summary" ? "doc" : "summary") selectText(indexStore.index, { start: firstRange[0], @@ -344,7 +317,7 @@ export default function Index() { consistent: label, note: note, }) - updateHistory() + historyStore.updateHistory(indexStore.index).then(() => {}) }} message="Check all types that apply below." /> @@ -381,7 +354,7 @@ export default function Index() { const bg_color = isBackendSlice ? score === 2 ? "#85e834" : getColor(normalColor[slice[4]]) : "#ffffff" const textColor = isLightColor(bg_color) ? "black" : "white" // const textColor= 'red' - return isBackendSlice && viewingRecord == null ? ( + return isBackendSlice && historyStore.viewingRecord == null ? ( {}) }} message="Select the type(s) of hallucinatin below." /> @@ -473,7 +446,7 @@ export default function Index() { gap: "1em", }} > - {JSON.stringify(firstRange) === "[-1,-1]" || viewingRecord != null ? ( + {JSON.stringify(firstRange) === "[-1,-1]" || historyStore.viewingRecord != null ? ( @@ -526,7 +499,7 @@ export default function Index() {

- {currentTask === null ? ( + {taskStore.current === null ? (

Loading...

) : (
{ checkSelection(event.target as HTMLSpanElement) }} > {serverSelection !== null && serverSelection.length > 0 && rangeId === "summary" ? ( - + ) : rangeId === "doc" ? ( - + ) : ( - currentTask.doc + taskStore.current.doc )} @@ -601,109 +574,20 @@ export default function Index() { id="summary" as="p" data-mercury-label-start={0} - data-mercury-label-end={currentTask.sum.length} + data-mercury-label-end={taskStore.current.sum.length} onMouseUp={event => checkSelection(event.target as HTMLSpanElement)} > {serverSelection !== null && rangeId === "doc" ? ( - + ) : rangeId === "summary" ? ( - + ) : ( - currentTask.sum + taskStore.current.sum )}
-
- - - Existing annotations - - - } - /> - {history === null ? ( -

Loading...

- ) : ( - - - - Source - {/* @ts-ignore */} - - Summary - {/* @ts-ignore */} - - Label(s) - {/* @ts-ignore */} - - Note - {/* @ts-ignore */} - - {/* TODO: Display the resizer. Now they are invisible. */} - Actions - - - - {history - .sort((a, b) => { - let c = a.source_start - b.source_start - if (c === 0) c = a.summary_start - b.summary_start - return c - }) - .map((record, _) => ( - - {currentTask.doc.slice(record.source_start, record.source_end)} - - {currentTask.sum.slice(record.summary_start, record.summary_end)} - - {record.consistent.join(", ")} - - {record.note} - - - {viewingRecord != null && viewingRecord.record_id === record.record_id ? ( - - ) : ( - - )} - - - - - ))} - -
- )} - -
+ diff --git a/components/editor/existing.tsx b/components/editor/existing.tsx new file mode 100644 index 0000000..bd6d25e --- /dev/null +++ b/components/editor/existing.tsx @@ -0,0 +1,119 @@ +import { + Body1, + Button, + Card, + CardHeader, + Table, TableBody, TableCell, + TableHeader, + TableHeaderCell, + TableRow, +} from "@fluentui/react-components" +import { ArrowSyncRegular, DeleteRegular, EyeOffRegular, EyeRegular } from "@fluentui/react-icons" +import ColumnResize from "react-table-column-resizer" +import { deleteRecord } from "../../utils/request" +import { useTrackedHistoryStore } from "../../store/useHistoryStore" +import { useTrackedTaskStore } from "../../store/useTaskStore" + +type Props = { + onRefresh?: Function, +} + +export default function ExistingPane({ onRefresh = () => {} }: Props) { + const historyStore = useTrackedHistoryStore() + const taskStore = useTrackedTaskStore() + + return ( +
+ + + Existing annotations + + + } + /> + + + + Source + {/* @ts-ignore */} + + Summary + {/* @ts-ignore */} + + Label(s) + {/* @ts-ignore */} + + Note + {/* @ts-ignore */} + + {/* TODO: Display the resizer. Now they are invisible. */} + Actions + + + + {historyStore.history + .sort((a, b) => { + let c = a.source_start - b.source_start + if (c === 0) c = a.summary_start - b.summary_start + return c + }) + .map((record, _) => ( + + {taskStore.current.doc.slice(record.source_start, record.source_end)} + + {taskStore.current.sum.slice(record.summary_start, record.summary_end)} + + {record.consistent.join(", ")} + + {record.note} + + + {historyStore.viewingRecord != null && historyStore.viewingRecord.record_id === record.record_id ? ( + + ) : ( + + )} + + + + + ))} + +
+ +
+ ) +} \ No newline at end of file diff --git a/store/useHistoryStore.ts b/store/useHistoryStore.ts new file mode 100644 index 0000000..2b6dba0 --- /dev/null +++ b/store/useHistoryStore.ts @@ -0,0 +1,30 @@ +import type { LabelData } from "../utils/types" +import { create } from "zustand" +import { getTaskHistory } from "../utils/request" +import { createTrackedSelector } from "react-tracked" + +interface HistoryStore { + history: LabelData[], + viewingRecord: LabelData | null, + setHistory: (history: LabelData[]) => void, + setViewingRecord: (viewingRecord: LabelData) => void, + updateHistory: (labelIndex: number) => Promise, +} + +export const useHistoryStore = create()((set) => ({ + history: [], + viewingRecord: null, + setHistory: (history: LabelData[]) => set({ history }), + setViewingRecord: (viewingRecord: LabelData) => set({ viewingRecord }), + updateHistory: async (labelIndex: number) => { + try { + const history = await getTaskHistory(labelIndex) + set({ history, viewingRecord: null }) + } catch (e) { + set({ history: [], viewingRecord: null }) + console.error(e) + } + }, +})) + +export const useTrackedHistoryStore = createTrackedSelector(useHistoryStore) \ No newline at end of file diff --git a/store/useTaskStore.ts b/store/useTaskStore.ts new file mode 100644 index 0000000..1664f03 --- /dev/null +++ b/store/useTaskStore.ts @@ -0,0 +1,26 @@ +import { create } from "zustand" +import { Task } from "../utils/types" +import { getSingleTask } from "../utils/request" +import { createTrackedSelector } from "react-tracked" + +interface TaskStore { + current: Task | null, + fetch: (index: number) => Promise +} + +export const useTaskStore = create()((set) => ({ + current: null, + fetch: async (index: number) => { + try { + const task = await getSingleTask(index) + if ("doc" in task) { + set({ current: task }) + } + } catch (e) { + set({ current: null }) + console.error(e) + } + }, +})) + +export const useTrackedTaskStore= createTrackedSelector(useTaskStore) \ No newline at end of file From 0f885d7f0a532dc3d4632893f168440943f1b3e1 Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Sat, 16 Nov 2024 16:29:09 +0800 Subject: [PATCH 15/43] refactor(component): Rewrite ExistingPane component --- components/editor/existing.tsx | 174 +++++++++++++++++++++++++-------- components/labelPagination.tsx | 5 +- 2 files changed, 137 insertions(+), 42 deletions(-) diff --git a/components/editor/existing.tsx b/components/editor/existing.tsx index bd6d25e..46edcb7 100644 --- a/components/editor/existing.tsx +++ b/components/editor/existing.tsx @@ -3,25 +3,98 @@ import { Button, Card, CardHeader, - Table, TableBody, TableCell, + Table, + TableBody, + TableCell, TableHeader, TableHeaderCell, TableRow, + TableColumnDefinition, + createTableColumn, + useTableFeatures, + useTableColumnSizing_unstable, + TableColumnSizingOptions, MenuItem, Menu, MenuTrigger, MenuPopover, MenuList, TableCellLayout, } from "@fluentui/react-components" import { ArrowSyncRegular, DeleteRegular, EyeOffRegular, EyeRegular } from "@fluentui/react-icons" import ColumnResize from "react-table-column-resizer" import { deleteRecord } from "../../utils/request" import { useTrackedHistoryStore } from "../../store/useHistoryStore" import { useTrackedTaskStore } from "../../store/useTaskStore" +import { LabelData } from "../../utils/types" +import { useMemo, useState } from "react" + +const columnsDef: TableColumnDefinition[] = [ + createTableColumn({ + columnId: "source", + renderHeaderCell: () => <>Source, + }), + createTableColumn({ + columnId: "summary", + renderHeaderCell: () => <>Summary, + }), + createTableColumn({ + columnId: "consistent", + renderHeaderCell: () => <>Label(s), + }), + createTableColumn({ + columnId: "note", + renderHeaderCell: () => <>Note, + }), + createTableColumn({ + columnId: "actions", + renderHeaderCell: () => <>Actions, + }), +] type Props = { onRefresh?: Function, } -export default function ExistingPane({ onRefresh = () => {} }: Props) { +export default function ExistingPane({ onRefresh = Function() }: Props) { const historyStore = useTrackedHistoryStore() const taskStore = useTrackedTaskStore() + const sortedHistory = useMemo(() => { + return historyStore.history + .sort((a, b) => { + let c = a.source_start - b.source_start + if (c === 0) c = a.summary_start - b.summary_start + return c + }) + }, [historyStore.history]) + + const [columns] = useState[]>(columnsDef) + const [columnSizingOptions] = useState({ + source: { + idealWidth: 300, + minWidth: 150, + }, + summary: { + idealWidth: 300, + minWidth: 150, + }, + consistent: { + idealWidth: 150, + minWidth: 100, + }, + note: { + idealWidth: 300, + minWidth: 150, + }, + actions: { + idealWidth: 120, + }, + }) + + const { getRows, columnSizing_unstable, tableRef } = useTableFeatures( + { + columns, + items: sortedHistory, + }, + [useTableColumnSizing_unstable({ columnSizingOptions })], + ) + const rows = getRows() + return (
{} }: Props) { } /> - + {/*@ts-ignore*/} +
- Source - {/* @ts-ignore */} - - Summary - {/* @ts-ignore */} - - Label(s) - {/* @ts-ignore */} - - Note - {/* @ts-ignore */} - - {/* TODO: Display the resizer. Now they are invisible. */} - Actions + {columns.map((column) => ( + + + + {column.renderHeaderCell()} + + + + + + Keyboard Column Resizing + + + + + ))} - {historyStore.history - .sort((a, b) => { - let c = a.source_start - b.source_start - if (c === 0) c = a.summary_start - b.summary_start - return c - }) - .map((record, _) => ( - - {taskStore.current.doc.slice(record.source_start, record.source_end)} - - {taskStore.current.sum.slice(record.summary_start, record.summary_end)} - - {record.consistent.join(", ")} - - {record.note} - - - {historyStore.viewingRecord != null && historyStore.viewingRecord.record_id === record.record_id ? ( + {rows.map(({ item }) => ( + + + {taskStore.current.doc.slice(item.source_start, item.source_end)} + + + {taskStore.current.sum.slice(item.summary_start, item.summary_end)} + + + {item.consistent.join(", ")} + + + {item.note} + + + {historyStore.viewingRecord != null && historyStore.viewingRecord.record_id === item.record_id ? ( - - - ))} + + ))}
diff --git a/components/labelPagination.tsx b/components/labelPagination.tsx index 44cce5e..b5d650c 100644 --- a/components/labelPagination.tsx +++ b/components/labelPagination.tsx @@ -8,7 +8,7 @@ type Props = { beforeChangeIndex?: Function, } -export default function LabelPagination({ beforeChangeIndex = () => {} }: Props) { +export default function LabelPagination({ beforeChangeIndex = Function() }: Props) { const indexStore = useTrackedIndexStore() return ( @@ -34,7 +34,8 @@ export default function LabelPagination({ beforeChangeIndex = () => {} }: Props) > Previous - + - ) : ( - - )} + historyStore.setViewingRecord(null) + onRefresh() + }}> + Restore + + ) : ( - + )} + + ))} diff --git a/components/userPopover.tsx b/components/userPopover.tsx index f1c4c83..7f459e4 100644 --- a/components/userPopover.tsx +++ b/components/userPopover.tsx @@ -2,7 +2,6 @@ import { Avatar, - Body1, Button, Field, Input, @@ -17,10 +16,11 @@ import { useEffect, useState } from "react" export default function UserPopover() { const userState = useTrackedUserStore() const [open, setOpen] = useState(false) - const handleOpenChange: PopoverProps["onOpenChange"] = (e, data) => setOpen(data.open || false) + const handleOpenChange: PopoverProps["onOpenChange"] = (_, data) => setOpen(data.open || false) useEffect(() => { - userState.fetch().then(() => {}) + userState.fetch().then(() => { + }) }, []) async function formSetName(formData) { From 0193e49995ec941553cf0cb58b0a94611f8ac92a Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Sat, 16 Nov 2024 16:40:47 +0800 Subject: [PATCH 17/43] fix(bug): Can not load summary slice --- server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server.py b/server.py index 76b4c78..8e080e2 100644 --- a/server.py +++ b/server.py @@ -242,7 +242,7 @@ async def post_selections(task_index: int, selection: Selection): else: text_type = "summary" - chunk_id_and_text = database.db.execute(sql_cmd, [text_type, task_index]).fetchall() + chunk_id_and_text = database.mercury_db.execute(sql_cmd, [text_type, task_index]).fetchall() search_chunk_ids = [row[0] for row in chunk_id_and_text] vecter_db_row_ids = [str(x + 1) for x in search_chunk_ids] # rowid starts from 1 while chunk_id starts from 0 @@ -270,7 +270,7 @@ async def post_selections(task_index: int, selection: Selection): # print ("SQL_CMD", sql_cmd) # vector_search_result = database.db.execute(sql_cmd, [*search_chunk_ids, serialize_f32(embedding)]).fetchall() - vector_search_result = database.db.execute(sql_cmd).fetchall() + vector_search_result = database.mercury_db.execute(sql_cmd).fetchall() # [(2, 0.20000001788139343), (1, 0.40000003576278687)] # turn this into a dict from chunk__id to distance/score chunk_id_to_score = {row[0]: row[1] for row in vector_search_result} @@ -280,7 +280,7 @@ async def post_selections(task_index: int, selection: Selection): sql_cmd = "SELECT chunk_id, text, char_offset FROM chunks WHERE chunk_id in ({0});".format( ', '.join('?' for _ in chunk_ids_of_top_k)) search_chunk_ids = [row[0] for row in vector_search_result] - response = database.db.execute(sql_cmd, search_chunk_ids).fetchall() + response = database.mercury_db.execute(sql_cmd, search_chunk_ids).fetchall() # [(1, 'This is a test.', 0, 14), (2, 'This is a test.', 15, 14)] # organize into a dict of keys "score", "offset", "len", "to_doc" From 441a4644a8ce1c2d319da1e170026f245a3b7909 Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Sat, 16 Nov 2024 17:38:08 +0800 Subject: [PATCH 18/43] chore: Remove redundant code --- app/page.tsx | 20 +++++++++----------- package.json | 1 - pnpm-lock.yaml | 20 -------------------- 3 files changed, 9 insertions(+), 32 deletions(-) diff --git a/app/page.tsx b/app/page.tsx index 9b94e33..a7f0168 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -9,7 +9,7 @@ import { Title1, Toast, Toaster, ToastTitle, useId, useToastController, } from "@fluentui/react-components" import _ from "lodash" -import { useEffect, useLayoutEffect, useRef, useState } from "react" +import { useEffect, useLayoutEffect, useState } from "react" import Tooltip from "../components/tooltip" import { updateSliceArray } from "../utils/mergeArray" import getRangeTextHandleableRange from "../utils/rangeTextNodes" @@ -71,17 +71,17 @@ const getColor = (score: number) => { // Function to determine if a color is light or dark const isLightColor = (color: string) => { // Remove the hash if present - let newcolor = color.replace("#", "") + let newColor = color.replace("#", "") // Convert 3-digit hex to 6-digit hex - if (newcolor.length === 3) { - newcolor = newcolor.split("").map(char => char + char).join("") + if (newColor.length === 3) { + newColor = newColor.split("").map(char => char + char).join("") } // Convert hex to RGB - const r = Number.parseInt(newcolor.substring(0, 2), 16) - const g = Number.parseInt(newcolor.substring(2, 4), 16) - const b = Number.parseInt(newcolor.substring(4, 6), 16) + const r = Number.parseInt(newColor.substring(0, 2), 16) + const g = Number.parseInt(newColor.substring(2, 4), 16) + const b = Number.parseInt(newColor.substring(4, 6), 16) // Calculate luminance const luminance = 0.2126 * r + 0.7152 * g + 0.0722 * b @@ -106,7 +106,6 @@ export default function Index() { const indexStore = useTrackedIndexStore() const historyStore = useHistoryStore() const taskStore = useTaskStore() - const getLock = useRef(false) const [firstRange, setFirstRange] = useState<[number, number] | null>(null) const [rangeId, setRangeId] = useState(null) const [serverSelection, setServerSelection] = useState(null) @@ -164,7 +163,6 @@ export default function Index() { useEffect(() => { getAllLabels().then((labels) => { setLabels(labels) - getLock.current = true }) }, []) @@ -463,8 +461,8 @@ export default function Index() { -
) -} \ No newline at end of file +} diff --git a/components/userPopover.tsx b/components/userPopover.tsx index 7f459e4..17021ac 100644 --- a/components/userPopover.tsx +++ b/components/userPopover.tsx @@ -11,18 +11,13 @@ import { } from "@fluentui/react-components" import { changeName } from "../utils/request" import { useTrackedUserStore } from "../store/useUserStore" -import { useEffect, useState } from "react" +import { useState } from "react" export default function UserPopover() { const userState = useTrackedUserStore() const [open, setOpen] = useState(false) const handleOpenChange: PopoverProps["onOpenChange"] = (_, data) => setOpen(data.open || false) - useEffect(() => { - userState.fetch().then(() => { - }) - }, []) - async function formSetName(formData) { changeName(formData.get("newName")).then(() => { userState.setName(formData.get("newName")) diff --git a/store/useIndexStore.tsx b/store/useIndexStore.ts similarity index 74% rename from store/useIndexStore.tsx rename to store/useIndexStore.ts index 356742a..11c1261 100644 --- a/store/useIndexStore.tsx +++ b/store/useIndexStore.ts @@ -5,22 +5,19 @@ import { getAllTasksLength } from "../utils/request" interface IndexStore { index: number max: number - fetchMaxIndex: () => Promise previous: () => void next: () => void setIndex: (index: number) => void + setMaxIndex: (max: number) => void } export const useIndexStore = create()((set) => ({ index: 0, max: 0, - fetchMaxIndex: async () => { - const tasks = await getAllTasksLength() - set({ max: tasks.all - 1 }) - }, previous: () => set((state) => ({ index: state.index - 1})), next: () => set((state) => ({ index: state.index + 1})), - setIndex: (index: number) => set({ index }) + setIndex: (index: number) => set({ index }), + setMaxIndex: (max: number) => set({ max }) })) export const useTrackedIndexStore = createTrackedSelector(useIndexStore) From 69c6d5ea81624b418eaeb1075ea8c5c4afe92784 Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Sun, 17 Nov 2024 11:55:04 +0800 Subject: [PATCH 20/43] refactor(frontend): Reduce useEffect usage to prevent unnecessary re-render --- app/login/page.tsx | 4 +- app/page.tsx | 61 ++++----- components/editor/existing.tsx | 8 +- package.json | 1 + pnpm-lock.yaml | 217 +++++++++++++++++---------------- 5 files changed, 147 insertions(+), 144 deletions(-) diff --git a/app/login/page.tsx b/app/login/page.tsx index 1b784c5..e9cfdd1 100644 --- a/app/login/page.tsx +++ b/app/login/page.tsx @@ -11,6 +11,7 @@ import { useToastController, } from "@fluentui/react-components" import { login } from "../../utils/request" +import { useRouter } from "next/navigation" const useStackClassName = makeResetStyles({ display: "flex", @@ -20,6 +21,7 @@ const useStackClassName = makeResetStyles({ }) export default function Login() { + const router = useRouter() const toasterId = useId("toaster") const { dispatchToast } = useToastController(toasterId) @@ -27,7 +29,7 @@ export default function Login() { if (formData.get("email") && formData.get("password")) { const success = await login(formData.get("email"), formData.get("password")) if (success) { - window.location.href = "/" + router.push("/") } else { dispatchToast( diff --git a/app/page.tsx b/app/page.tsx index 7ddf745..cc21aba 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -36,6 +36,7 @@ import ExistingPane from "../components/editor/existing" import { useTrackedHistoryStore } from "../store/useHistoryStore" import { useTrackedTaskStore } from "../store/useTaskStore" import { useTrackedUserStore } from "../store/useUserStore" +import { useRouter } from "next/navigation" enum Stage { None = 0, @@ -105,9 +106,11 @@ const exportJSON = () => { export default function Index() { const indexStore = useTrackedIndexStore() + const [prevIndex, setPrevIndex] = useState(indexStore.index) const historyStore = useTrackedHistoryStore() const taskStore = useTrackedTaskStore() const userStore = useTrackedUserStore() + const router = useRouter() const [firstRange, setFirstRange] = useState<[number, number] | null>(null) const [rangeId, setRangeId] = useState(null) const [serverSelection, setServerSelection] = useState(null) @@ -119,6 +122,17 @@ export default function Index() { const toasterId = useId("toaster") const { dispatchToast } = useToastController(toasterId) + function washHand() { + setFirstRange(null) + setRangeId(null) + setWaiting(null) + setServerSelection(null) + setStage(Stage.None) + setUserSelection(null) + historyStore.setViewingRecord(null) + window.getSelection()?.removeAllRanges() + } + useEffect(() => { const access_token = localStorage.getItem("access_token") if (access_token == "" || access_token == null) { @@ -128,7 +142,7 @@ export default function Index() { , { intent: "error" }, ) - window.location.href = "/login" + router.push("/login") return } checkUserMe(access_token).then(valid => { @@ -140,29 +154,29 @@ export default function Index() { , { intent: "error" }, ) + router.push("/login") } return }) }, []) - useEffect(() => { - getAllTasksLength().then((task) => { - indexStore.setMaxIndex(task.all - 1) - }) - userStore.fetch() - }, []) + getAllTasksLength().then((task) => { + indexStore.setMaxIndex(task.all - 1) + }) + userStore.fetch() - useEffect(() => { - const url = new URL(window.location.href) - const index = url.searchParams.get("sample") - if (index !== null) { + console.log("Re-rendered") + + const url = new URL(window.location.href) + const index = url.searchParams.get("sample") + if (index !== null) { + const indexNumber = Number.parseInt(index) + if (!Number.isNaN(indexNumber) && indexNumber >= 0 && indexNumber <= indexStore.max && indexNumber !== prevIndex) { + setPrevIndex(indexNumber) washHand() - const indexNumber = Number.parseInt(index) - if (!Number.isNaN(indexNumber) && indexNumber >= 0 && indexNumber <= indexStore.max) { - indexStore.setIndex(indexNumber) - } + indexStore.setIndex(indexNumber) } - }, [indexStore.max]) + } useEffect(() => { getAllLabels().then((labels) => { @@ -170,10 +184,7 @@ export default function Index() { }) }, []) - useEffect(() => { - taskStore.fetch(indexStore.index).then(() => { - }) - }, [indexStore.index]) + taskStore.fetch(indexStore.index) useEffect(() => { historyStore.updateHistory(indexStore.index).then(() => { @@ -283,16 +294,6 @@ export default function Index() { }, 100)() }, [firstRange, rangeId, indexStore.index]) - const washHand = () => { - setFirstRange(null) - setRangeId(null) - setWaiting(null) - setServerSelection(null) - setStage(Stage.None) - setUserSelection(null) - window.getSelection()?.removeAllRanges() - } - const JustSliceText = (props: { text: string; startAndEndOffset: [number, number] }) => { const fakeResponse = userSectionResponse( props.startAndEndOffset[0], diff --git a/components/editor/existing.tsx b/components/editor/existing.tsx index 38b7ad2..774177d 100644 --- a/components/editor/existing.tsx +++ b/components/editor/existing.tsx @@ -68,19 +68,15 @@ export default function ExistingPane({ onRefresh = Function() }: Props) { const [columnSizingOptions] = useState({ source: { idealWidth: 300, - minWidth: 150, }, summary: { idealWidth: 300, - minWidth: 150, }, consistent: { - idealWidth: 150, - minWidth: 100, + idealWidth: 100, }, note: { - idealWidth: 300, - minWidth: 150, + idealWidth: 150, }, actions: { idealWidth: 120, diff --git a/package.json b/package.json index f7eccb2..2dc4dce 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "react-dom": "^18.2.0", "react-table-column-resizer": "^1.2.3", "react-tracked": "^2.0.1", + "scheduler": "^0.23.2", "zustand": "^5.0.1" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3db7b83..be22ef9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,7 +13,7 @@ importers: version: 0.26.12(react-dom@18.2.0(react@18.2.0))(react@18.2.0) '@fluentui/react-components': specifier: ^9.47.3 - version: 9.47.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + version: 9.47.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-icons': specifier: ^2.0.250 version: 2.0.250(react@18.2.0) @@ -40,7 +40,10 @@ importers: version: 1.2.4(react@18.2.0) react-tracked: specifier: ^2.0.1 - version: 2.0.1(react@18.2.0)(scheduler@0.23.0) + version: 2.0.1(react@18.2.0)(scheduler@0.23.2) + scheduler: + specifier: ^0.23.2 + version: 0.23.2 zustand: specifier: ^5.0.1 version: 5.0.1(@types/react@18.2.33)(react@18.2.0) @@ -819,8 +822,8 @@ packages: rtl-css-js@1.16.1: resolution: {integrity: sha512-lRQgou1mu19e+Ya0LsTvKrVJ5TYUbqCVPAiImX3UfLTenarvPUl1QFdvu5Z3PYmHT9RCcwIfbjRQBntExyj3Zg==} - scheduler@0.23.0: - resolution: {integrity: sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw==} + scheduler@0.23.2: + resolution: {integrity: sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==} source-map-js@1.2.0: resolution: {integrity: sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==} @@ -981,10 +984,10 @@ snapshots: dependencies: '@swc/helpers': 0.5.2 - '@fluentui/react-accordion@9.3.47(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-accordion@9.3.47(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: '@fluentui/react-aria': 9.10.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-context-selector': 9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-context-selector': 9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-icons': 2.0.250(react@18.2.0) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-shared-contexts': 9.16.0(@types/react@18.2.33)(react@18.2.0) @@ -1000,9 +1003,9 @@ snapshots: transitivePeerDependencies: - scheduler - '@fluentui/react-alert@9.0.0-beta.115(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-alert@9.0.0-beta.115(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: - '@fluentui/react-avatar': 9.6.20(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-avatar': 9.6.20(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-button': 9.3.74(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) '@fluentui/react-icons': 2.0.250(react@18.2.0) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) @@ -1031,13 +1034,13 @@ snapshots: react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - '@fluentui/react-avatar@9.6.20(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-avatar@9.6.20(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: '@fluentui/react-badge': 9.2.30(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-context-selector': 9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-context-selector': 9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-icons': 2.0.250(react@18.2.0) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) - '@fluentui/react-popover': 9.9.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-popover': 9.9.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-shared-contexts': 9.16.0(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-tabster': 9.19.6(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) '@fluentui/react-theme': 9.1.19 @@ -1115,9 +1118,9 @@ snapshots: react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - '@fluentui/react-checkbox@9.2.19(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-checkbox@9.2.19(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: - '@fluentui/react-field': 9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-field': 9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-icons': 2.0.250(react@18.2.0) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-label': 9.1.67(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) @@ -1134,12 +1137,12 @@ snapshots: transitivePeerDependencies: - scheduler - '@fluentui/react-combobox@9.9.5(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-combobox@9.9.5(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: '@fluentui/keyboard-keys': 9.0.7 '@fluentui/react-aria': 9.10.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-context-selector': 9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) - '@fluentui/react-field': 9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-context-selector': 9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) + '@fluentui/react-field': 9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-icons': 2.0.250(react@18.2.0) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-portal': 9.4.19(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) @@ -1157,57 +1160,57 @@ snapshots: transitivePeerDependencies: - scheduler - '@fluentui/react-components@9.47.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-components@9.47.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: - '@fluentui/react-accordion': 9.3.47(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) - '@fluentui/react-alert': 9.0.0-beta.115(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-accordion': 9.3.47(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) + '@fluentui/react-alert': 9.0.0-beta.115(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-aria': 9.10.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-avatar': 9.6.20(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-avatar': 9.6.20(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-badge': 9.2.30(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) '@fluentui/react-breadcrumb': 9.0.20(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) '@fluentui/react-button': 9.3.74(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) '@fluentui/react-card': 9.0.73(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-checkbox': 9.2.19(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) - '@fluentui/react-combobox': 9.9.5(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) - '@fluentui/react-dialog': 9.9.16(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-checkbox': 9.2.19(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) + '@fluentui/react-combobox': 9.9.5(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) + '@fluentui/react-dialog': 9.9.16(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-divider': 9.2.66(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-drawer': 9.1.10(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) - '@fluentui/react-field': 9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-drawer': 9.1.10(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) + '@fluentui/react-field': 9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-image': 9.1.63(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-infobutton': 9.0.0-beta.99(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) - '@fluentui/react-infolabel': 9.0.27(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) - '@fluentui/react-input': 9.4.70(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-infobutton': 9.0.0-beta.99(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) + '@fluentui/react-infolabel': 9.0.27(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) + '@fluentui/react-input': 9.4.70(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-label': 9.1.67(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) '@fluentui/react-link': 9.2.16(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-menu': 9.13.6(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-menu': 9.13.6(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-message-bar': 9.0.25(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-overflow': 9.1.16(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) - '@fluentui/react-persona': 9.2.79(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) - '@fluentui/react-popover': 9.9.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-overflow': 9.1.16(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) + '@fluentui/react-persona': 9.2.79(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) + '@fluentui/react-popover': 9.9.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-portal': 9.4.19(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) '@fluentui/react-positioning': 9.14.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-progress': 9.1.70(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-progress': 9.1.70(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-provider': 9.13.17(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-radio': 9.2.14(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-radio': 9.2.14(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-rating': 9.0.2(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-select': 9.1.70(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-select': 9.1.70(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-shared-contexts': 9.16.0(@types/react@18.2.33)(react@18.2.0) - '@fluentui/react-skeleton': 9.0.58(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) - '@fluentui/react-slider': 9.1.76(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) - '@fluentui/react-spinbutton': 9.2.70(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-skeleton': 9.0.58(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) + '@fluentui/react-slider': 9.1.76(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) + '@fluentui/react-spinbutton': 9.2.70(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-spinner': 9.4.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-switch': 9.1.76(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) - '@fluentui/react-table': 9.12.1(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) - '@fluentui/react-tabs': 9.4.15(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-switch': 9.1.76(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) + '@fluentui/react-table': 9.12.1(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) + '@fluentui/react-tabs': 9.4.15(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-tabster': 9.19.6(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-tags': 9.2.1(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-tags': 9.2.1(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-text': 9.4.15(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-textarea': 9.3.70(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-textarea': 9.3.70(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-theme': 9.1.19 '@fluentui/react-toast': 9.3.36(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-toolbar': 9.1.77(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-toolbar': 9.1.77(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-tooltip': 9.4.22(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-tree': 9.4.37(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-tree': 9.4.37(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-utilities': 9.18.6(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-virtualizer': 9.0.0-alpha.74(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) '@griffel/react': 1.5.21(react@18.2.0) @@ -1219,7 +1222,7 @@ snapshots: transitivePeerDependencies: - scheduler - '@fluentui/react-context-selector@9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-context-selector@9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: '@fluentui/react-utilities': 9.18.6(@types/react@18.2.33)(react@18.2.0) '@swc/helpers': 0.5.2 @@ -1227,13 +1230,13 @@ snapshots: '@types/react-dom': 18.2.14 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - scheduler: 0.23.0 + scheduler: 0.23.2 - '@fluentui/react-dialog@9.9.16(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-dialog@9.9.16(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: '@fluentui/keyboard-keys': 9.0.7 '@fluentui/react-aria': 9.10.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-context-selector': 9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-context-selector': 9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-icons': 2.0.250(react@18.2.0) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-portal': 9.4.19(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) @@ -1264,9 +1267,9 @@ snapshots: react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - '@fluentui/react-drawer@9.1.10(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-drawer@9.1.10(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: - '@fluentui/react-dialog': 9.9.16(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-dialog': 9.9.16(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-motion-preview': 0.5.18(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) '@fluentui/react-shared-contexts': 9.16.0(@types/react@18.2.33)(react@18.2.0) @@ -1282,9 +1285,9 @@ snapshots: transitivePeerDependencies: - scheduler - '@fluentui/react-field@9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-field@9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: - '@fluentui/react-context-selector': 9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-context-selector': 9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-icons': 2.0.250(react@18.2.0) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-label': 9.1.67(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) @@ -1318,12 +1321,12 @@ snapshots: react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - '@fluentui/react-infobutton@9.0.0-beta.99(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-infobutton@9.0.0-beta.99(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: '@fluentui/react-icons': 2.0.250(react@18.2.0) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-label': 9.1.67(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-popover': 9.9.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-popover': 9.9.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-tabster': 9.19.6(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) '@fluentui/react-theme': 9.1.19 '@fluentui/react-utilities': 9.18.6(@types/react@18.2.33)(react@18.2.0) @@ -1336,12 +1339,12 @@ snapshots: transitivePeerDependencies: - scheduler - '@fluentui/react-infolabel@9.0.27(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-infolabel@9.0.27(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: '@fluentui/react-icons': 2.0.250(react@18.2.0) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-label': 9.1.67(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-popover': 9.9.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-popover': 9.9.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-tabster': 9.19.6(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) '@fluentui/react-theme': 9.1.19 '@fluentui/react-utilities': 9.18.6(@types/react@18.2.33)(react@18.2.0) @@ -1354,9 +1357,9 @@ snapshots: transitivePeerDependencies: - scheduler - '@fluentui/react-input@9.4.70(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-input@9.4.70(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: - '@fluentui/react-field': 9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-field': 9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-shared-contexts': 9.16.0(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-theme': 9.1.19 @@ -1406,11 +1409,11 @@ snapshots: react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - '@fluentui/react-menu@9.13.6(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-menu@9.13.6(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: '@fluentui/keyboard-keys': 9.0.7 '@fluentui/react-aria': 9.10.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-context-selector': 9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-context-selector': 9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-icons': 2.0.250(react@18.2.0) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-portal': 9.4.19(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) @@ -1457,10 +1460,10 @@ snapshots: react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - '@fluentui/react-overflow@9.1.16(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-overflow@9.1.16(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: '@fluentui/priority-overflow': 9.1.11 - '@fluentui/react-context-selector': 9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-context-selector': 9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-theme': 9.1.19 '@fluentui/react-utilities': 9.18.6(@types/react@18.2.33)(react@18.2.0) '@griffel/react': 1.5.21(react@18.2.0) @@ -1472,9 +1475,9 @@ snapshots: transitivePeerDependencies: - scheduler - '@fluentui/react-persona@9.2.79(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-persona@9.2.79(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: - '@fluentui/react-avatar': 9.6.20(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-avatar': 9.6.20(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-badge': 9.2.30(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-shared-contexts': 9.16.0(@types/react@18.2.33)(react@18.2.0) @@ -1489,11 +1492,11 @@ snapshots: transitivePeerDependencies: - scheduler - '@fluentui/react-popover@9.9.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-popover@9.9.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: '@fluentui/keyboard-keys': 9.0.7 '@fluentui/react-aria': 9.10.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-context-selector': 9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-context-selector': 9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-portal': 9.4.19(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) '@fluentui/react-positioning': 9.14.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) @@ -1537,9 +1540,9 @@ snapshots: react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - '@fluentui/react-progress@9.1.70(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-progress@9.1.70(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: - '@fluentui/react-field': 9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-field': 9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-shared-contexts': 9.16.0(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-theme': 9.1.19 @@ -1569,9 +1572,9 @@ snapshots: react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - '@fluentui/react-radio@9.2.14(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-radio@9.2.14(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: - '@fluentui/react-field': 9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-field': 9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-label': 9.1.67(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) '@fluentui/react-shared-contexts': 9.16.0(@types/react@18.2.33)(react@18.2.0) @@ -1601,9 +1604,9 @@ snapshots: react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - '@fluentui/react-select@9.1.70(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-select@9.1.70(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: - '@fluentui/react-field': 9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-field': 9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-icons': 2.0.250(react@18.2.0) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-shared-contexts': 9.16.0(@types/react@18.2.33)(react@18.2.0) @@ -1625,9 +1628,9 @@ snapshots: '@types/react': 18.2.33 react: 18.2.0 - '@fluentui/react-skeleton@9.0.58(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-skeleton@9.0.58(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: - '@fluentui/react-field': 9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-field': 9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-shared-contexts': 9.16.0(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-theme': 9.1.19 @@ -1641,9 +1644,9 @@ snapshots: transitivePeerDependencies: - scheduler - '@fluentui/react-slider@9.1.76(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-slider@9.1.76(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: - '@fluentui/react-field': 9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-field': 9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-shared-contexts': 9.16.0(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-tabster': 9.19.6(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) @@ -1658,10 +1661,10 @@ snapshots: transitivePeerDependencies: - scheduler - '@fluentui/react-spinbutton@9.2.70(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-spinbutton@9.2.70(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: '@fluentui/keyboard-keys': 9.0.7 - '@fluentui/react-field': 9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-field': 9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-icons': 2.0.250(react@18.2.0) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-shared-contexts': 9.16.0(@types/react@18.2.33)(react@18.2.0) @@ -1690,9 +1693,9 @@ snapshots: react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - '@fluentui/react-switch@9.1.76(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-switch@9.1.76(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: - '@fluentui/react-field': 9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-field': 9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-icons': 2.0.250(react@18.2.0) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-label': 9.1.67(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) @@ -1709,16 +1712,16 @@ snapshots: transitivePeerDependencies: - scheduler - '@fluentui/react-table@9.12.1(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-table@9.12.1(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: '@fluentui/keyboard-keys': 9.0.7 '@fluentui/react-aria': 9.10.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-avatar': 9.6.20(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) - '@fluentui/react-checkbox': 9.2.19(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) - '@fluentui/react-context-selector': 9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-avatar': 9.6.20(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) + '@fluentui/react-checkbox': 9.2.19(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) + '@fluentui/react-context-selector': 9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-icons': 2.0.250(react@18.2.0) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) - '@fluentui/react-radio': 9.2.14(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-radio': 9.2.14(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-shared-contexts': 9.16.0(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-tabster': 9.19.6(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) '@fluentui/react-theme': 9.1.19 @@ -1732,9 +1735,9 @@ snapshots: transitivePeerDependencies: - scheduler - '@fluentui/react-tabs@9.4.15(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-tabs@9.4.15(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: - '@fluentui/react-context-selector': 9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-context-selector': 9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-shared-contexts': 9.16.0(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-tabster': 9.19.6(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) @@ -1763,11 +1766,11 @@ snapshots: react-dom: 18.2.0(react@18.2.0) tabster: 6.1.0 - '@fluentui/react-tags@9.2.1(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-tags@9.2.1(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: '@fluentui/keyboard-keys': 9.0.7 '@fluentui/react-aria': 9.10.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-avatar': 9.6.20(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-avatar': 9.6.20(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-icons': 2.0.250(react@18.2.0) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-shared-contexts': 9.16.0(@types/react@18.2.33)(react@18.2.0) @@ -1796,9 +1799,9 @@ snapshots: react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - '@fluentui/react-textarea@9.3.70(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-textarea@9.3.70(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: - '@fluentui/react-field': 9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-field': 9.1.60(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-shared-contexts': 9.16.0(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-theme': 9.1.19 @@ -1836,13 +1839,13 @@ snapshots: react-dom: 18.2.0(react@18.2.0) react-transition-group: 4.4.5(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-toolbar@9.1.77(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-toolbar@9.1.77(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: '@fluentui/react-button': 9.3.74(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-context-selector': 9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-context-selector': 9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-divider': 9.2.66(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) - '@fluentui/react-radio': 9.2.14(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-radio': 9.2.14(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-shared-contexts': 9.16.0(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-tabster': 9.19.6(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) '@fluentui/react-theme': 9.1.19 @@ -1873,17 +1876,17 @@ snapshots: react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - '@fluentui/react-tree@9.4.37(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0)': + '@fluentui/react-tree@9.4.37(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2)': dependencies: '@fluentui/keyboard-keys': 9.0.7 '@fluentui/react-aria': 9.10.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-avatar': 9.6.20(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-avatar': 9.6.20(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-button': 9.3.74(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@fluentui/react-checkbox': 9.2.19(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) - '@fluentui/react-context-selector': 9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-checkbox': 9.2.19(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) + '@fluentui/react-context-selector': 9.1.57(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-icons': 2.0.250(react@18.2.0) '@fluentui/react-jsx-runtime': 9.0.35(@types/react@18.2.33)(react@18.2.0) - '@fluentui/react-radio': 9.2.14(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.0) + '@fluentui/react-radio': 9.2.14(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(scheduler@0.23.2) '@fluentui/react-shared-contexts': 9.16.0(@types/react@18.2.33)(react@18.2.0) '@fluentui/react-tabster': 9.19.6(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) '@fluentui/react-theme': 9.1.19 @@ -2094,7 +2097,7 @@ snapshots: dependencies: loose-envify: 1.4.0 react: 18.2.0 - scheduler: 0.23.0 + scheduler: 0.23.2 react-is@16.13.1: {} @@ -2104,12 +2107,12 @@ snapshots: dependencies: react: 18.2.0 - react-tracked@2.0.1(react@18.2.0)(scheduler@0.23.0): + react-tracked@2.0.1(react@18.2.0)(scheduler@0.23.2): dependencies: proxy-compare: 3.0.0 react: 18.2.0 - scheduler: 0.23.0 - use-context-selector: 2.0.0(react@18.2.0)(scheduler@0.23.0) + scheduler: 0.23.2 + use-context-selector: 2.0.0(react@18.2.0)(scheduler@0.23.2) react-transition-group@4.4.5(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: @@ -2130,7 +2133,7 @@ snapshots: dependencies: '@babel/runtime': 7.24.1 - scheduler@0.23.0: + scheduler@0.23.2: dependencies: loose-envify: 1.4.0 @@ -2158,10 +2161,10 @@ snapshots: undici-types@5.26.5: {} - use-context-selector@2.0.0(react@18.2.0)(scheduler@0.23.0): + use-context-selector@2.0.0(react@18.2.0)(scheduler@0.23.2): dependencies: react: 18.2.0 - scheduler: 0.23.0 + scheduler: 0.23.2 use-disposable@1.0.2(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: From 2fa0608dec128c2e9c6bf604e1a93ab9ef6a8bc1 Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Sun, 17 Nov 2024 14:05:51 +0800 Subject: [PATCH 21/43] refactor(frontend): Rewrite initialization logic --- app/page.tsx | 62 ++++++++++++++++++----------------------- store/useIndexStore.ts | 7 +++-- store/useLabelsStore.ts | 18 ++++++++++++ 3 files changed, 50 insertions(+), 37 deletions(-) create mode 100644 store/useLabelsStore.ts diff --git a/app/page.tsx b/app/page.tsx index cc21aba..77c2bce 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -36,7 +36,8 @@ import ExistingPane from "../components/editor/existing" import { useTrackedHistoryStore } from "../store/useHistoryStore" import { useTrackedTaskStore } from "../store/useTaskStore" import { useTrackedUserStore } from "../store/useUserStore" -import { useRouter } from "next/navigation" +import { useRouter, useSearchParams } from "next/navigation" +import { useTrackedLabelsStore } from "../store/useLabelsStore" enum Stage { None = 0, @@ -108,16 +109,18 @@ export default function Index() { const indexStore = useTrackedIndexStore() const [prevIndex, setPrevIndex] = useState(indexStore.index) const historyStore = useTrackedHistoryStore() + const [prevViewingRecord, setPrevViewingRecord] = useState(historyStore.viewingRecord) const taskStore = useTrackedTaskStore() const userStore = useTrackedUserStore() + const labelsStore = useTrackedLabelsStore() const router = useRouter() + const searchParams = useSearchParams() const [firstRange, setFirstRange] = useState<[number, number] | null>(null) const [rangeId, setRangeId] = useState(null) const [serverSelection, setServerSelection] = useState(null) const [userSelection, setUserSelection] = useState<[number, number] | null>(null) const [waiting, setWaiting] = useState(null) const [stage, setStage] = useState(Stage.None) - const [labels, setLabels] = useState<(string | object)[]>([]) const toasterId = useId("toaster") const { dispatchToast } = useToastController(toasterId) @@ -129,10 +132,24 @@ export default function Index() { setServerSelection(null) setStage(Stage.None) setUserSelection(null) - historyStore.setViewingRecord(null) window.getSelection()?.removeAllRanges() } + indexStore.fetchMax() + labelsStore.fetch() + + if (searchParams.has("sample")) { + const index = searchParams.get("sample") + const indexNumber = Number.parseInt(index) + if (!Number.isNaN(indexNumber) && indexNumber >= 0 && indexNumber <= indexStore.max && indexNumber !== prevIndex) { + setPrevIndex(indexNumber) + washHand() + indexStore.setIndex(indexNumber) + } + } + + taskStore.fetch(indexStore.index) + useEffect(() => { const access_token = localStorage.getItem("access_token") if (access_token == "" || access_token == null) { @@ -158,40 +175,15 @@ export default function Index() { } return }) + userStore.fetch() }, []) - getAllTasksLength().then((task) => { - indexStore.setMaxIndex(task.all - 1) - }) - userStore.fetch() - - console.log("Re-rendered") - - const url = new URL(window.location.href) - const index = url.searchParams.get("sample") - if (index !== null) { - const indexNumber = Number.parseInt(index) - if (!Number.isNaN(indexNumber) && indexNumber >= 0 && indexNumber <= indexStore.max && indexNumber !== prevIndex) { - setPrevIndex(indexNumber) - washHand() - indexStore.setIndex(indexNumber) - } - } - useEffect(() => { - getAllLabels().then((labels) => { - setLabels(labels) - }) - }, []) - - taskStore.fetch(indexStore.index) - - useEffect(() => { - historyStore.updateHistory(indexStore.index).then(() => { - }) + historyStore.updateHistory(indexStore.index) }, [taskStore.current]) - useEffect(() => { + if (historyStore.viewingRecord !== prevViewingRecord) { + setPrevViewingRecord(historyStore.viewingRecord) if (historyStore.viewingRecord === null || taskStore.current === null) { washHand() return @@ -199,7 +191,7 @@ export default function Index() { setFirstRange([historyStore.viewingRecord.source_start, historyStore.viewingRecord.source_end]) setRangeId("doc") setServerSelection([userSectionResponse(historyStore.viewingRecord.summary_start, historyStore.viewingRecord.summary_end, true)]) - }, [historyStore.viewingRecord]) + } useLayoutEffect(() => { const func = (event) => { @@ -310,7 +302,7 @@ export default function Index() { backgroundColor="#79c5fb" textColor="black" text={props.text.slice(slice[0], slice[1] + 1)} - labels={labels} + labels={labelsStore.labels} onLabel={async (label, note) => { if (firstRange === null || rangeId === null) { return Promise.resolve() @@ -369,7 +361,7 @@ export default function Index() { textColor={textColor} text={props.text.slice(slice[0], slice[1] + 1)} score={score} - labels={labels} + labels={labelsStore.labels} onLabel={async (label, note) => { if (firstRange === null || rangeId === null) { return Promise.resolve() diff --git a/store/useIndexStore.ts b/store/useIndexStore.ts index 11c1261..2e8b827 100644 --- a/store/useIndexStore.ts +++ b/store/useIndexStore.ts @@ -8,7 +8,7 @@ interface IndexStore { previous: () => void next: () => void setIndex: (index: number) => void - setMaxIndex: (max: number) => void + fetchMax: () => Promise } export const useIndexStore = create()((set) => ({ @@ -17,7 +17,10 @@ export const useIndexStore = create()((set) => ({ previous: () => set((state) => ({ index: state.index - 1})), next: () => set((state) => ({ index: state.index + 1})), setIndex: (index: number) => set({ index }), - setMaxIndex: (max: number) => set({ max }) + fetchMax: async () => { + const task = await getAllTasksLength() + set({ max: task.all - 1 }) + } })) export const useTrackedIndexStore = createTrackedSelector(useIndexStore) diff --git a/store/useLabelsStore.ts b/store/useLabelsStore.ts new file mode 100644 index 0000000..dc3175c --- /dev/null +++ b/store/useLabelsStore.ts @@ -0,0 +1,18 @@ +import { create } from "zustand" +import { getAllLabels } from "../utils/request" +import { createTrackedSelector } from "react-tracked" + +interface LabelsStore { + labels: (string | object)[], + fetch: () => Promise, +} + +export const useLabelsStore = create()((set) => ({ + labels: [], + fetch: async () => { + const labels = await getAllLabels() + set({ labels }) + }, +})) + +export const useTrackedLabelsStore = createTrackedSelector(useLabelsStore) From cd4f7c255b82c22a78b811c7597f1c17be4e9a40 Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Mon, 18 Nov 2024 21:48:54 +0800 Subject: [PATCH 22/43] chore(frontend): Use immer to improve codes --- package.json | 3 ++- pnpm-lock.yaml | 13 +++++++++++-- store/useUserStore.ts | 3 ++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 2dc4dce..f55fbf3 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "@fluentui/react-components": "^9.47.3", "@fluentui/react-icons": "^2.0.250", "allotment": "^1.20.2", + "immer": "^10.1.1", "js-sha512": "^0.9.0", "lodash": "^4.17.21", "next": "latest", @@ -29,5 +30,5 @@ "@types/react-dom": "18.2.14", "typescript": "^5.2.2" }, - "packageManager": "pnpm@9.12.3+sha512.cce0f9de9c5a7c95bef944169cc5dfe8741abfb145078c0d508b868056848a87c81e626246cb60967cbd7fd29a6c062ef73ff840d96b3c86c40ac92cf4a813ee" + "packageManager": "pnpm@9.13.2+sha512.88c9c3864450350e65a33587ab801acf946d7c814ed1134da4a924f6df5a2120fd36b46aab68f7cd1d413149112d53c7db3a4136624cfd00ff1846a0c6cef48a" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index be22ef9..ba8d082 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,6 +20,9 @@ importers: allotment: specifier: ^1.20.2 version: 1.20.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + immer: + specifier: ^10.1.1 + version: 10.1.1 js-sha512: specifier: ^0.9.0 version: 0.9.0 @@ -46,7 +49,7 @@ importers: version: 0.23.2 zustand: specifier: ^5.0.1 - version: 5.0.1(@types/react@18.2.33)(react@18.2.0) + version: 5.0.1(@types/react@18.2.33)(immer@10.1.1)(react@18.2.0) devDependencies: '@biomejs/biome': specifier: 1.6.4 @@ -722,6 +725,9 @@ packages: graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + immer@10.1.1: + resolution: {integrity: sha512-s2MPrmjovJcoMaHtx6K11Ra7oD05NT97w1IC5zpMkT6Atjr7H8LjaDd81iIxUYpMKSRRNMJE703M1Fhr/TctHw==} + js-sha512@0.9.0: resolution: {integrity: sha512-mirki9WS/SUahm+1TbAPkqvbCiCfOAAsyXeHxK1UkullnJVVqoJG2pL9ObvT05CN+tM7fxhfYm0NbXn+1hWoZg==} @@ -2030,6 +2036,8 @@ snapshots: graceful-fs@4.2.11: {} + immer@10.1.1: {} + js-sha512@0.9.0: {} js-tokens@4.0.0: {} @@ -2179,7 +2187,8 @@ snapshots: react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - zustand@5.0.1(@types/react@18.2.33)(react@18.2.0): + zustand@5.0.1(@types/react@18.2.33)(immer@10.1.1)(react@18.2.0): optionalDependencies: '@types/react': 18.2.33 + immer: 10.1.1 react: 18.2.0 diff --git a/store/useUserStore.ts b/store/useUserStore.ts index 1d6a4b7..3210732 100644 --- a/store/useUserStore.ts +++ b/store/useUserStore.ts @@ -2,6 +2,7 @@ import { create } from "zustand" import { User } from "../utils/types" import { getUserMe } from "../utils/request" import { createTrackedSelector } from "react-tracked" +import { produce } from "immer" interface UserStore { user: User @@ -15,7 +16,7 @@ export const useUserStore = create()((set) => ({ const user = await getUserMe() set({ user }) }, - setName: (name: string) => set((state) => ({ user: { ...state.user, name } })), + setName: (name: string) => set(produce((state: UserStore) => { state.user.name = name })), })) export const useTrackedUserStore = createTrackedSelector(useUserStore) \ No newline at end of file From 818e184b1b5c338f5bde9cb09dfc0cb27488e6f8 Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Mon, 18 Nov 2024 21:50:03 +0800 Subject: [PATCH 23/43] chore(frontend): Rename store types --- store/useHistoryStore.ts | 4 ++-- store/useIndexStore.ts | 4 ++-- store/useLabelsStore.ts | 4 ++-- store/useTaskStore.ts | 4 ++-- store/useUserStore.ts | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/store/useHistoryStore.ts b/store/useHistoryStore.ts index 2b6dba0..b587b62 100644 --- a/store/useHistoryStore.ts +++ b/store/useHistoryStore.ts @@ -3,7 +3,7 @@ import { create } from "zustand" import { getTaskHistory } from "../utils/request" import { createTrackedSelector } from "react-tracked" -interface HistoryStore { +interface HistoryState { history: LabelData[], viewingRecord: LabelData | null, setHistory: (history: LabelData[]) => void, @@ -11,7 +11,7 @@ interface HistoryStore { updateHistory: (labelIndex: number) => Promise, } -export const useHistoryStore = create()((set) => ({ +export const useHistoryStore = create()((set) => ({ history: [], viewingRecord: null, setHistory: (history: LabelData[]) => set({ history }), diff --git a/store/useIndexStore.ts b/store/useIndexStore.ts index 2e8b827..781fbd2 100644 --- a/store/useIndexStore.ts +++ b/store/useIndexStore.ts @@ -2,7 +2,7 @@ import { create } from "zustand" import { createTrackedSelector } from "react-tracked" import { getAllTasksLength } from "../utils/request" -interface IndexStore { +interface IndexState { index: number max: number previous: () => void @@ -11,7 +11,7 @@ interface IndexStore { fetchMax: () => Promise } -export const useIndexStore = create()((set) => ({ +export const useIndexStore = create()((set) => ({ index: 0, max: 0, previous: () => set((state) => ({ index: state.index - 1})), diff --git a/store/useLabelsStore.ts b/store/useLabelsStore.ts index dc3175c..bcff967 100644 --- a/store/useLabelsStore.ts +++ b/store/useLabelsStore.ts @@ -2,12 +2,12 @@ import { create } from "zustand" import { getAllLabels } from "../utils/request" import { createTrackedSelector } from "react-tracked" -interface LabelsStore { +interface LabelsState { labels: (string | object)[], fetch: () => Promise, } -export const useLabelsStore = create()((set) => ({ +export const useLabelsStore = create()((set) => ({ labels: [], fetch: async () => { const labels = await getAllLabels() diff --git a/store/useTaskStore.ts b/store/useTaskStore.ts index 1664f03..b7282c2 100644 --- a/store/useTaskStore.ts +++ b/store/useTaskStore.ts @@ -3,12 +3,12 @@ import { Task } from "../utils/types" import { getSingleTask } from "../utils/request" import { createTrackedSelector } from "react-tracked" -interface TaskStore { +interface TaskState { current: Task | null, fetch: (index: number) => Promise } -export const useTaskStore = create()((set) => ({ +export const useTaskStore = create()((set) => ({ current: null, fetch: async (index: number) => { try { diff --git a/store/useUserStore.ts b/store/useUserStore.ts index 3210732..db70a5d 100644 --- a/store/useUserStore.ts +++ b/store/useUserStore.ts @@ -4,19 +4,19 @@ import { getUserMe } from "../utils/request" import { createTrackedSelector } from "react-tracked" import { produce } from "immer" -interface UserStore { +interface UserState { user: User fetch: () => Promise setName: (name: string) => void } -export const useUserStore = create()((set) => ({ +export const useUserStore = create()((set) => ({ user: {} as User, fetch: async () => { const user = await getUserMe() set({ user }) }, - setName: (name: string) => set(produce((state: UserStore) => { state.user.name = name })), + setName: (name: string) => set(produce((state: UserState) => { state.user.name = name })), })) export const useTrackedUserStore = createTrackedSelector(useUserStore) \ No newline at end of file From 04d3d2d9895a28c1a5a5d1ac70b841f2b481dc56 Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Wed, 20 Nov 2024 07:49:52 +0800 Subject: [PATCH 24/43] docs: Update documentation for user_utils.py --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1776e79..1b2914a 100644 --- a/README.md +++ b/README.md @@ -83,8 +83,8 @@ Mercury uses [`sqlite-vec`](https://github.com/asg017/sqlite-vec) to store and s out. 6. `python3 server.py`. Be sure to set the candidate labels to choose from in the `server.py` file. -Admin who has access to the SQLite file can modify user data (e.g. reset user password) via `user_utils.py`. For more details, run -`python3 user_utils.py -h`. +Admin who has access to the SQLite file can modify user data (e.g. reset user password), register new users, and delete users +via `user_utils.py`. For more details, run `python3 user_utils.py -h`. The annotations are stored in the `annotations` table in a SQLite database (hardcoded name `mercury.sqlite`). See the section [`annotations` table](#annotations-table-the-human-annotations) for the schema. From ed55a2463e4bc64cf9f77481c5b74c2416d3689b Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Wed, 20 Nov 2024 12:51:19 +0800 Subject: [PATCH 25/43] Revert "feat(backend): Add update annotation endpoint" This reverts commit 89430b3285e1f293e56f75e449579631526d43d7. --- database.py | 13 ------------- server.py | 18 ------------------ 2 files changed, 31 deletions(-) diff --git a/database.py b/database.py index ced4984..8f8e012 100644 --- a/database.py +++ b/database.py @@ -354,19 +354,6 @@ def delete_annotation(self, record_id: str, annotator: str): self.mercury_db.execute(sql_cmd, (int(record_id), annotator)) self.mercury_db.commit() - @database_lock() - def update_annotation(self, label_data: OldLabelData, annotator: str): - record_id = label_data["record_id"] - sql_cmd = "UPDATE annotations SET annot_spans = ?, label = ?, note = ? WHERE annot_id = ? and annotator = ?" - self.mercury_db.execute(sql_cmd, ( - json.dumps(label_data["annot_spans"]), - label_data["label"], - label_data["note"], - record_id, - annotator, - )) - - @database_lock() def add_user(self, user_id: str, user_name: str): #TODO: remove this method since now only admin can add user sql_cmd = "INSERT INTO users (user_id, user_name) VALUES (?, ?)" diff --git a/server.py b/server.py index 4155349..8e080e2 100644 --- a/server.py +++ b/server.py @@ -326,24 +326,6 @@ async def delete_annotation(record_id: str, user: Annotated[User, Depends(get_u database.delete_annotation(record_id, user.id) return {"message": f"delete anntation {record_id} success"} -@app.patch("/record/{record_id}") -async def update_annotation(record_id: str, label: Label, user: Annotated[User, Depends(get_user)]): - annot_spans = {} - if label.summary_start != -1: - annot_spans["summary"] = (label.summary_start, label.summary_end) - if label.source_start != -1: - annot_spans["source"] = (label.source_start, label.source_end) - - label_string = json.dumps(label.consistent) - - database.update_annotation({ - "record_id": record_id, - "label": label_string, - "annot_spans": annot_spans, - "note": label.note - }, user.id) - return {"message": "success"} - @app.get("/labels") async def get_labels(): From 6fa4ce94658deffe86bde1cc6cebb5bf48e4a5c7 Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Wed, 20 Nov 2024 20:02:11 +0800 Subject: [PATCH 26/43] refactor(frontend): error handling and loading state --- app/page.tsx | 22 ++-- components/editor/existing.tsx | 215 +++++++++++++++++++-------------- components/editor/fallback.tsx | 30 +++++ components/labelPagination.tsx | 12 ++ store/useHistoryStore.ts | 3 +- store/useIndexStore.ts | 15 ++- store/useLabelsStore.ts | 9 +- store/useTaskStore.ts | 16 ++- store/useUserStore.ts | 9 +- utils/request.ts | 10 +- utils/types.ts | 10 +- 11 files changed, 229 insertions(+), 122 deletions(-) create mode 100644 components/editor/fallback.tsx diff --git a/app/page.tsx b/app/page.tsx index 77c2bce..102e922 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -135,8 +135,12 @@ export default function Index() { window.getSelection()?.removeAllRanges() } - indexStore.fetchMax() - labelsStore.fetch() + indexStore.fetchMax().catch(e => { + console.log(e) + }) + labelsStore.fetch().catch(e => { + console.log(e) + }) if (searchParams.has("sample")) { const index = searchParams.get("sample") @@ -148,7 +152,9 @@ export default function Index() { } } - taskStore.fetch(indexStore.index) + taskStore.fetch(indexStore.index).catch(error => { + console.log(error) + }) useEffect(() => { const access_token = localStorage.getItem("access_token") @@ -175,13 +181,11 @@ export default function Index() { } return }) - userStore.fetch() + userStore.fetch().catch(e => { + console.log(e) + }) }, []) - useEffect(() => { - historyStore.updateHistory(indexStore.index) - }, [taskStore.current]) - if (historyStore.viewingRecord !== prevViewingRecord) { setPrevViewingRecord(historyStore.viewingRecord) if (historyStore.viewingRecord === null || taskStore.current === null) { @@ -587,7 +591,7 @@ export default function Index() { - + diff --git a/components/editor/existing.tsx b/components/editor/existing.tsx index 774177d..d4c216e 100644 --- a/components/editor/existing.tsx +++ b/components/editor/existing.tsx @@ -22,7 +22,9 @@ import { deleteRecord } from "../../utils/request" import { useTrackedHistoryStore } from "../../store/useHistoryStore" import { useTrackedTaskStore } from "../../store/useTaskStore" import { LabelData } from "../../utils/types" -import { useMemo, useState } from "react" +import { useCallback, useEffect, useMemo, useState } from "react" +import { useTrackedIndexStore } from "../../store/useIndexStore" +import { HasError, Loading } from "./fallback" const columnsDef: TableColumnDefinition[] = [ createTableColumn({ @@ -48,13 +50,41 @@ const columnsDef: TableColumnDefinition[] = [ ] type Props = { - onRefresh?: Function, + onRestore?: Function, } -export default function ExistingPane({ onRefresh = Function() }: Props) { +export default function ExistingPane({ onRestore = Function() }: Props) { const historyStore = useTrackedHistoryStore() + const indexStore = useTrackedIndexStore() const taskStore = useTrackedTaskStore() + const [isLoading, setIsLoading] = useState(true) + const [hasError, setHasError] = useState(false) + + const onRefreshHistory = useCallback(async () => { + setHasError(false) + setIsLoading(true) + try { + await historyStore.updateHistory(indexStore.index) + } + catch (e) { + setHasError(true) + } + setIsLoading(false) + }, [indexStore.index]) + + useEffect(() => { + let ignore = false + + if (!ignore) { + onRefreshHistory() + } + + return () => { + ignore = true + } + }, [indexStore.index]) + const sortedHistory = useMemo(() => { return historyStore.history .sort((a, b) => { @@ -105,105 +135,108 @@ export default function ExistingPane({ onRefresh = Function() }: Props) { Existing annotations } /> - {/*@ts-ignore*/} - - - - {columns.map((column) => ( - - - } + {hasError && } + {!isLoading && !hasError && taskStore.current && ( + // @ts-ignore +
+ + + {columns.map((column) => ( + + + + {column.renderHeaderCell()} + + + + + + Keyboard Column Resizing + + + + + ))} + + + + {rows.map(({ item }) => ( + + - {column.renderHeaderCell()} - - - - - - Keyboard Column Resizing - - - - - ))} - - - - {rows.map(({ item }) => ( - - - {taskStore.current.doc.slice(item.source_start, item.source_end)} - - - {taskStore.current.sum.slice(item.summary_start, item.summary_end)} - - - {item.consistent.join(", ")} - - - {item.note} - - - {historyStore.viewingRecord != null && historyStore.viewingRecord.record_id === item.record_id ? ( - + ) : ( + - ) : ( + historyStore.setViewingRecord(item) + }} + > + Show + + )} - )} - - - - ))} - -
+ + + ))} + + + )} ) diff --git a/components/editor/fallback.tsx b/components/editor/fallback.tsx new file mode 100644 index 0000000..a646b35 --- /dev/null +++ b/components/editor/fallback.tsx @@ -0,0 +1,30 @@ +"use client" + +import { Button } from "@fluentui/react-components" + +export function Loading() { + return ( +

+ Loading... +

+ ) +} + +type hasErrorProps = { + onRetry?: Function +} + +export function HasError({ onRetry = null }: hasErrorProps) { + return ( +
+

+ Error loading data. + { onRetry != null && + + } +

+
+ ) +} diff --git a/components/labelPagination.tsx b/components/labelPagination.tsx index b5d650c..25efe98 100644 --- a/components/labelPagination.tsx +++ b/components/labelPagination.tsx @@ -3,6 +3,9 @@ import { Button, Field, ProgressBar } from "@fluentui/react-components" import { ChevronLeftRegular, IosChevronRightRegular } from "@fluentui/react-icons" import { useTrackedIndexStore } from "../store/useIndexStore" +import { useCallback } from "react" +import { useTrackedHistoryStore } from "../store/useHistoryStore" +import { useTrackedEditorStore } from "../store/useEditorStore" type Props = { beforeChangeIndex?: Function, @@ -10,6 +13,13 @@ type Props = { export default function LabelPagination({ beforeChangeIndex = Function() }: Props) { const indexStore = useTrackedIndexStore() + const editorStore = useTrackedEditorStore() + const historyStore = useTrackedHistoryStore() + + const onReset = useCallback(() => { + editorStore.clearSelection() + historyStore.setViewingRecord(null) + }, []) return (
{ beforeChangeIndex() + onReset() indexStore.previous() }} > @@ -45,6 +56,7 @@ export default function LabelPagination({ beforeChangeIndex = Function() }: Prop iconPosition="after" onClick={() => { beforeChangeIndex() + onReset() indexStore.next() }} > diff --git a/store/useHistoryStore.ts b/store/useHistoryStore.ts index b587b62..959d933 100644 --- a/store/useHistoryStore.ts +++ b/store/useHistoryStore.ts @@ -22,7 +22,8 @@ export const useHistoryStore = create()((set) => ({ set({ history, viewingRecord: null }) } catch (e) { set({ history: [], viewingRecord: null }) - console.error(e) + console.log(e) + throw e } }, })) diff --git a/store/useIndexStore.ts b/store/useIndexStore.ts index 781fbd2..0cdfaf6 100644 --- a/store/useIndexStore.ts +++ b/store/useIndexStore.ts @@ -14,13 +14,18 @@ interface IndexState { export const useIndexStore = create()((set) => ({ index: 0, max: 0, - previous: () => set((state) => ({ index: state.index - 1})), - next: () => set((state) => ({ index: state.index + 1})), + previous: () => set((state) => ({ index: state.index - 1 })), + next: () => set((state) => ({ index: state.index + 1 })), setIndex: (index: number) => set({ index }), fetchMax: async () => { - const task = await getAllTasksLength() - set({ max: task.all - 1 }) - } + try { + const task = await getAllTasksLength() + set({ max: task.all - 1 }) + } catch (e) { + console.log(e) + throw e + } + }, })) export const useTrackedIndexStore = createTrackedSelector(useIndexStore) diff --git a/store/useLabelsStore.ts b/store/useLabelsStore.ts index bcff967..4152159 100644 --- a/store/useLabelsStore.ts +++ b/store/useLabelsStore.ts @@ -10,8 +10,13 @@ interface LabelsState { export const useLabelsStore = create()((set) => ({ labels: [], fetch: async () => { - const labels = await getAllLabels() - set({ labels }) + try { + const labels = await getAllLabels() + set({ labels }) + } catch (e) { + console.log(e) + throw e + } }, })) diff --git a/store/useTaskStore.ts b/store/useTaskStore.ts index b7282c2..7cf6208 100644 --- a/store/useTaskStore.ts +++ b/store/useTaskStore.ts @@ -1,5 +1,5 @@ import { create } from "zustand" -import { Task } from "../utils/types" +import { handleRequestError, isRequestError, Task } from "../utils/types" import { getSingleTask } from "../utils/request" import { createTrackedSelector } from "react-tracked" @@ -13,14 +13,18 @@ export const useTaskStore = create()((set) => ({ fetch: async (index: number) => { try { const task = await getSingleTask(index) - if ("doc" in task) { - set({ current: task }) + if (isRequestError(task)) { + handleRequestError(task) + return } - } catch (e) { + set({ current: task }) + } + catch (e) { set({ current: null }) - console.error(e) + console.log(e) + throw e } }, })) -export const useTrackedTaskStore= createTrackedSelector(useTaskStore) \ No newline at end of file +export const useTrackedTaskStore = createTrackedSelector(useTaskStore) \ No newline at end of file diff --git a/store/useUserStore.ts b/store/useUserStore.ts index db70a5d..4a5f759 100644 --- a/store/useUserStore.ts +++ b/store/useUserStore.ts @@ -13,8 +13,13 @@ interface UserState { export const useUserStore = create()((set) => ({ user: {} as User, fetch: async () => { - const user = await getUserMe() - set({ user }) + try { + const user = await getUserMe() + set({ user }) + } catch (e) { + console.log(e) + throw e + } }, setName: (name: string) => set(produce((state: UserState) => { state.user.name = name })), })) diff --git a/utils/request.ts b/utils/request.ts index e644cfe..64d5162 100644 --- a/utils/request.ts +++ b/utils/request.ts @@ -2,7 +2,7 @@ import type { AllTasksLength, LabelData, LabelRequest, - Normal, + Normal, RequestError, SectionResponse, SelectionRequest, Task, @@ -96,13 +96,13 @@ const getAllTasksLength = async (): Promise => { return data as AllTasksLength } -const getSingleTask = async (taskIndex: number): Promise => { +const getSingleTask = async (taskIndex: number): Promise => { const response = await fetch(`${backend}/task/${taskIndex}`) const data = await response.json() - return data as Task | Error + return data as Task | RequestError } -const selectText = async (taskIndex: number, req: SelectionRequest): Promise => { +const selectText = async (taskIndex: number, req: SelectionRequest): Promise => { const response = await fetch(`${backend}/task/${taskIndex}/select`, { method: "POST", headers: { @@ -111,7 +111,7 @@ const selectText = async (taskIndex: number, req: SelectionRequest): Promise => { diff --git a/utils/types.ts b/utils/types.ts index b323726..8276633 100644 --- a/utils/types.ts +++ b/utils/types.ts @@ -40,10 +40,18 @@ export function userSectionResponse(start: number, end: number, toDoc: boolean): } } -export type Error = { +export type RequestError = { error: string } +export function isRequestError(obj: any): obj is RequestError { + return typeof obj.error === "string"; +} + +export function handleRequestError(e: RequestError) { + throw Error(e.error) +} + export type Normal = { message: string } From ef0af1a22fd50c6db12188db963ce5f8126355eb Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Wed, 20 Nov 2024 22:20:31 +0800 Subject: [PATCH 27/43] chore: Remove redundant code --- app/layout.tsx | 18 +++++++++--------- app/page.css | 12 ------------ app/page.tsx | 1 - 3 files changed, 9 insertions(+), 22 deletions(-) delete mode 100644 app/page.css diff --git a/app/layout.tsx b/app/layout.tsx index 58cb243..852ba5a 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -3,7 +3,7 @@ import { FluentProvider, webLightTheme } from "@fluentui/react-components" export default function RootLayout({ children }) { return ( - + Mercury - -
{children}
- + +
{children}
+ - + ) } diff --git a/app/page.css b/app/page.css deleted file mode 100644 index fbd846f..0000000 --- a/app/page.css +++ /dev/null @@ -1,12 +0,0 @@ -.column_resize_table th::before { - content: ""; - display: block; - width: var(--column_resize_before_width); -} - -.columnResizer:hover { - border-right: 1px solid #777; - border-left: 1px solid #777; - /* background-color: #f9f9f9; */ - background-color: #777; -} diff --git a/app/page.tsx b/app/page.tsx index 102e922..193a467 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -28,7 +28,6 @@ import { } from "@fluentui/react-icons" import { Allotment } from "allotment" import "allotment/dist/style.css" -import "./page.css" import UserPopover from "../components/userPopover" import LabelPagination from "../components/labelPagination" import { useTrackedIndexStore } from "../store/useIndexStore" From 4d564ee91c23d4d997f2e8fc75e704b422acd281 Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Wed, 20 Nov 2024 22:22:07 +0800 Subject: [PATCH 28/43] refactor: Rewrite index page's basic features --- app/new/page.tsx | 150 +++++++++++++++++++++++++++++ components/editor/controls.tsx | 65 +++++++++++++ components/editor/editor.tsx | 169 +++++++++++++++++++++++++++++++++ store/useEditorStore.ts | 56 +++++++++++ 4 files changed, 440 insertions(+) create mode 100644 app/new/page.tsx create mode 100644 components/editor/controls.tsx create mode 100644 components/editor/editor.tsx create mode 100644 store/useEditorStore.ts diff --git a/app/new/page.tsx b/app/new/page.tsx new file mode 100644 index 0000000..2ad49cb --- /dev/null +++ b/app/new/page.tsx @@ -0,0 +1,150 @@ +"use client" + +import { + Button, + Title1, + Toast, + Toaster, + ToastTitle, + ToastTrigger, + useId, + useToastController, +} from "@fluentui/react-components" +import Controls from "../../components/editor/controls" +import LabelPagination from "../../components/labelPagination" +import Editor from "../../components/editor/editor" +import { useEffect, useState } from "react" +import { useTrackedIndexStore } from "../../store/useIndexStore" +import { useTrackedLabelsStore } from "../../store/useLabelsStore" +import { useRouter, useSearchParams } from "next/navigation" +import { checkUserMe } from "../../utils/request" +import { useTrackedUserStore } from "../../store/useUserStore" + +let didInit = false + +export default function New() { + const indexStore = useTrackedIndexStore() + const labelsStore = useTrackedLabelsStore() + const userStore = useTrackedUserStore() + + const router = useRouter() + const searchParams = useSearchParams() + + const [prevIndex, setPrevIndex] = useState(indexStore.index) + + const toasterId = useId("toaster") + const { dispatchToast } = useToastController(toasterId) + + useEffect(() => { + if (!didInit) { + didInit = true + + indexStore.fetchMax().catch(e => { + console.log(e) + dispatchToast( + + + + + } + > + Fail loading index + + , + { intent: "error" }, + ) + }) + + labelsStore.fetch().catch(e => { + console.log(e) + dispatchToast( + + + + + } + > + Fail loading labels + + , + { intent: "error" }, + ) + }) + + if (searchParams.has("sample")) { + const index = searchParams.get("sample") + const indexNumber = Number.parseInt(index) + if (!Number.isNaN(indexNumber) && indexNumber >= 0 && indexNumber <= indexStore.max && indexNumber !== prevIndex) { + setPrevIndex(indexNumber) + indexStore.setIndex(indexNumber) + } + } + + const access_token = localStorage.getItem("access_token") + if (access_token == "" || access_token == null) { + dispatchToast( + + Not logged in + , + { intent: "error" }, + ) + router.push("/login") + return + } + checkUserMe(access_token).then(valid => { + if (!valid) { + localStorage.removeItem("access_token") + dispatchToast( + + Session expired + , + { intent: "error" }, + ) + router.push("/login") + } + return + }) + userStore.fetch().catch(e => { + console.log(e) + dispatchToast( + + + + + } + > + Fail loading user data + + , + { intent: "error" }, + ) + }) + } + }, []) + + return ( + <> + + Mercury Label +
+
+ +
+ +
+ + + ) +} diff --git a/components/editor/controls.tsx b/components/editor/controls.tsx new file mode 100644 index 0000000..187fa28 --- /dev/null +++ b/components/editor/controls.tsx @@ -0,0 +1,65 @@ +"use client" + +import { exportLabel } from "../../utils/request" +import { Button } from "@fluentui/react-components" +import { ArrowExportRegular, HandRightRegular, ShareRegular } from "@fluentui/react-icons" +import UserPopover from "../userPopover" +import { useTrackedIndexStore } from "../../store/useIndexStore" +import { useTrackedEditorStore } from "../../store/useEditorStore" +import { useCallback } from "react" +import { useTrackedHistoryStore } from "../../store/useHistoryStore" + +export default function Controls() { + const indexStore = useTrackedIndexStore() + const editorStore = useTrackedEditorStore() + const historyStore = useTrackedHistoryStore() + + const onReset = useCallback(() => { + editorStore.clearSelection() + historyStore.setViewingRecord(null) + }, []) + + const onExportJSON = useCallback(async () => { + try { + const data = await exportLabel() + const blob = new Blob([JSON.stringify(data, null, 2)], { type: "application/json" }) + const downloadLink = document.createElement("a") + const downloadUrl = URL.createObjectURL(blob) + downloadLink.href = downloadUrl + downloadLink.download = "label.json" + downloadLink.click() + URL.revokeObjectURL(downloadUrl) + } + catch (error) { + console.error("Failed to export JSON:", error) + } + }, []) + + return ( +
+ + + + +
+ ) +} \ No newline at end of file diff --git a/components/editor/editor.tsx b/components/editor/editor.tsx new file mode 100644 index 0000000..cd6aedf --- /dev/null +++ b/components/editor/editor.tsx @@ -0,0 +1,169 @@ +"use client" + +import { useTrackedEditorStore } from "../../store/useEditorStore" +import { useTrackedTaskStore } from "../../store/useTaskStore" +import { useCallback, useEffect, useState } from "react" +import { Allotment } from "allotment" +import "allotment/dist/style.css" +import { Body1, Card, CardHeader, Text } from "@fluentui/react-components" +import ExistingPane from "./existing" +import { useTrackedIndexStore } from "../../store/useIndexStore" +import { useTrackedHistoryStore } from "../../store/useHistoryStore" +import { HasError, Loading } from "./fallback" + +export default function Editor() { + const editorStore = useTrackedEditorStore() + const taskStore = useTrackedTaskStore() + const indexStore = useTrackedIndexStore() + const historyStore = useTrackedHistoryStore() + + const [suspendSource, setSuspendSource] = useState(false) + const [suspendSummary, setSuspendSummary] = useState(false) + const [isLoading, setIsLoading] = useState(true) + const [hasError, setHasError] = useState(false) + + const handleMouseUp = useCallback((element: HTMLSpanElement) => { + const selection = window.getSelection() + if (!selection || selection.rangeCount <= 0) return + if (!selection.containsNode(element, true)) return + + const range = selection.getRangeAt(0) + // if (!element.contains(range.commonAncestorContainer)) return + if (element.id == "source") { + editorStore.setSourceSelection(range.startOffset, range.endOffset) + } + else if (element.id == "summary") { + editorStore.setSummarySelection(range.startOffset, range.endOffset) + } + }, []) + + const onRestoreViewingHistory = useCallback(() => { + editorStore.clearSelection() + historyStore.setViewingRecord(null) + }, []) + + const onFetchTask = useCallback(async () => { + setHasError(false) + setIsLoading(true) + try { + await taskStore.fetch(indexStore.index) + } + catch (e) { + setHasError(true) + } + setIsLoading(false) + }, [indexStore.index]) + + useEffect(() => { + let ignore = false + + if (!ignore) { + onFetchTask() + } + + return () => { + ignore = true + } + }, [indexStore.index]) + + return ( +
+ + +
+ + + Source + + } + /> + {isLoading && } + {hasError && } + {!isLoading && !hasError && taskStore.current && ( + suspendSource ? + + {taskStore.current.doc} + + : + { + handleMouseUp(event.target as HTMLSpanElement) + }} + > + {taskStore.current.doc} + + )} + +
+ + + +
+ + + Summary + + } + /> + {isLoading && } + {hasError && } + {!isLoading && !hasError && taskStore.current && ( + suspendSummary ? + + {taskStore.current.sum} + + : + { + handleMouseUp(event.target as HTMLSpanElement) + }} + > + {taskStore.current.sum} + + )} + +
+ + + + +
+ ) +} diff --git a/store/useEditorStore.ts b/store/useEditorStore.ts new file mode 100644 index 0000000..885301b --- /dev/null +++ b/store/useEditorStore.ts @@ -0,0 +1,56 @@ +import { create } from "zustand" +import { createTrackedSelector } from "react-tracked" +import { handleRequestError, isRequestError, SectionResponse, SelectionRequest } from "../utils/types" +import { produce } from "immer" +import { selectText } from "../utils/request" + +interface EditorState { + sourceSelection: SelectionRequest | null + summarySelection: SelectionRequest | null + initiator: "source" | "summary" | null + serverSection: SectionResponse + setSourceSelection: (start: number, end: number) => void + setSummarySelection: (start: number, end: number) => void + clearSelection: () => void + fetchServerSection: (index: number) => Promise +} + +export const useEditorStore = create()((set, get) => ({ + sourceSelection: null, + summarySelection: null, + initiator: null, + serverSection: [], + setSourceSelection: (start: number, end: number) => set(produce((state: EditorState) => { + state.sourceSelection = { start, end, from_summary: false } + if (state.initiator === null) state.initiator = "source" + })), + setSummarySelection: (start: number, end: number) => set(produce((state: EditorState) => { + state.summarySelection = { start, end, from_summary: true } + if (state.initiator === null) state.initiator = "summary" + })), + clearSelection: () => set(produce((state: EditorState) => { + state.sourceSelection = null + state.summarySelection = null + state.serverSection = [] + state.initiator = null + window.getSelection()?.removeAllRanges() + })), + fetchServerSection: async (index: number) => { + if (get().initiator == "source") { + try { + const response = await selectText(index, get().sourceSelection) + if (isRequestError(response)) { + handleRequestError(response) + return + } + set({ serverSection: response }) + } + catch (e) { + console.log(e) + throw e + } + } + }, +})) + +export const useTrackedEditorStore = createTrackedSelector(useEditorStore) From 8d755929be92ef95a729c4adf1096ce50dafde44 Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Wed, 20 Nov 2024 22:27:51 +0800 Subject: [PATCH 29/43] style(fallback): Update Loading component's style --- components/editor/fallback.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/editor/fallback.tsx b/components/editor/fallback.tsx index a646b35..8a8f63e 100644 --- a/components/editor/fallback.tsx +++ b/components/editor/fallback.tsx @@ -19,12 +19,12 @@ export function HasError({ onRetry = null }: hasErrorProps) {

Error loading data. - { onRetry != null && +

+ {onRetry != null && - } -

+ }
) } From c4a1340b83cd513cb22ad853634e12c71f349dc7 Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Thu, 21 Nov 2024 08:49:16 +0800 Subject: [PATCH 30/43] feat(frontend): Optimize loading state --- components/editor/editor.tsx | 105 +++++++++++++++++---------------- components/editor/existing.tsx | 7 ++- 2 files changed, 59 insertions(+), 53 deletions(-) diff --git a/components/editor/editor.tsx b/components/editor/editor.tsx index cd6aedf..d36eaac 100644 --- a/components/editor/editor.tsx +++ b/components/editor/editor.tsx @@ -10,6 +10,7 @@ import ExistingPane from "./existing" import { useTrackedIndexStore } from "../../store/useIndexStore" import { useTrackedHistoryStore } from "../../store/useHistoryStore" import { HasError, Loading } from "./fallback" +import _ from "lodash" export default function Editor() { const editorStore = useTrackedEditorStore() @@ -22,6 +23,8 @@ export default function Editor() { const [isLoading, setIsLoading] = useState(true) const [hasError, setHasError] = useState(false) + const debounceSetIsLoading = _.debounce(setIsLoading, 500) + const handleMouseUp = useCallback((element: HTMLSpanElement) => { const selection = window.getSelection() if (!selection || selection.rangeCount <= 0) return @@ -44,14 +47,14 @@ export default function Editor() { const onFetchTask = useCallback(async () => { setHasError(false) - setIsLoading(true) + debounceSetIsLoading(true) try { await taskStore.fetch(indexStore.index) } catch (e) { setHasError(true) } - setIsLoading(false) + debounceSetIsLoading(false) }, [indexStore.index]) useEffect(() => { @@ -73,74 +76,73 @@ export default function Editor() { margin: "auto", }} > - - -
} + {hasError && } + {!isLoading && !hasError && taskStore.current && + + +
- + - + Source } - /> - {isLoading && } - {hasError && } - {!isLoading && !hasError && taskStore.current && ( - suspendSource ? - - {taskStore.current.doc} - - : - { - handleMouseUp(event.target as HTMLSpanElement) - }} - > - {taskStore.current.doc} - - )} - -
- - - -
+ {( + suspendSource ? + + {taskStore.current.doc} + + : + { + handleMouseUp(event.target as HTMLSpanElement) + }} + > + {taskStore.current.doc} + + )} + +
+ + + +
- + - + Summary } - /> - {isLoading && } - {hasError && } - {!isLoading && !hasError && taskStore.current && ( + /> + { suspendSummary ? {taskStore.current.sum} - )} - -
- - - - + } + +
+ + + + + }
) } diff --git a/components/editor/existing.tsx b/components/editor/existing.tsx index d4c216e..39ff154 100644 --- a/components/editor/existing.tsx +++ b/components/editor/existing.tsx @@ -25,6 +25,7 @@ import { LabelData } from "../../utils/types" import { useCallback, useEffect, useMemo, useState } from "react" import { useTrackedIndexStore } from "../../store/useIndexStore" import { HasError, Loading } from "./fallback" +import _ from "lodash" const columnsDef: TableColumnDefinition[] = [ createTableColumn({ @@ -61,16 +62,18 @@ export default function ExistingPane({ onRestore = Function() }: Props) { const [isLoading, setIsLoading] = useState(true) const [hasError, setHasError] = useState(false) + const debounceSetIsLoading = _.debounce(setIsLoading, 500) + const onRefreshHistory = useCallback(async () => { setHasError(false) - setIsLoading(true) + debounceSetIsLoading(true) try { await historyStore.updateHistory(indexStore.index) } catch (e) { setHasError(true) } - setIsLoading(false) + debounceSetIsLoading(false) }, [indexStore.index]) useEffect(() => { From ff7cf094c5bc1080f9fcd274c028b708a4e84174 Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Thu, 21 Nov 2024 22:45:09 +0800 Subject: [PATCH 31/43] feat(frontend): Implement user select for rewrited editor --- components/editor/controls.tsx | 2 +- components/editor/editor.tsx | 127 ++++++++++++++++++++++++++++++--- components/labelPagination.tsx | 2 +- package.json | 2 + pnpm-lock.yaml | 16 +++++ store/useEditorStore.ts | 38 ++++++---- 6 files changed, 162 insertions(+), 25 deletions(-) diff --git a/components/editor/controls.tsx b/components/editor/controls.tsx index 187fa28..20e8925 100644 --- a/components/editor/controls.tsx +++ b/components/editor/controls.tsx @@ -15,7 +15,7 @@ export default function Controls() { const historyStore = useTrackedHistoryStore() const onReset = useCallback(() => { - editorStore.clearSelection() + editorStore.clearAllSelection() historyStore.setViewingRecord(null) }, []) diff --git a/components/editor/editor.tsx b/components/editor/editor.tsx index d36eaac..fd54d94 100644 --- a/components/editor/editor.tsx +++ b/components/editor/editor.tsx @@ -11,12 +11,18 @@ import { useTrackedIndexStore } from "../../store/useIndexStore" import { useTrackedHistoryStore } from "../../store/useHistoryStore" import { HasError, Loading } from "./fallback" import _ from "lodash" +import { labelText } from "../../utils/request" +import Tooltip from "../tooltip" +import { useTrackedLabelsStore } from "../../store/useLabelsStore" +import rangy from "rangy" +import "rangy/lib/rangy-textrange" export default function Editor() { const editorStore = useTrackedEditorStore() const taskStore = useTrackedTaskStore() const indexStore = useTrackedIndexStore() const historyStore = useTrackedHistoryStore() + const labelsStore = useTrackedLabelsStore() const [suspendSource, setSuspendSource] = useState(false) const [suspendSummary, setSuspendSummary] = useState(false) @@ -25,23 +31,33 @@ export default function Editor() { const debounceSetIsLoading = _.debounce(setIsLoading, 500) - const handleMouseUp = useCallback((element: HTMLSpanElement) => { - const selection = window.getSelection() + const handleMouseUp = useCallback((element: HTMLElement) => { + const selection = rangy.getSelection() if (!selection || selection.rangeCount <= 0) return - if (!selection.containsNode(element, true)) return - const range = selection.getRangeAt(0) - // if (!element.contains(range.commonAncestorContainer)) return + + if (range.toString().trim() == "") { + if (element.id == "source") { + editorStore.clearSourceSelection() + } + else if (element.id == "summary") { + editorStore.clearSummarySelection() + } + return + } + + const { start, end } = range.toCharacterRange(element) + if (element.id == "source") { - editorStore.setSourceSelection(range.startOffset, range.endOffset) + editorStore.setSourceSelection(start, end) } else if (element.id == "summary") { - editorStore.setSummarySelection(range.startOffset, range.endOffset) + editorStore.setSummarySelection(start, end) } }, []) const onRestoreViewingHistory = useCallback(() => { - editorStore.clearSelection() + editorStore.clearAllSelection() historyStore.setViewingRecord(null) }, []) @@ -69,6 +85,91 @@ export default function Editor() { } }, [indexStore.index]) + function renderHighlight(target: "source" | "summary") { + if (target == "source") { + if (editorStore.sourceSelection.start == -1) return taskStore.current.doc + + const segments = [] + + if (editorStore.sourceSelection.start > 0) { + segments.push(taskStore.current.doc.slice(0, editorStore.sourceSelection.start)) + } + + segments.push( + { + await labelText(indexStore.index, { + source_start: editorStore.sourceSelection.start, + source_end: editorStore.sourceSelection.end, + summary_start: editorStore.summarySelection.start, + summary_end: editorStore.summarySelection.end, + consistent: label, + note: note, + }) + historyStore.updateHistory(indexStore.index).then(() => { + }) + }} + message="Check all types that apply below." + />, + ) + + if (editorStore.sourceSelection.end < taskStore.current.doc.length) { + segments.push(taskStore.current.doc.slice(editorStore.sourceSelection.end)) + } + + return segments + } + else { + if (editorStore.summarySelection.start == -1) return taskStore.current.sum + + const segments = [] + + if (editorStore.summarySelection.start > 0) { + segments.push(taskStore.current.sum.slice(0, editorStore.summarySelection.start)) + } + + segments.push( + { + await labelText(indexStore.index, { + source_start: editorStore.summarySelection.start, + source_end: editorStore.summarySelection.end, + summary_start: editorStore.summarySelection.start, + summary_end: editorStore.summarySelection.end, + consistent: label, + note: note, + }) + historyStore.updateHistory(indexStore.index).then(() => { + }) + }} + message="Check all types that apply below." + />, + ) + + if (editorStore.summarySelection.end < taskStore.current.sum.length) { + segments.push(taskStore.current.sum.slice(editorStore.summarySelection.end)) + } + + return segments + } + } + return (
{ handleMouseUp(event.target as HTMLSpanElement) }} > - {taskStore.current.doc} + {renderHighlight("source")} )} @@ -153,11 +257,14 @@ export default function Editor() { { handleMouseUp(event.target as HTMLSpanElement) }} > - {taskStore.current.sum} + {renderHighlight("summary")} } diff --git a/components/labelPagination.tsx b/components/labelPagination.tsx index 25efe98..7f69ba2 100644 --- a/components/labelPagination.tsx +++ b/components/labelPagination.tsx @@ -17,7 +17,7 @@ export default function LabelPagination({ beforeChangeIndex = Function() }: Prop const historyStore = useTrackedHistoryStore() const onReset = useCallback(() => { - editorStore.clearSelection() + editorStore.clearAllSelection() historyStore.setViewingRecord(null) }, []) diff --git a/package.json b/package.json index f55fbf3..e856e8a 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "js-sha512": "^0.9.0", "lodash": "^4.17.21", "next": "latest", + "rangy": "^1.3.2", "react": "^18.2.0", "react-dom": "^18.2.0", "react-table-column-resizer": "^1.2.3", @@ -26,6 +27,7 @@ "@biomejs/biome": "1.6.4", "@types/lodash": "^4.17.0", "@types/node": "20.8.10", + "@types/rangy": "^0.0.38", "@types/react": "18.2.33", "@types/react-dom": "18.2.14", "typescript": "^5.2.2" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ba8d082..4fc704e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -32,6 +32,9 @@ importers: next: specifier: latest version: 14.1.4(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + rangy: + specifier: ^1.3.2 + version: 1.3.2 react: specifier: ^18.2.0 version: 18.2.0 @@ -60,6 +63,9 @@ importers: '@types/node': specifier: 20.8.10 version: 20.8.10 + '@types/rangy': + specifier: ^0.0.38 + version: 0.0.38 '@types/react': specifier: 18.2.33 version: 18.2.33 @@ -685,6 +691,9 @@ packages: '@types/prop-types@15.7.12': resolution: {integrity: sha512-5zvhXYtRNRluoE/jAp4GVsSduVUzNWKkOZrCDBWYtE7biZywwdC2AcEzg+cSMLFRfVgeAFqpfNabiPjxFddV1Q==} + '@types/rangy@0.0.38': + resolution: {integrity: sha512-KMybA3NQLSc5Fl5VOyLVSZ10AMSY6anQqLxP8dxgNsui3dScPIB7smywq69gwJkud2ODVTIImMNN3RtHcFKYgA==} + '@types/react-dom@18.2.14': resolution: {integrity: sha512-V835xgdSVmyQmI1KLV2BEIUgqEuinxp9O4G6g3FqO/SqLac049E53aysv0oEFD2kHfejeKU+ZqL2bcFWj9gLAQ==} @@ -790,6 +799,9 @@ packages: proxy-compare@3.0.0: resolution: {integrity: sha512-y44MCkgtZUCT9tZGuE278fB7PWVf7fRYy0vbRXAts2o5F0EfC4fIQrvQQGBJo1WJbFcVLXzApOscyJuZqHQc1w==} + rangy@1.3.2: + resolution: {integrity: sha512-fS1C4MOyk8T+ZJZdLcgrukPWxkyDXa+Hd2Kj+Zg4wIK71yrWgmjzHubzPMY1G+WD9EgGxMp3fIL0zQ1ickmSWA==} + react-dom@18.2.0: resolution: {integrity: sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==} peerDependencies: @@ -1992,6 +2004,8 @@ snapshots: '@types/prop-types@15.7.12': {} + '@types/rangy@0.0.38': {} + '@types/react-dom@18.2.14': dependencies: '@types/react': 18.2.33 @@ -2101,6 +2115,8 @@ snapshots: proxy-compare@3.0.0: {} + rangy@1.3.2: {} + react-dom@18.2.0(react@18.2.0): dependencies: loose-envify: 1.4.0 diff --git a/store/useEditorStore.ts b/store/useEditorStore.ts index 885301b..00c5133 100644 --- a/store/useEditorStore.ts +++ b/store/useEditorStore.ts @@ -5,19 +5,21 @@ import { produce } from "immer" import { selectText } from "../utils/request" interface EditorState { - sourceSelection: SelectionRequest | null - summarySelection: SelectionRequest | null + sourceSelection: SelectionRequest + summarySelection: SelectionRequest initiator: "source" | "summary" | null serverSection: SectionResponse setSourceSelection: (start: number, end: number) => void setSummarySelection: (start: number, end: number) => void - clearSelection: () => void + clearAllSelection: () => void + clearSourceSelection: () => void + clearSummarySelection: () => void fetchServerSection: (index: number) => Promise } export const useEditorStore = create()((set, get) => ({ - sourceSelection: null, - summarySelection: null, + sourceSelection: { start: -1, end: -1, from_summary: false }, + summarySelection: { start: -1, end: -1, from_summary: true }, initiator: null, serverSection: [], setSourceSelection: (start: number, end: number) => set(produce((state: EditorState) => { @@ -28,16 +30,18 @@ export const useEditorStore = create()((set, get) => ({ state.summarySelection = { start, end, from_summary: true } if (state.initiator === null) state.initiator = "summary" })), - clearSelection: () => set(produce((state: EditorState) => { - state.sourceSelection = null - state.summarySelection = null + clearAllSelection: () => set(produce((state: EditorState) => { + state.sourceSelection = { start: -1, end: -1, from_summary: false } + state.summarySelection = { start: -1, end: -1, from_summary: true } state.serverSection = [] state.initiator = null window.getSelection()?.removeAllRanges() })), + clearSourceSelection: () => set({ sourceSelection: null }), + clearSummarySelection: () => set({ summarySelection: null }), fetchServerSection: async (index: number) => { - if (get().initiator == "source") { - try { + try { + if (get().initiator == "source") { const response = await selectText(index, get().sourceSelection) if (isRequestError(response)) { handleRequestError(response) @@ -45,11 +49,19 @@ export const useEditorStore = create()((set, get) => ({ } set({ serverSection: response }) } - catch (e) { - console.log(e) - throw e + else if (get().initiator == "summary") { + const response = await selectText(index, get().summarySelection) + if (isRequestError(response)) { + handleRequestError(response) + return + } + set({ serverSection: response }) } } + catch (e) { + console.log(e) + throw e + } }, })) From ae985c1e1ae1003cbdfa4f0a812124ddb5fe2469 Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Fri, 22 Nov 2024 12:21:39 +0800 Subject: [PATCH 32/43] fix: null pointer --- components/editor/controls.tsx | 2 +- components/editor/editor.tsx | 10 ++++++---- components/editor/existing.tsx | 4 +++- store/useEditorStore.ts | 10 ++++++++-- utils/request.ts | 16 ++++++++++++++-- 5 files changed, 32 insertions(+), 10 deletions(-) diff --git a/components/editor/controls.tsx b/components/editor/controls.tsx index 20e8925..140d710 100644 --- a/components/editor/controls.tsx +++ b/components/editor/controls.tsx @@ -43,7 +43,7 @@ export default function Controls() { }} > + + } + > + Fail fetching server section + + , + { intent: "error" }, + ) + } + }, [indexStore.index]) + const handleMouseUp = useCallback((element: HTMLElement) => { const selection = rangy.getSelection() if (!selection || selection.rangeCount <= 0) return @@ -47,6 +107,7 @@ export default function Editor() { } const { start, end } = range.toCharacterRange(element) + console.log("Mouse up", start, end) if (element.id == "source") { editorStore.setSourceSelection(start, end) @@ -54,24 +115,13 @@ export default function Editor() { else if (element.id == "summary") { editorStore.setSummarySelection(start, end) } - }, []) - const onRestoreViewingHistory = useCallback(() => { - editorStore.clearAllSelection() - historyStore.setViewingRecord(null) - }, []) - - const onFetchTask = useCallback(async () => { - setHasError(false) - debounceSetIsLoading(true) - try { - await taskStore.fetch(indexStore.index) - } - catch (e) { - setHasError(true) + if (editorStore.initiator == element.id || editorStore.initiator == null) { + startTransition(async () => { + await onFetchServerSection() + }) } - debounceSetIsLoading(false) - }, [indexStore.index]) + }, [editorStore.initiator]) useEffect(() => { let ignore = false @@ -122,6 +172,67 @@ export default function Editor() { return segments } + function renderServerSection(target: "source" | "summary") { + const toDoc = target == "source" + const text = target === "source" ? taskStore.current.doc : taskStore.current.sum + const processedServerSection = processServerSection(editorStore.serverSection, toDoc) + + const normalizedColors = normalizationScore(editorStore.serverSection.map( + (section) => section.score, + )) + + const segments = [] + let lastIndex = 0 + for (const section of processedServerSection) { + if (section.offset > lastIndex) { + segments.push(text.slice(lastIndex, section.offset)) + } + + const sectionStart = section.offset + const sectionEnd = section.offset + section.len + + segments.push( + { + await labelText(indexStore.index, { + source_start: editorStore.initiator == "source" ? editorStore.sourceSelection.start : sectionStart, + source_end: editorStore.initiator == "source" ? editorStore.sourceSelection.end : sectionEnd, + summary_start: editorStore.initiator == "summary" ? editorStore.summarySelection.start : sectionStart, + summary_end: editorStore.initiator == "summary" ? editorStore.summarySelection.end : sectionEnd, + consistent: label, + note: note, + }) + historyStore.updateHistory(indexStore.index).then(() => { + editorStore.clearAllSelection() + }) + }} + message="Check all types that apply below." + />, + ) + lastIndex = section.offset + section.len + } + if (lastIndex < text.length) segments.push(text.slice(lastIndex)) + return segments + } + + function render(target: "source" | "summary") { + console.log("Render", target) + const selection = target === "source" ? editorStore.sourceSelection : editorStore.summarySelection + if (selection.start != -1 || editorStore.serverSection.length == 0) { + return renderHighlight(target) + } + + return renderServerSection(target) + } + return (
{( - suspendSource ? - - {taskStore.current.doc} - + isSuspendingSource ? + : - {renderHighlight("source")} + {render("source")} )} @@ -187,8 +294,8 @@ export default function Editor() { > { - suspendSummary ? - - {taskStore.current.sum} - + isSuspendingSummary ? + : - {renderHighlight("summary")} + {render("summary")} } @@ -229,3 +332,13 @@ export default function Editor() {
) } + +function SuspendText({ text }: { text: string }) { + return ( + + {text} + + ) +} diff --git a/store/useEditorStore.ts b/store/useEditorStore.ts index 73a19fd..f2c2a1e 100644 --- a/store/useEditorStore.ts +++ b/store/useEditorStore.ts @@ -39,15 +39,21 @@ export const useEditorStore = create()((set, get) => ({ })), clearSourceSelection: () => set(produce((state: EditorState) => { state.sourceSelection = { start: -1, end: -1, from_summary: false } - if (state.initiator === "source") state.initiator = null + if (state.initiator === "source") { + state.initiator = null + if (state.serverSection.length > 0) state.serverSection = [] + } })), clearSummarySelection: () => set(produce((state: EditorState) => { state.summarySelection = { start: -1, end: -1, from_summary: true } - if (state.initiator === "summary") state.initiator = null + if (state.initiator === "summary") { + state.initiator = null + if (state.serverSection.length > 0) state.serverSection = [] + } })), fetchServerSection: async (index: number) => { try { - if (get().initiator == "source") { + if (get().initiator === "source") { const response = await selectText(index, get().sourceSelection) if (isRequestError(response)) { handleRequestError(response) @@ -55,7 +61,7 @@ export const useEditorStore = create()((set, get) => ({ } set({ serverSection: response }) } - else if (get().initiator == "summary") { + else if (get().initiator === "summary") { const response = await selectText(index, get().summarySelection) if (isRequestError(response)) { handleRequestError(response) diff --git a/utils/processServerSection.ts b/utils/processServerSection.ts new file mode 100644 index 0000000..7f57189 --- /dev/null +++ b/utils/processServerSection.ts @@ -0,0 +1,11 @@ +import { SectionResponse, ServerSection } from "./types" + +export function processServerSection(section: SectionResponse, toDoc: boolean) { + const indexedSection = section + .filter((section) => section.to_doc == toDoc) + .map((s, i) => ({ ...s, index: i } as ServerSection) + ) + + return indexedSection + .sort((a, b) => a.offset - b.offset) +} diff --git a/utils/types.ts b/utils/types.ts index 8276633..1432b34 100644 --- a/utils/types.ts +++ b/utils/types.ts @@ -31,6 +31,14 @@ export type SectionResponseSlice = { export type SectionResponse = SectionResponseSlice[] +export type ServerSection = { + score: number + offset: number + len: number + to_doc: boolean + index: number +} + export function userSectionResponse(start: number, end: number, toDoc: boolean): SectionResponseSlice { return { score: 2, @@ -45,7 +53,7 @@ export type RequestError = { } export function isRequestError(obj: any): obj is RequestError { - return typeof obj.error === "string"; + return typeof obj.error === "string" } export function handleRequestError(e: RequestError) { From c993e1288329ffc34a4a0fc5edf46cd72eceb63c Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Sat, 23 Nov 2024 02:01:12 +0800 Subject: [PATCH 37/43] fix(frontend): Can not select after render server selection --- components/editor/editor.tsx | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/components/editor/editor.tsx b/components/editor/editor.tsx index 33cd4b9..7e1ea5e 100644 --- a/components/editor/editor.tsx +++ b/components/editor/editor.tsx @@ -91,32 +91,32 @@ export default function Editor() { } }, [indexStore.index]) - const handleMouseUp = useCallback((element: HTMLElement) => { + const handleMouseUp = useCallback((target: "source" | "summary") => { const selection = rangy.getSelection() if (!selection || selection.rangeCount <= 0) return const range = selection.getRangeAt(0) if (range.toString().trim() == "") { - if (element.id == "source") { + if (target == "source") { editorStore.clearSourceSelection() } - else if (element.id == "summary") { + else { editorStore.clearSummarySelection() } return } + const element = document.getElementById(target) const { start, end } = range.toCharacterRange(element) - console.log("Mouse up", start, end) - if (element.id == "source") { + if (target == "source") { editorStore.setSourceSelection(start, end) } - else if (element.id == "summary") { + else { editorStore.setSummarySelection(start, end) } - if (editorStore.initiator == element.id || editorStore.initiator == null) { + if (editorStore.initiator == target || editorStore.initiator == null) { startTransition(async () => { await onFetchServerSection() }) @@ -224,7 +224,6 @@ export default function Editor() { } function render(target: "source" | "summary") { - console.log("Render", target) const selection = target === "source" ? editorStore.sourceSelection : editorStore.summarySelection if (selection.start != -1 || editorStore.serverSection.length == 0) { return renderHighlight(target) @@ -274,8 +273,8 @@ export default function Editor() { style={{ whiteSpace: "pre-wrap", }} - onMouseUp={event => { - handleMouseUp(event.target as HTMLSpanElement) + onMouseUp={_ => { + handleMouseUp("source") }} > {render("source")} @@ -315,8 +314,8 @@ export default function Editor() { style={{ whiteSpace: "pre-wrap", }} - onMouseUp={event => { - handleMouseUp(event.target as HTMLSpanElement) + onMouseUp={_ => { + handleMouseUp("summary") }} > {render("summary")} From e2cd5b7d6fcadb779bfa3997d11635c417d70fdb Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Sat, 23 Nov 2024 02:09:31 +0800 Subject: [PATCH 38/43] fix(frontend): Can not dispatch toast --- app/new/page.tsx | 2 +- components/editor/editor.tsx | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/app/new/page.tsx b/app/new/page.tsx index 2ad49cb..acb815e 100644 --- a/app/new/page.tsx +++ b/app/new/page.tsx @@ -144,7 +144,7 @@ export default function New() {

- + ) } diff --git a/components/editor/editor.tsx b/components/editor/editor.tsx index 7e1ea5e..0e3c032 100644 --- a/components/editor/editor.tsx +++ b/components/editor/editor.tsx @@ -11,8 +11,6 @@ import { Text, Toast, ToastTitle, ToastTrigger, - useId, - useToastController, } from "@fluentui/react-components" import ExistingPane from "./existing" import { useTrackedIndexStore } from "../../store/useIndexStore" @@ -27,7 +25,7 @@ import "rangy/lib/rangy-textrange" import { getColor, normalizationScore } from "../../utils/color" import { processServerSection } from "../../utils/processServerSection" -export default function Editor() { +export default function Editor({ dispatchToast }: { dispatchToast: Function}) { const editorStore = useTrackedEditorStore() const taskStore = useTrackedTaskStore() const indexStore = useTrackedIndexStore() @@ -45,9 +43,6 @@ export default function Editor() { return isLoadingServerSection && editorStore.initiator === "source" }, [isLoadingServerSection, editorStore.initiator]) - const toasterId = useId("toaster") - const { dispatchToast } = useToastController(toasterId) - const debounceSetIsLoading = _.debounce(setIsLoading, 500) const onRestoreViewingHistory = useCallback(() => { From 179f6d1b670e0b84a2d996e638e6986a02e382ac Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Sat, 23 Nov 2024 12:47:41 +0800 Subject: [PATCH 39/43] refactor(frontend): Replace old page with rewrited one --- app/new/page.tsx | 150 ----------- app/page.tsx | 634 ++++++++--------------------------------------- 2 files changed, 103 insertions(+), 681 deletions(-) delete mode 100644 app/new/page.tsx diff --git a/app/new/page.tsx b/app/new/page.tsx deleted file mode 100644 index acb815e..0000000 --- a/app/new/page.tsx +++ /dev/null @@ -1,150 +0,0 @@ -"use client" - -import { - Button, - Title1, - Toast, - Toaster, - ToastTitle, - ToastTrigger, - useId, - useToastController, -} from "@fluentui/react-components" -import Controls from "../../components/editor/controls" -import LabelPagination from "../../components/labelPagination" -import Editor from "../../components/editor/editor" -import { useEffect, useState } from "react" -import { useTrackedIndexStore } from "../../store/useIndexStore" -import { useTrackedLabelsStore } from "../../store/useLabelsStore" -import { useRouter, useSearchParams } from "next/navigation" -import { checkUserMe } from "../../utils/request" -import { useTrackedUserStore } from "../../store/useUserStore" - -let didInit = false - -export default function New() { - const indexStore = useTrackedIndexStore() - const labelsStore = useTrackedLabelsStore() - const userStore = useTrackedUserStore() - - const router = useRouter() - const searchParams = useSearchParams() - - const [prevIndex, setPrevIndex] = useState(indexStore.index) - - const toasterId = useId("toaster") - const { dispatchToast } = useToastController(toasterId) - - useEffect(() => { - if (!didInit) { - didInit = true - - indexStore.fetchMax().catch(e => { - console.log(e) - dispatchToast( - - - - - } - > - Fail loading index - - , - { intent: "error" }, - ) - }) - - labelsStore.fetch().catch(e => { - console.log(e) - dispatchToast( - - - - - } - > - Fail loading labels - - , - { intent: "error" }, - ) - }) - - if (searchParams.has("sample")) { - const index = searchParams.get("sample") - const indexNumber = Number.parseInt(index) - if (!Number.isNaN(indexNumber) && indexNumber >= 0 && indexNumber <= indexStore.max && indexNumber !== prevIndex) { - setPrevIndex(indexNumber) - indexStore.setIndex(indexNumber) - } - } - - const access_token = localStorage.getItem("access_token") - if (access_token == "" || access_token == null) { - dispatchToast( - - Not logged in - , - { intent: "error" }, - ) - router.push("/login") - return - } - checkUserMe(access_token).then(valid => { - if (!valid) { - localStorage.removeItem("access_token") - dispatchToast( - - Session expired - , - { intent: "error" }, - ) - router.push("/login") - } - return - }) - userStore.fetch().catch(e => { - console.log(e) - dispatchToast( - - - - - } - > - Fail loading user data - - , - { intent: "error" }, - ) - }) - } - }, []) - - return ( - <> - - Mercury Label -
-
- -
- -
- - - ) -} diff --git a/app/page.tsx b/app/page.tsx index 1ad6ed2..c46dfb3 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -1,418 +1,138 @@ "use client" import { - Body1, Button, - Card, - CardHeader, - Text, - Title1, Toast, Toaster, ToastTitle, useId, useToastController, + Title1, + Toast, + Toaster, + ToastTitle, + ToastTrigger, + useId, + useToastController, } from "@fluentui/react-components" -import _ from "lodash" -import { useEffect, useLayoutEffect, useState } from "react" -import Tooltip from "../components/tooltip" -import { updateSliceArray } from "../utils/mergeArray" -import getRangeTextHandleableRange from "../utils/rangeTextNodes" -import { - exportLabel, - labelText, - selectText, - getAllLabels, - checkUserMe, getAllTasksLength, -} from "../utils/request" -import { type SectionResponse, userSectionResponse } from "../utils/types" -import { - ArrowExportRegular, - HandRightRegular, - ShareRegular, -} from "@fluentui/react-icons" -import { Allotment } from "allotment" -import "allotment/dist/style.css" -import UserPopover from "../components/userPopover" +import Controls from "../components/editor/controls" import LabelPagination from "../components/labelPagination" +import Editor from "../components/editor/editor" +import { useEffect, useState } from "react" import { useTrackedIndexStore } from "../store/useIndexStore" -import ExistingPane from "../components/editor/existing" -import { useTrackedHistoryStore } from "../store/useHistoryStore" -import { useTrackedTaskStore } from "../store/useTaskStore" -import { useTrackedUserStore } from "../store/useUserStore" -import { useRouter, useSearchParams } from "next/navigation" import { useTrackedLabelsStore } from "../store/useLabelsStore" -import { getColor, normalizationScore } from "../utils/color" - -enum Stage { - None = 0, - First = 1, -} - -const DISABLE_QUERY = false - - -// Function to determine if a color is light or dark -const isLightColor = (color: string) => { - // Remove the hash if present - let newColor = color.replace("#", "") - - // Convert 3-digit hex to 6-digit hex - if (newColor.length === 3) { - newColor = newColor.split("").map(char => char + char).join("") - } - - // Convert hex to RGB - const r = Number.parseInt(newColor.substring(0, 2), 16) - const g = Number.parseInt(newColor.substring(2, 4), 16) - const b = Number.parseInt(newColor.substring(4, 6), 16) - - // Calculate luminance - const luminance = 0.2126 * r + 0.7152 * g + 0.0722 * b - - // Return true if luminance is greater than 128 (light color) - return luminance > 200 -} +import { useRouter, useSearchParams } from "next/navigation" +import { checkUserMe } from "../utils/request" +import { useTrackedUserStore } from "../store/useUserStore" -const exportJSON = () => { - exportLabel().then(data => { - const blob = new Blob([JSON.stringify(data, null, 2)], { type: "application/json" }) - const url = URL.createObjectURL(blob) - const a = document.createElement("a") - a.href = url - a.download = "label.json" - a.click() - URL.revokeObjectURL(url) - }) -} +let didInit = false -export default function Index() { +export default function New() { const indexStore = useTrackedIndexStore() - const [prevIndex, setPrevIndex] = useState(indexStore.index) - const historyStore = useTrackedHistoryStore() - const [prevViewingRecord, setPrevViewingRecord] = useState(historyStore.viewingRecord) - const taskStore = useTrackedTaskStore() - const userStore = useTrackedUserStore() const labelsStore = useTrackedLabelsStore() + const userStore = useTrackedUserStore() + const router = useRouter() const searchParams = useSearchParams() - const [firstRange, setFirstRange] = useState<[number, number] | null>(null) - const [rangeId, setRangeId] = useState(null) - const [serverSelection, setServerSelection] = useState(null) - const [userSelection, setUserSelection] = useState<[number, number] | null>(null) - const [waiting, setWaiting] = useState(null) - const [stage, setStage] = useState(Stage.None) + + const [prevIndex, setPrevIndex] = useState(indexStore.index) const toasterId = useId("toaster") const { dispatchToast } = useToastController(toasterId) - function washHand() { - setFirstRange(null) - setRangeId(null) - setWaiting(null) - setServerSelection(null) - setStage(Stage.None) - setUserSelection(null) - window.getSelection()?.removeAllRanges() - } - - indexStore.fetchMax().catch(e => { - console.log(e) - }) - labelsStore.fetch().catch(e => { - console.log(e) - }) - - if (searchParams.has("sample")) { - const index = searchParams.get("sample") - const indexNumber = Number.parseInt(index) - if (!Number.isNaN(indexNumber) && indexNumber >= 0 && indexNumber <= indexStore.max && indexNumber !== prevIndex) { - setPrevIndex(indexNumber) - washHand() - indexStore.setIndex(indexNumber) - } - } - - taskStore.fetch(indexStore.index).catch(error => { - console.log(error) - }) - useEffect(() => { - const access_token = localStorage.getItem("access_token") - if (access_token == "" || access_token == null) { - dispatchToast( - - Not logged in - , - { intent: "error" }, - ) - router.push("/login") - return - } - checkUserMe(access_token).then(valid => { - if (!valid) { - localStorage.removeItem("access_token") + if (!didInit) { + didInit = true + + indexStore.fetchMax().catch(e => { + console.log(e) dispatchToast( - Session expired + + + + } + > + Fail loading index + , { intent: "error" }, ) - router.push("/login") - } - return - }) - userStore.fetch().catch(e => { - console.log(e) - }) - }, []) - - if (historyStore.viewingRecord !== prevViewingRecord) { - setPrevViewingRecord(historyStore.viewingRecord) - if (historyStore.viewingRecord === null || taskStore.current === null) { - washHand() - return - } - setFirstRange([historyStore.viewingRecord.source_start, historyStore.viewingRecord.source_end]) - setRangeId("doc") - setServerSelection([userSectionResponse(historyStore.viewingRecord.summary_start, historyStore.viewingRecord.summary_end, true)]) - } - - useLayoutEffect(() => { - const func = (event) => { - const selection = window.getSelection() - const target = event.target as HTMLElement - - const mercuryElements = document.querySelectorAll("[data-mercury-disable-selection]") + }) - // console.log(mercuryElements) + labelsStore.fetch().catch(e => { + console.log(e) + dispatchToast( + + + + + } + > + Fail loading labels + + , + { intent: "error" }, + ) + }) - for (const element of mercuryElements) { - if (element.contains(target)) { - return + if (searchParams.has("sample")) { + const index = searchParams.get("sample") + const indexNumber = Number.parseInt(index) + if (!Number.isNaN(indexNumber) && indexNumber >= 0 && indexNumber <= indexStore.max && indexNumber !== prevIndex) { + setPrevIndex(indexNumber) + indexStore.setIndex(indexNumber) } } - if (target.id.startsWith("label-")) { + const access_token = localStorage.getItem("access_token") + if (access_token == "" || access_token == null) { + dispatchToast( + + Not logged in + , + { intent: "error" }, + ) + router.push("/login") return } - - if ( - !selection.containsNode(document.getElementById("summary"), true) && - !selection.containsNode(document.getElementById("doc"), true) - ) { - if (userSelection !== null) { - setUserSelection(null) - } - else { - washHand() + checkUserMe(access_token).then(valid => { + if (!valid) { + localStorage.removeItem("access_token") + dispatchToast( + + Session expired + , + { intent: "error" }, + ) + router.push("/login") } return - } - - if (selection.toString().trim() === "") { - if (target.tagName === "SPAN") { - const span = target as HTMLSpanElement - if (span.parentElement?.id === "summary" || span.parentElement?.id === "doc") { - return - } - } - else if (target.tagName === "P") { - const p = target as HTMLParagraphElement - if (p.id === "summary" || p.id === "doc") { - return - } - } - else { - if (userSelection !== null) { - setUserSelection(null) - } - else { - washHand() - } - return - } - } - } - document.body.addEventListener("mouseup", func) - return () => { - document.body.removeEventListener("mouseup", func) - } - }, [userSelection]) - - useEffect(() => { - if (firstRange === null || rangeId === null) { - setServerSelection(null) - return - } - _.debounce(() => { - if (DISABLE_QUERY || historyStore.viewingRecord != null) return - setWaiting(rangeId === "summary" ? "doc" : "summary") - selectText(indexStore.index, { - start: firstRange[0], - end: firstRange[1], - from_summary: rangeId === "summary", }) - .then(response => { - setWaiting(null) - if ("error" in response) { - console.error(response.error) - return - } - if (firstRange === null || rangeId === null) { - setServerSelection(null) - return - } - setServerSelection(response as SectionResponse) - }) - .catch(error => { - console.error(error) - }) - }, 100)() - }, [firstRange, rangeId, indexStore.index]) - - const JustSliceText = (props: { text: string; startAndEndOffset: [number, number] }) => { - const fakeResponse = userSectionResponse( - props.startAndEndOffset[0], - props.startAndEndOffset[1], - rangeId === "summary", - ) - const sliceArray = updateSliceArray(props.text, [fakeResponse]) - return sliceArray.map(slice => { - return slice[3] === 2 ? ( - { - if (firstRange === null || rangeId === null) { - return Promise.resolve() - } - await labelText(indexStore.index, { - source_start: rangeId === "summary" ? -1 : firstRange[0], - source_end: rangeId === "summary" ? -1 + 1 : firstRange[1], - summary_start: rangeId === "summary" ? firstRange[0] : -1, - summary_end: rangeId === "summary" ? firstRange[1] : -1, - consistent: label, - note: note, - }) - historyStore.updateHistory(indexStore.index).then(() => { - }) - }} - message="Check all types that apply below." - /> - ) : ( - - {props.text.slice(slice[0], slice[1] + 1)} - - ) - }) - } - - const SliceText = (props: { text: string; slices: SectionResponse; user: [number, number] | null }) => { - const newSlices = - props.user === null - ? props.slices - : [userSectionResponse(props.user[0], props.user[1], rangeId === "summary")] - const sliceArray = updateSliceArray(props.text, newSlices) - const allScore = [] - for (const slice of newSlices) { - allScore.push(slice.score) - } - const normalColor = normalizationScore(allScore) - return ( - <> - {sliceArray.map(slice => { - const isBackendSlice = slice[2] - const score = slice[3] - const bg_color = isBackendSlice ? score === 2 ? "#85e834" : getColor(normalColor[slice[4]]) : "#ffffff" - const textColor = isLightColor(bg_color) ? "black" : "white" - // const textColor= 'red' - return isBackendSlice && historyStore.viewingRecord == null ? ( - { - if (firstRange === null || rangeId === null) { - return Promise.resolve() - } - await labelText(indexStore.index, { - source_start: rangeId === "summary" ? slice[0] : firstRange[0], - source_end: rangeId === "summary" ? slice[1] + 1 : firstRange[1], - summary_start: rangeId === "summary" ? firstRange[0] : slice[0], - summary_end: rangeId === "summary" ? firstRange[1] : slice[1] + 1, - consistent: label, - note: note, - }) - historyStore.updateHistory(indexStore.index).then(() => { - }) - }} - message="Select the type(s) of hallucinatin below." - /> - ) : ( - - {props.text.slice(slice[0], slice[1] + 1)} - - ) - })} - - ) - } - - const checkSelection = (element: HTMLSpanElement) => { - const selection = window.getSelection() - if (selection === null || selection === undefined) return - if (!selection.containsNode(element, true)) return - if (selection.toString().trim() === "" && JSON.stringify(firstRange) !== "[-1,-1]") return - const range = selection.getRangeAt(0) - switch (stage) { - case Stage.None: { - if ( - range.intersectsNode(element) && - range.startContainer === range.endContainer && - range.startContainer === element.firstChild && - range.startOffset !== range.endOffset - ) { - setFirstRange([range.startOffset, range.endOffset]) - setUserSelection(null) - setRangeId(element.id) - } - if (selection.containsNode(element, false)) { - setFirstRange([range.startOffset, element.firstChild?.textContent?.length]) - setUserSelection(null) - setRangeId(element.id) - } - setStage(Stage.First) - break - } - case Stage.First: { - if (element.id === rangeId || element.parentElement?.id === rangeId) { - setFirstRange(getRangeTextHandleableRange(range)) - setUserSelection(null) - } - else { - setUserSelection(getRangeTextHandleableRange(range)) - } - break - } + userStore.fetch().catch(e => { + console.log(e) + dispatchToast( + + + + + } + > + Fail loading user data + + , + { intent: "error" }, + ) + }) } - } + }, []) return ( <> @@ -420,159 +140,11 @@ export default function Index() { Mercury Label

-
- {JSON.stringify(firstRange) === "[-1,-1]" || historyStore.viewingRecord != null ? ( - - ) : ( - - )} - - - - - - {/* - - */} - {/* - */} -
+
- +
- {taskStore.current === null ? ( -

Loading...

- ) : ( -
- - -
- - - Source - - } - /> - { - checkSelection(event.target as HTMLSpanElement) - }} - > - {serverSelection !== null && serverSelection.length > 0 && rangeId === "summary" ? ( - - ) : rangeId === "doc" ? ( - - ) : ( - taskStore.current.doc - )} - - -
- - - -
- - - Summary - - } - /> - checkSelection(event.target as HTMLSpanElement)} - > - {serverSelection !== null && rangeId === "doc" ? ( - - ) : rangeId === "summary" ? ( - - ) : ( - taskStore.current.sum - )} - - -
- - - - -
- )} + ) } From df27e5a018f4aa4037609859479d26a829f30430 Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Sat, 23 Nov 2024 12:59:19 +0800 Subject: [PATCH 40/43] refactor(frontend): combine historyStore and editorStore --- components/editor/controls.tsx | 4 +--- components/editor/editor.tsx | 8 +++----- components/editor/existing.tsx | 18 +++++++++--------- components/labelPagination.tsx | 4 +--- store/useEditorStore.ts | 24 ++++++++++++++++++++++-- store/useHistoryStore.ts | 31 ------------------------------- 6 files changed, 36 insertions(+), 53 deletions(-) delete mode 100644 store/useHistoryStore.ts diff --git a/components/editor/controls.tsx b/components/editor/controls.tsx index 5a11c85..11ae1a2 100644 --- a/components/editor/controls.tsx +++ b/components/editor/controls.tsx @@ -7,16 +7,14 @@ import UserPopover from "../userPopover" import { useTrackedIndexStore } from "../../store/useIndexStore" import { useTrackedEditorStore } from "../../store/useEditorStore" import { useCallback } from "react" -import { useTrackedHistoryStore } from "../../store/useHistoryStore" export default function Controls() { const indexStore = useTrackedIndexStore() const editorStore = useTrackedEditorStore() - const historyStore = useTrackedHistoryStore() const onReset = useCallback(() => { editorStore.clearAllSelection() - historyStore.setViewingRecord(null) + editorStore.setViewing(null) }, []) const onExportJSON = useCallback(async () => { diff --git a/components/editor/editor.tsx b/components/editor/editor.tsx index 0e3c032..7d0d0d2 100644 --- a/components/editor/editor.tsx +++ b/components/editor/editor.tsx @@ -14,7 +14,6 @@ import { } from "@fluentui/react-components" import ExistingPane from "./existing" import { useTrackedIndexStore } from "../../store/useIndexStore" -import { useTrackedHistoryStore } from "../../store/useHistoryStore" import { HasError, Loading } from "./fallback" import _ from "lodash" import { labelText } from "../../utils/request" @@ -29,7 +28,6 @@ export default function Editor({ dispatchToast }: { dispatchToast: Function}) { const editorStore = useTrackedEditorStore() const taskStore = useTrackedTaskStore() const indexStore = useTrackedIndexStore() - const historyStore = useTrackedHistoryStore() const labelsStore = useTrackedLabelsStore() const [isLoadingServerSection, startTransition] = useTransition() @@ -47,7 +45,7 @@ export default function Editor({ dispatchToast }: { dispatchToast: Function}) { const onRestoreViewingHistory = useCallback(() => { editorStore.clearAllSelection() - historyStore.setViewingRecord(null) + editorStore.setViewing(null) }, []) const onFetchTask = useCallback(async () => { @@ -156,7 +154,7 @@ export default function Editor({ dispatchToast }: { dispatchToast: Function}) { consistent: label, note: note, }, editorStore.initiator === target ? target : null) - historyStore.updateHistory(indexStore.index).then(() => { + editorStore.updateHistory(indexStore.index).then(() => { editorStore.clearAllSelection() }) }} @@ -205,7 +203,7 @@ export default function Editor({ dispatchToast }: { dispatchToast: Function}) { consistent: label, note: note, }) - historyStore.updateHistory(indexStore.index).then(() => { + editorStore.updateHistory(indexStore.index).then(() => { editorStore.clearAllSelection() }) }} diff --git a/components/editor/existing.tsx b/components/editor/existing.tsx index 32760ff..077fc89 100644 --- a/components/editor/existing.tsx +++ b/components/editor/existing.tsx @@ -19,13 +19,13 @@ import { } from "@fluentui/react-components" import { ArrowSyncRegular, DeleteRegular, EyeOffRegular, EyeRegular } from "@fluentui/react-icons" import { deleteRecord } from "../../utils/request" -import { useTrackedHistoryStore } from "../../store/useHistoryStore" import { useTrackedTaskStore } from "../../store/useTaskStore" import { LabelData } from "../../utils/types" import { useCallback, useEffect, useMemo, useState } from "react" import { useTrackedIndexStore } from "../../store/useIndexStore" import { HasError, Loading } from "./fallback" import _ from "lodash" +import { useTrackedEditorStore } from "../../store/useEditorStore" const columnsDef: TableColumnDefinition[] = [ createTableColumn({ @@ -55,7 +55,7 @@ type Props = { } export default function ExistingPane({ onRestore = Function() }: Props) { - const historyStore = useTrackedHistoryStore() + const editorStore = useTrackedEditorStore() const indexStore = useTrackedIndexStore() const taskStore = useTrackedTaskStore() @@ -68,7 +68,7 @@ export default function ExistingPane({ onRestore = Function() }: Props) { setHasError(false) debounceSetIsLoading(true) try { - await historyStore.updateHistory(indexStore.index) + await editorStore.updateHistory(indexStore.index) } catch (e) { setHasError(true) @@ -89,13 +89,13 @@ export default function ExistingPane({ onRestore = Function() }: Props) { }, [indexStore.index]) const sortedHistory = useMemo(() => { - return historyStore.history + return editorStore.history .sort((a, b) => { let c = a.source_start - b.source_start if (c === 0) c = a.summary_start - b.summary_start return c }) - }, [historyStore.history]) + }, [editorStore.history]) const [columns] = useState[]>(columnsDef) const [columnSizingOptions] = useState({ @@ -206,10 +206,10 @@ export default function ExistingPane({ onRestore = Function() }: Props) { {item.note} - {historyStore.viewingRecord != null && historyStore.viewingRecord.record_id === item.record_id ? ( + {editorStore.viewing != null && editorStore.viewing.record_id === item.record_id ? ( - - } - > - Fail loading user data - - , - { intent: "error" }, - ) - }) + checkUserMe(access_token).then(valid => { + if (!valid) { + localStorage.removeItem("access_token") + dispatchToast( + + Session expired + , + { intent: "error" }, + ) + router.push("/login") + } + return + }) + userStore.fetch().catch(e => { + console.log(e) + dispatchToast( + + + + + } + > + Fail loading user data + + , + { intent: "error" }, + ) + }) + } } }, []) @@ -144,7 +143,15 @@ export default function New() {

- + ) } + +export default function Index() { + return ( + + + + ) +} diff --git a/components/editor/controls.tsx b/components/editor/controls.tsx index 11ae1a2..04de717 100644 --- a/components/editor/controls.tsx +++ b/components/editor/controls.tsx @@ -33,6 +33,32 @@ export default function Controls() { } }, []) + const onShare = useCallback(async () => { + const url = `${window.location.origin}${window.location.pathname}?sample=${indexStore.index}` + if (navigator.clipboard && window.isSecureContext) { + await navigator.clipboard.writeText(url) + } + else { + const textArea = document.createElement("textarea") + textArea.value = url + textArea.style.position = "absolute" + textArea.style.left = "-999999px" + + document.body.prepend(textArea) + textArea.select() + + try { + document.execCommand("copy") + } + catch (error) { + console.error(error) + } + finally { + textArea.remove() + } + } + }, [indexStore.index]) + return (
} onClick={onExportJSON}> Export Labels - diff --git a/components/editor/editor.tsx b/components/editor/editor.tsx index d3d3323..9a29e98 100644 --- a/components/editor/editor.tsx +++ b/components/editor/editor.tsx @@ -85,6 +85,7 @@ export default function Editor({ dispatchToast }: { dispatchToast: Function }) { }, [indexStore.index]) const handleMouseUp = useCallback((target: "source" | "summary") => { + if (typeof window === "undefined") return const selection = rangy.getSelection() if (!selection || selection.rangeCount <= 0) return if (!editorStore.editable) return diff --git a/store/useIndexStore.ts b/store/useIndexStore.ts index 0cdfaf6..ec19975 100644 --- a/store/useIndexStore.ts +++ b/store/useIndexStore.ts @@ -1,6 +1,7 @@ import { create } from "zustand" import { createTrackedSelector } from "react-tracked" import { getAllTasksLength } from "../utils/request" +import { produce } from "immer" interface IndexState { index: number @@ -20,7 +21,10 @@ export const useIndexStore = create()((set) => ({ fetchMax: async () => { try { const task = await getAllTasksLength() - set({ max: task.all - 1 }) + set(produce((state: IndexState) => { + state.max = task.all - 1 + if (state.index > state.max) state.index = 0 + })) } catch (e) { console.log(e) throw e From 16ae94dc1e38c6db15ec1336b40a7411b8affb8f Mon Sep 17 00:00:00 2001 From: Nanami Nakano <64841155+NanamiNakano@users.noreply.github.com> Date: Sat, 23 Nov 2024 16:01:52 +0800 Subject: [PATCH 43/43] chore: Code cleanup --- app/page.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/page.tsx b/app/page.tsx index 48a0df9..a33897e 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -13,7 +13,7 @@ import { import Controls from "../components/editor/controls" import LabelPagination from "../components/labelPagination" import Editor from "../components/editor/editor" -import { Suspense, useEffect, useState } from "react" +import { Suspense, useEffect } from "react" import { useTrackedIndexStore } from "../store/useIndexStore" import { useTrackedLabelsStore } from "../store/useLabelsStore" import { useRouter, useSearchParams } from "next/navigation"