From 068be1345f803cf580dbccf481077bdb41165dd9 Mon Sep 17 00:00:00 2001 From: Yicheng Wang Date: Mon, 26 Sep 2016 12:21:34 -0400 Subject: [PATCH 1/3] Started conversion to MongoDB, added new_user, authenticate and update_pwd --- database-sqlite.py | 520 +++++++++++++++++++++++++++++++++++++++ database.py | 597 ++++++++------------------------------------- 2 files changed, 616 insertions(+), 501 deletions(-) create mode 100644 database-sqlite.py diff --git a/database-sqlite.py b/database-sqlite.py new file mode 100644 index 0000000..8b9d268 --- /dev/null +++ b/database-sqlite.py @@ -0,0 +1,520 @@ +################################################################################ +# Backend Database management for Annotation project # +# # +# Authors # +# Yicheng Wang # +# # +# Description # +# Deals with database in various fashion # +# # +################################################################################ + +# TODO +# TODO-List + +# Dev Log +# Project Created: 2015-12-19 15:00 - Yicheng W. + +import sqlite3 +from time import time, sleep +from search import * +from bs4 import BeautifulSoup + +def new_user(email, password_hash, first, last): + """ + new_use: adds a new user to the database, returns False if unsuccessful + + Args: + email (string): the email of the user + password_hash (string): the hex string of a hased password + first (string): the first name of the user + last (string): the last name of the user + + Returns: + True if successfully added, False otherwise (email has already been + taken) + + Example: + >>> new_user("alex.wyc2098@gmail.com", "7b77e4d3de87423f0c98716ad54bd2f3", "Yicheng", "Wang") + True + >>> new_user("alex.wyc2098@gmail.com", "12354ab6a7e87af879dbadf87124faab", "Yicheng", "Wang") + False (because the email has already been registered) + """ + conn = sqlite3.connect("./db/infos.db") + c = conn.cursor() + + q = """SELECT users.email + FROM users + WHERE users.email = ?""" + + usernames = c.execute(q, (email,)).fetchall() + + if (len(usernames)) == 0: + q = "INSERT INTO users VALUES (?,?,?,?)" + c.execute(q, (email, password_hash, first, last)) + conn.commit() + return True + + return False + +def authenticate(email, password_hash): + """ + authenticate: authenticates an user login + + Args: + email (string): the email to authenticate + password_hash (string): the hash of the password that the user inputed + + Returns: + True if the two match, False otherwise + + Example: + >>> authenticate("alex.wyc2098@gmail.com", "7b77e4d3de87423f0c98716ad54bd2f3") + True + >>> authenticate("alex.wyc2098@gmail.com", "12354ab6a7e87af879dbadf87124faab") + False + """ + + conn = sqlite3.connect("./db/infos.db") + c = conn.cursor() + + q = """SELECT users.email, users.password + FROM users + WHERE users.email = ? AND users.password = ?""" + + result = c.execute(q, (email, password_hash)).fetchall() + + return (len(result) != 0) + +def update_pwd(email, new_password): + """ + update_pwd: updates password for an user + + Args: + email (string): the user + new_password (string): the new password + + Returns: + True if successful, False otherwise + """ + + conn = sqlite3.connect("./db/infos.db") + c = conn.cursor() + + q = """SELECT * FROM users WHERE users.email = ?""" + + result = c.execute(q, (email,)).fetchall() + + if len(result) == 0: + return False + + q = """UPDATE users SET password = ? WHERE email = ?""" + + c.execute(q, (new_password, email)) + + conn.commit() + + return True + +def get_name_from_email(email): + """ + get_name_from_email: returns the name of the client based on the email + entered + + Args: + email (string): the email you are looking for + + Returns: + a string of the form "first last", empty string if the email doesn't + exist + + Example: + get_name_from_email("alex.wyc2098@gmail.com") --> Yicheng Wang + """ + + conn = sqlite3.connect("./db/infos.db") + c = conn.cursor() + + q = """SELECT users.first, users.last FROM users WHERE users.email = ?""" + + result = c.execute(q, (email,)).fetchall() + + if len(result) == 0: + return "" + + return result[0][0] + " " + result[0][1] + +def next_avaliable_id(): + """ + next_a: gives out the next avaliable id for the sites table + + Returns: + an integer that represents the next possible id + """ + + conn = sqlite3.connect("./db/infos.db") + c = conn.cursor() + + q = """SELECT sites.id FROM sites ORDER BY sites.id""" + + result = c.execute(q).fetchall() + + ##print result + + if (len(result) == 0): + return 0 + + if result[0][0] != 0: + #print "0 index: " + str(result[0][0]) + return 0 + + if (len(result) == 1): + return 1 + + for i in range(1, len(result)): + if result[i][0] - result[i - 1][0] > 1: + return result[i - 1][0] + 1 + + return len(result) + +def add_to_sites(email, title, site, comments, notes): + """ + add_to_sites: add a site to the user's list + + Args: + email (string): the user + title (string): title of the site + site (string): the html of the site + comments (string): the comments on the site + notes (string): the notes on the site + + Returns: + the id of the new site if successful, -1 otherwise + """ + + conn = sqlite3.connect("./db/infos.db") + c = conn.cursor() + + q = """INSERT INTO sites VALUES (?, ?, ?, ?, ?, ?, ?, ?)""" + + id = next_avaliable_id() + c.execute(q, (id, email, title, site, comments, notes, 0, int(time()))) # default permission is private + conn.commit() + return id + +def get_list_of_sites(email): + """ + get_list_of_sites: returns a list of sites based on a certain email + + Args: + email (string): the user + + Returns: + a list of sites formatted in the following manner ranked by last edited + time: + [(site-id1, site-title1, permission1), (site-id2, site-title2, permission2), ...] + """ + + conn = sqlite3.connect("./db/infos.db") + c = conn.cursor() + + q = """SELECT sites.id, sites.title, sites.shared + FROM sites + WHERE sites.email = ? + ORDER BY t DESC""" + + r = c.execute(q, (email,)).fetchall() + return r + +def get_site_on_id(email, id): + """ + get_site_on_id: get the site title, site, the notes and the comments + + Args: + email (string): the user + id (int): the site id + + Returns: + a tuple in the form of: + (title, site, notes, comments) + + or none if retrival was not successful + """ + + conn = sqlite3.connect("./db/infos.db") + c = conn.cursor() + + q = """SELECT sites.title, sites.site, sites.notes, sites.comments + FROM sites + WHERE sites.email = ? AND sites.id = ?""" + + r = c.execute(q, (email, id)).fetchall() + + ##print len(r) + + #for i in r: + # #print i[0] + ##print r + + if (len(r) != 1): + return None + + return r[0] + +def get_site_for_sharing(id): + """ + get_site_for_sharing: get the content of one site for sharing, returns None + if the site doesn't exist or is private + + Args: + id (int): the ID of the site + + Returns: + a tuple in the form of: + (title, site, notes, comments) + + or none if retrival was not successful + """ + + conn = sqlite3.connect("./db/infos.db") + c = conn.cursor() + + q = """SELECT sites.title, sites.site, sites.notes, + sites.comments + FROM sites + WHERE sites.id = ? AND sites.shared = 1""" + + r = c.execute(q, (id, )).fetchall() + + if (len(r) == 0): + return None + + else: + return r[0] + +def update_site(email, site_id, new_site, new_comments, new_notes): + """ + update_site: updates the site entry for the user + + Args: + email (string): the user + site_id (int): the id of the site in the database + new_site (string): updated markup for the site + new_comments (string): updated comments for the site + new_notes (string): updated notes for the site + + Returns: + True if successful, False otherwise + """ + + conn = sqlite3.connect("./db/infos.db") + c = conn.cursor() + + q = """SELECT * FROM sites WHERE sites.email = ? AND sites.id = ?""" + + result = c.execute(q, (email, site_id)).fetchall() + + if (len(result) == 0): + return False + + q = """UPDATE sites SET site = ?, comments = ?, notes = ?, t = ? WHERE id = ?""" + + c.execute(q, (new_site, new_comments, new_notes, int(time()), site_id)) + + conn.commit() + + return True + +def change_site_permission(email, id): + """ + change_site_permission: changes the permission of a site (public -> private + or private -> public) + + Args: + email (string): the user + id (int): the id of the site + + Returns: + True if successful, False if the id and the email doesn't match + """ + + conn = sqlite3.connect('./db/infos.db') + c = conn.cursor() + + q = """SELECT sites.shared + FROM sites + WHERE sites.email = ? AND sites.id = ?""" + + r = c.execute(q, (email, id)).fetchall() + + if (len(r) == 0): + return False + + q = """UPDATE sites + SET shared = ? + WHERE id = ?""" + + if (r[0][0] == 0): # used to be private, now becomes public + c.execute(q, (1, id)) + else: # used to be public, now becomes private + c.execute(q, (0, id)) + + conn.commit() + + return True + +def fork_shared_site(site_id, email): + """ + fork_shared_site: this makes a copy of the shared site with a specific + site_id within the user's private library + + Args: + site_id (int): the site_id of the shared site + email (string): the user who wishes to fork the site + + Returns: + The new ID of the forked site, or -1 if unsuccessful + """ + + conn = sqlite3.connect("./db/infos.db") + c = conn.cursor() + + q = """SELECT sites.shared, sites.title, sites.site, sites.comments, sites.notes + FROM sites + WHERE sites.id = ?""" + + r = c.execute(q, (site_id,)).fetchall() + + if (len(r) == 0): + return -1 + + if r[0][0] == 0: + return -1 + + return add_to_sites(email, r[0][1], r[0][2], r[0][3], r[0][4]) + +def delete_site(email, site_id): + """ + delete_site: deletes a site from the user's "library" according to id + + Args: + email (string): the user + site_id (int): id of the site + + Returns: + True if successful, False otherwise + """ + + conn = sqlite3.connect("./db/infos.db") + c = conn.cursor() + + q = """SELECT * FROM sites WHERE sites.email = ? AND sites.id = ?""" + + result = c.execute(q, (email, site_id)).fetchall() + + if (len(result) == 0): + return False + + q = """DELETE FROM sites WHERE sites.id = ?""" + + c.execute(q, (site_id,)) + + conn.commit() + + return True + +def search_user_sites(email, search_string): + """ + search_user_sites: searches a specific user's pages for a specific set of + strings + + Args: + email (string): the user + search_string (string): the string to search + + Returns: + a list of abstracted articles in which each element is a dictionary of + the form: + { + 'index' : index-of-proximity (see search.py) (float), + 'id' : site id (int), + 'abstract' : abstracted site (string), + 'snippet' : list of matched words (see search.py) + } + + ordered by 'index' + """ + + ret_val = [] + search = search_string.split() + + conn = sqlite3.connect("./db/infos.db") + c = conn.cursor() + + q = """SELECT sites.id, sites.site, sites.title, sites.comments, sites.notes + FROM sites WHERE sites.email = ?""" + + result = c.execute(q, (email,)).fetchall() + + for i in result: + soup = BeautifulSoup(i[1] + i[3]) + site_cleaned = i[2] + ' ' + soup.get_text() + i[4] + snippets = get_snippets_from_site(site_cleaned, search) + + if (len(snippets) > 0): + index = get_index_of_proximity(site_cleaned, search) + abstract = abstract_site_from_words(site_cleaned, snippets) + + ret_val.append({ + 'index':index, + 'id':i[0], + 'abstract':abstract, + 'snippet':snippets, + 'title':i[2] + }) + + return sorted(ret_val, key=lambda entry: entry['index'], reverse=True) + +if __name__ == "__main__": + #print "new_user test" + #print new_user("alex.wyc2098@gmail.com", "12345", "Yicheng", "Wang") + #print new_user("alex.wyc2098@gmail.com", "dgjsadkfhsa", "Yicheng", "Wang") + #print new_user("alex.wyc2098@protonmail.ch", "ajdfsadfk", "Yicheng", "Wang") + + #print "\nauthentication test" + #print authenticate("alex.wyc2098@gmail.com", "12345") + #print authenticate("alex.wyc2098@protonmail.ch", "12345") + #print authenticate("asdf@asdf.asdf", "12435") + + #print "\nchange password test" + #print update_pwd("alex.wyc2098@gmail.com", "54321") + #print update_pwd("asdf@asdf.com", "12345") + + #print authenticate("alex.wyc2098@gmail.com", "12345") + #print authenticate("alex.wyc2098@gmail.com", "54321") + + #print "\nadd_to_sites test" + #print add_to_sites("alex.wyc2098@gmail.com", "123456789") + #print add_to_sites("alex.wyc2098@protonmail.ch", "fkjhsadgfkvasv") + #print add_to_sites("alex.wyc2098@gmail.com", "12hsadffghas") + + #print get_list_of_sites("alex.wyc2098@gmail.com") + #print get_list_of_sites("alex.wyc2098@protonmail.ch") + + sleep(1) + + #print "\nupdate_site test" + #print update_site("alex.wyc2098@gmail.com", "0", "new_site!") + + #print get_list_of_sites("alex.wyc2098@gmail.com") + + #print "\ndelete_site test" + #print delete_site("alex.wyc2098@gmail.com", 2) + #print delete_site("alex.wyc2098@gmail.com", 3) + + #print add_to_sites("alex.wyc2098@gmail.com", "this is the new site 2") + + #print get_list_of_sites("alex.wyc2098@gmail.com") + + #print "searching" + search_user_sites("alex.wyc2098@gmail.com", 'asdf') diff --git a/database.py b/database.py index 8b9d268..186ecfe 100644 --- a/database.py +++ b/database.py @@ -1,11 +1,14 @@ ################################################################################ -# Backend Database management for Annotation project # +# Backend # +# database management for Mawrginalia, backed by MongoDB # # # # Authors # -# Yicheng Wang # +# Yicheng # +# Wang # # # # Description # -# Deals with database in various fashion # +# Deals # +# with database in various fashions, using MongoDB # # # ################################################################################ @@ -13,508 +16,100 @@ # TODO-List # Dev Log -# Project Created: 2015-12-19 15:00 - Yicheng W. +# Project Created: 2016-08-21 15:41 - Yicheng W. -import sqlite3 +from pymongo import MongoClient from time import time, sleep from search import * from bs4 import BeautifulSoup -def new_user(email, password_hash, first, last): - """ - new_use: adds a new user to the database, returns False if unsuccessful - - Args: - email (string): the email of the user - password_hash (string): the hex string of a hased password - first (string): the first name of the user - last (string): the last name of the user - - Returns: - True if successfully added, False otherwise (email has already been - taken) - - Example: - >>> new_user("alex.wyc2098@gmail.com", "7b77e4d3de87423f0c98716ad54bd2f3", "Yicheng", "Wang") - True - >>> new_user("alex.wyc2098@gmail.com", "12354ab6a7e87af879dbadf87124faab", "Yicheng", "Wang") - False (because the email has already been registered) - """ - conn = sqlite3.connect("./db/infos.db") - c = conn.cursor() - - q = """SELECT users.email - FROM users - WHERE users.email = ?""" - - usernames = c.execute(q, (email,)).fetchall() - - if (len(usernames)) == 0: - q = "INSERT INTO users VALUES (?,?,?,?)" - c.execute(q, (email, password_hash, first, last)) - conn.commit() - return True - - return False - -def authenticate(email, password_hash): - """ - authenticate: authenticates an user login - - Args: - email (string): the email to authenticate - password_hash (string): the hash of the password that the user inputed - - Returns: - True if the two match, False otherwise - - Example: - >>> authenticate("alex.wyc2098@gmail.com", "7b77e4d3de87423f0c98716ad54bd2f3") - True - >>> authenticate("alex.wyc2098@gmail.com", "12354ab6a7e87af879dbadf87124faab") - False - """ - - conn = sqlite3.connect("./db/infos.db") - c = conn.cursor() - - q = """SELECT users.email, users.password - FROM users - WHERE users.email = ? AND users.password = ?""" - - result = c.execute(q, (email, password_hash)).fetchall() - - return (len(result) != 0) - -def update_pwd(email, new_password): - """ - update_pwd: updates password for an user - - Args: - email (string): the user - new_password (string): the new password - - Returns: - True if successful, False otherwise - """ - - conn = sqlite3.connect("./db/infos.db") - c = conn.cursor() - - q = """SELECT * FROM users WHERE users.email = ?""" - - result = c.execute(q, (email,)).fetchall() - - if len(result) == 0: - return False - - q = """UPDATE users SET password = ? WHERE email = ?""" - - c.execute(q, (new_password, email)) - - conn.commit() - - return True - -def get_name_from_email(email): - """ - get_name_from_email: returns the name of the client based on the email - entered - - Args: - email (string): the email you are looking for - - Returns: - a string of the form "first last", empty string if the email doesn't - exist - - Example: - get_name_from_email("alex.wyc2098@gmail.com") --> Yicheng Wang - """ - - conn = sqlite3.connect("./db/infos.db") - c = conn.cursor() - - q = """SELECT users.first, users.last FROM users WHERE users.email = ?""" - - result = c.execute(q, (email,)).fetchall() - - if len(result) == 0: - return "" - - return result[0][0] + " " + result[0][1] - -def next_avaliable_id(): - """ - next_a: gives out the next avaliable id for the sites table - - Returns: - an integer that represents the next possible id - """ - - conn = sqlite3.connect("./db/infos.db") - c = conn.cursor() - - q = """SELECT sites.id FROM sites ORDER BY sites.id""" - - result = c.execute(q).fetchall() - - ##print result - - if (len(result) == 0): - return 0 - - if result[0][0] != 0: - #print "0 index: " + str(result[0][0]) - return 0 - - if (len(result) == 1): - return 1 - - for i in range(1, len(result)): - if result[i][0] - result[i - 1][0] > 1: - return result[i - 1][0] + 1 - - return len(result) - -def add_to_sites(email, title, site, comments, notes): - """ - add_to_sites: add a site to the user's list - - Args: - email (string): the user - title (string): title of the site - site (string): the html of the site - comments (string): the comments on the site - notes (string): the notes on the site - - Returns: - the id of the new site if successful, -1 otherwise - """ - - conn = sqlite3.connect("./db/infos.db") - c = conn.cursor() - - q = """INSERT INTO sites VALUES (?, ?, ?, ?, ?, ?, ?, ?)""" - - id = next_avaliable_id() - c.execute(q, (id, email, title, site, comments, notes, 0, int(time()))) # default permission is private - conn.commit() - return id - -def get_list_of_sites(email): - """ - get_list_of_sites: returns a list of sites based on a certain email - - Args: - email (string): the user - - Returns: - a list of sites formatted in the following manner ranked by last edited - time: - [(site-id1, site-title1, permission1), (site-id2, site-title2, permission2), ...] - """ - - conn = sqlite3.connect("./db/infos.db") - c = conn.cursor() - - q = """SELECT sites.id, sites.title, sites.shared - FROM sites - WHERE sites.email = ? - ORDER BY t DESC""" - - r = c.execute(q, (email,)).fetchall() - return r - -def get_site_on_id(email, id): - """ - get_site_on_id: get the site title, site, the notes and the comments - - Args: - email (string): the user - id (int): the site id - - Returns: - a tuple in the form of: - (title, site, notes, comments) - - or none if retrival was not successful - """ - - conn = sqlite3.connect("./db/infos.db") - c = conn.cursor() - - q = """SELECT sites.title, sites.site, sites.notes, sites.comments - FROM sites - WHERE sites.email = ? AND sites.id = ?""" - - r = c.execute(q, (email, id)).fetchall() - - ##print len(r) - - #for i in r: - # #print i[0] - ##print r - - if (len(r) != 1): - return None - - return r[0] - -def get_site_for_sharing(id): - """ - get_site_for_sharing: get the content of one site for sharing, returns None - if the site doesn't exist or is private - - Args: - id (int): the ID of the site - - Returns: - a tuple in the form of: - (title, site, notes, comments) - - or none if retrival was not successful - """ - - conn = sqlite3.connect("./db/infos.db") - c = conn.cursor() - - q = """SELECT sites.title, sites.site, sites.notes, - sites.comments - FROM sites - WHERE sites.id = ? AND sites.shared = 1""" - - r = c.execute(q, (id, )).fetchall() - - if (len(r) == 0): - return None - - else: - return r[0] - -def update_site(email, site_id, new_site, new_comments, new_notes): - """ - update_site: updates the site entry for the user - - Args: - email (string): the user - site_id (int): the id of the site in the database - new_site (string): updated markup for the site - new_comments (string): updated comments for the site - new_notes (string): updated notes for the site - - Returns: - True if successful, False otherwise - """ - - conn = sqlite3.connect("./db/infos.db") - c = conn.cursor() - - q = """SELECT * FROM sites WHERE sites.email = ? AND sites.id = ?""" - - result = c.execute(q, (email, site_id)).fetchall() - - if (len(result) == 0): - return False - - q = """UPDATE sites SET site = ?, comments = ?, notes = ?, t = ? WHERE id = ?""" - - c.execute(q, (new_site, new_comments, new_notes, int(time()), site_id)) - - conn.commit() - - return True - -def change_site_permission(email, id): - """ - change_site_permission: changes the permission of a site (public -> private - or private -> public) - - Args: - email (string): the user - id (int): the id of the site - - Returns: - True if successful, False if the id and the email doesn't match - """ - - conn = sqlite3.connect('./db/infos.db') - c = conn.cursor() - - q = """SELECT sites.shared - FROM sites - WHERE sites.email = ? AND sites.id = ?""" - - r = c.execute(q, (email, id)).fetchall() - - if (len(r) == 0): - return False - - q = """UPDATE sites - SET shared = ? - WHERE id = ?""" - - if (r[0][0] == 0): # used to be private, now becomes public - c.execute(q, (1, id)) - else: # used to be public, now becomes private - c.execute(q, (0, id)) - - conn.commit() - - return True - -def fork_shared_site(site_id, email): - """ - fork_shared_site: this makes a copy of the shared site with a specific - site_id within the user's private library - - Args: - site_id (int): the site_id of the shared site - email (string): the user who wishes to fork the site - - Returns: - The new ID of the forked site, or -1 if unsuccessful - """ - - conn = sqlite3.connect("./db/infos.db") - c = conn.cursor() - - q = """SELECT sites.shared, sites.title, sites.site, sites.comments, sites.notes - FROM sites - WHERE sites.id = ?""" +class DBManager: + """ + Class to manage the annotation database + """ + + def __init__(self, user_db_name, sites_db_name): + self.conn = MongoClient() + self.user_db = conn[user_db_name] + self.sites_db = conn[sites_db_name] + + def new_user(self, email, password_hash, first, last): + """ + new_user: adds a new user to the database, returns False if + unsuccessful + + Args: + email (string): email of the new user + password_hash (string): hex string of a hashed password + first (string): the first name of the user + last (string): the last name of the user + + Returns: + True if successfully added, False otherwise (email has already been + taken) + + Example: + >>> new_user("alex.wyc2098@gmail.com", "7b77e4d3de87423f0c98716ad54bd2f3", "Yicheng", "Wang") + True + >>> new_user("alex.wyc2098@gmail.com", "12354ab6a7e87af879dbadf87124faab", "Yicheng", "Wang") + False (because the email has already been registered) + """ + + ps = list(self.user_db.find({'email':email})) + + if ps == []: + user = {'email': email, + 'password': password_hash, + 'first': first, + 'last': last} + self.user_db.insert(user) + return True + + else: + return False + + def authenticate(self, email, password_hash): + """ + authenticate: authenticates an user login + + Args: + email (string): the email to authenticate + password_hash (string): the hash of the password that the user inputed + + Returns: + True if the two match, False otherwise + + Example: + >>> authenticate("alex.wyc2098@gmail.com", "7b77e4d3de87423f0c98716ad54bd2f3") + True + >>> authenticate("alex.wyc2098@gmail.com", "12354ab6a7e87af879dbadf87124faab") + False + """ + + result = list(self.user_db.find({'email':email, 'password':password_hash})) + + return (len(result) != 0) + + def update_pwd(self, email, new_password): + """ + update_pwd: updates password for an user + + Args: + email (string): the user + new_password (string): the new password + + Returns: + True if successful, False otherwise + """ + + return self.user_db.update( + {"email": email}, + { + '$set': + { + 'password': new_password + } + } + )['n'] == 1 - r = c.execute(q, (site_id,)).fetchall() - - if (len(r) == 0): - return -1 - - if r[0][0] == 0: - return -1 - - return add_to_sites(email, r[0][1], r[0][2], r[0][3], r[0][4]) - -def delete_site(email, site_id): - """ - delete_site: deletes a site from the user's "library" according to id - - Args: - email (string): the user - site_id (int): id of the site - - Returns: - True if successful, False otherwise - """ - - conn = sqlite3.connect("./db/infos.db") - c = conn.cursor() - - q = """SELECT * FROM sites WHERE sites.email = ? AND sites.id = ?""" - - result = c.execute(q, (email, site_id)).fetchall() - - if (len(result) == 0): - return False - - q = """DELETE FROM sites WHERE sites.id = ?""" - - c.execute(q, (site_id,)) - - conn.commit() - - return True - -def search_user_sites(email, search_string): - """ - search_user_sites: searches a specific user's pages for a specific set of - strings - - Args: - email (string): the user - search_string (string): the string to search - - Returns: - a list of abstracted articles in which each element is a dictionary of - the form: - { - 'index' : index-of-proximity (see search.py) (float), - 'id' : site id (int), - 'abstract' : abstracted site (string), - 'snippet' : list of matched words (see search.py) - } - - ordered by 'index' - """ - - ret_val = [] - search = search_string.split() - - conn = sqlite3.connect("./db/infos.db") - c = conn.cursor() - - q = """SELECT sites.id, sites.site, sites.title, sites.comments, sites.notes - FROM sites WHERE sites.email = ?""" - - result = c.execute(q, (email,)).fetchall() - - for i in result: - soup = BeautifulSoup(i[1] + i[3]) - site_cleaned = i[2] + ' ' + soup.get_text() + i[4] - snippets = get_snippets_from_site(site_cleaned, search) - - if (len(snippets) > 0): - index = get_index_of_proximity(site_cleaned, search) - abstract = abstract_site_from_words(site_cleaned, snippets) - - ret_val.append({ - 'index':index, - 'id':i[0], - 'abstract':abstract, - 'snippet':snippets, - 'title':i[2] - }) - - return sorted(ret_val, key=lambda entry: entry['index'], reverse=True) - -if __name__ == "__main__": - #print "new_user test" - #print new_user("alex.wyc2098@gmail.com", "12345", "Yicheng", "Wang") - #print new_user("alex.wyc2098@gmail.com", "dgjsadkfhsa", "Yicheng", "Wang") - #print new_user("alex.wyc2098@protonmail.ch", "ajdfsadfk", "Yicheng", "Wang") - - #print "\nauthentication test" - #print authenticate("alex.wyc2098@gmail.com", "12345") - #print authenticate("alex.wyc2098@protonmail.ch", "12345") - #print authenticate("asdf@asdf.asdf", "12435") - - #print "\nchange password test" - #print update_pwd("alex.wyc2098@gmail.com", "54321") - #print update_pwd("asdf@asdf.com", "12345") - - #print authenticate("alex.wyc2098@gmail.com", "12345") - #print authenticate("alex.wyc2098@gmail.com", "54321") - - #print "\nadd_to_sites test" - #print add_to_sites("alex.wyc2098@gmail.com", "123456789") - #print add_to_sites("alex.wyc2098@protonmail.ch", "fkjhsadgfkvasv") - #print add_to_sites("alex.wyc2098@gmail.com", "12hsadffghas") - - #print get_list_of_sites("alex.wyc2098@gmail.com") - #print get_list_of_sites("alex.wyc2098@protonmail.ch") - - sleep(1) - - #print "\nupdate_site test" - #print update_site("alex.wyc2098@gmail.com", "0", "new_site!") - - #print get_list_of_sites("alex.wyc2098@gmail.com") - - #print "\ndelete_site test" - #print delete_site("alex.wyc2098@gmail.com", 2) - #print delete_site("alex.wyc2098@gmail.com", 3) - - #print add_to_sites("alex.wyc2098@gmail.com", "this is the new site 2") - - #print get_list_of_sites("alex.wyc2098@gmail.com") - #print "searching" - search_user_sites("alex.wyc2098@gmail.com", 'asdf') From a06b18ba241fda77f330de6e818a19d3e3992bef Mon Sep 17 00:00:00 2001 From: Yicheng Wang Date: Mon, 26 Sep 2016 15:19:47 -0400 Subject: [PATCH 2/3] Finished backend conversion to Mongo --- database.py | 322 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 308 insertions(+), 14 deletions(-) diff --git a/database.py b/database.py index 186ecfe..defacb3 100644 --- a/database.py +++ b/database.py @@ -18,21 +18,24 @@ # Dev Log # Project Created: 2016-08-21 15:41 - Yicheng W. -from pymongo import MongoClient from time import time, sleep from search import * from bs4 import BeautifulSoup +from bson.objectid import ObjectId +import pymongo class DBManager: """ Class to manage the annotation database """ - def __init__(self, user_db_name, sites_db_name): - self.conn = MongoClient() - self.user_db = conn[user_db_name] - self.sites_db = conn[sites_db_name] + def __init__(self, database_name, user_collection_name, site_collection_name): + self.client = pymongo.MongoClient() + self.db = self.client[database_name] + self.user_collection = self.db[user_collection_name] + self.site_collection = self.db[site_collection_name] + # {{{ User Functions def new_user(self, email, password_hash, first, last): """ new_user: adds a new user to the database, returns False if @@ -49,20 +52,20 @@ def new_user(self, email, password_hash, first, last): taken) Example: - >>> new_user("alex.wyc2098@gmail.com", "7b77e4d3de87423f0c98716ad54bd2f3", "Yicheng", "Wang") + >>> new_user("test@email.com", "7b77e4d3de87423f0c98716ad54bd2f3", "Yicheng", "Wang") True - >>> new_user("alex.wyc2098@gmail.com", "12354ab6a7e87af879dbadf87124faab", "Yicheng", "Wang") + >>> new_user("test@email.com", "12354ab6a7e87af879dbadf87124faab", "Yicheng", "Wang") False (because the email has already been registered) """ - ps = list(self.user_db.find({'email':email})) + ps = list(self.user_collection.find({'email':email})) if ps == []: user = {'email': email, 'password': password_hash, 'first': first, 'last': last} - self.user_db.insert(user) + self.user_collection.insert(user) return True else: @@ -80,13 +83,13 @@ def authenticate(self, email, password_hash): True if the two match, False otherwise Example: - >>> authenticate("alex.wyc2098@gmail.com", "7b77e4d3de87423f0c98716ad54bd2f3") + >>> authenticate("test@email.com", "7b77e4d3de87423f0c98716ad54bd2f3") True - >>> authenticate("alex.wyc2098@gmail.com", "12354ab6a7e87af879dbadf87124faab") + >>> authenticate("test@email.com", "12354ab6a7e87af879dbadf87124faab") False """ - result = list(self.user_db.find({'email':email, 'password':password_hash})) + result = list(self.user_collection.find({'email':email, 'password':password_hash})) return (len(result) != 0) @@ -102,7 +105,7 @@ def update_pwd(self, email, new_password): True if successful, False otherwise """ - return self.user_db.update( + return self.user_collection.update_one( {"email": email}, { '$set': @@ -110,6 +113,297 @@ def update_pwd(self, email, new_password): 'password': new_password } } - )['n'] == 1 + ).matched_count == 1 + def get_name_from_email(self, email): + """ + get_name_from_email: returns the name of the client based on the email + entered + + Args: + email (string): the email you are looking for + + Returns: + a string of the form "first last", empty string if the email doesn't + exist + + Example: + get_name_from_email("test@email.com") --> Yicheng Wang + """ + + if self.user_collection.count({'email':email}) != 1: + return "" + + res = self.user_collection.find_one({'email': email}) + + return res['first'] + ' ' + res['last'] + + # }}} + # {{{ Sites Functions + def add_to_sites(self, email, title, site, comments = "", notes = ""): + """ + add_to_sites: add a site to the user's list + + Args: + email (string): the user + title (string): title of the site + site (string): the html source code of the site + comments (string): the comments on the site (default to empty string) + notes (string): the notes on the site (default to empty string) + + Returns: + the id of the new site if successful, -1 otherwise + """ + + if self.user_collection.count({"email": email}) != 1: # possible js/package sniffed atk + return -1 + + res = self.site_collection.insert_one( + { + 'email': email, + 'title': title, + 'site': site, + 'comments': comments, + 'notes': notes, + 'shared': False, + 'last_edited': time() + } + ) + + return res.inserted_id + + def get_list_of_sites(self, email): + """ + get_list_of_sites: returns a list of sites based on a certain email + + Args: + email (string): the user + + Returns: + a list of sites in dictionary format ranked by last edited + time: + [{ + 'email': email, + 'title': title, + 'site': site, + 'comments': comments, + 'notes': notes, + 'shared': False, + 'last_edited': time(), + '_id': ObjectId('hex string') + }, ...] + """ + + r = self.site_collection.find({ + 'email': email + }).sort("last_edited", pymongo.DESCENDING) + + return list(r) + + def get_site_on_id(self, email, id): + """ + get_site_on_id: get the site title, site, the notes and the comments + + Args: + email (string): the user + id (string): the site id + + Returns: + the site dictionary, as outlined by previous functions, + or none if retrival was not successful + """ + + return self.site_collection.find_one({ + '_id': ObjectId(id), + 'email': email + }) + + def update_site(self, email, site_id, new_site, new_comments, new_notes): + """ + update_site: updates the site entry for the user + + Args: + email (string): the user + site_id (string): the ObjectId of the site in the database + new_site (string): updated markup for the site + new_comments (string): updated comments for the site + new_notes (string): updated notes for the site + + Returns: + True if successful, False otherwise + """ + + return self.site_collection.update_one( + { + '_id': ObjectId(site_id), + 'email': email + }, + { + '$set': { + 'site': new_site, + 'comments': new_comments, + 'notes': new_notes, + 'last_edited': time() + } + } + ).matched_count == 1 + + def delete_site(self, email, site_id): + """ + delete_site: deletes a site from the user's "library" according to id + + Args: + email (string): the user + site_id (string): id of the site + + Returns: + True if successful, False otherwise + """ + return self.site_collection.find_one_and_delete( + { + '_id': ObjectId(site_id), + 'email': email + } + ) != None + + def change_site_permission(self, email, id): + """ + change_site_permission: changes the permission of a site (public -> private + or private -> public) + + Args: + email (string): the user + id (string): the id of the site + + Returns: + True if successful, False if the id and the email doesn't match + """ + + doc = self.site_collection.find_one( + { + '_id': ObjectId(id), + 'email': email + } + ) + + if doc: + return self.site_collection.update_one( + { + '_id': ObjectId(id), + 'email': email + }, + { + '$set': {'shared': not doc['shared']} + } + ).matched_count == 1 + + else: + return False + + def get_site_for_sharing(self, id): + """ + get_site_for_sharing: get the content of one site for sharing, returns None + if the site doesn't exist or is private + + Args: + id (string): the ID of the site + + Returns: + the site dictionary or none if retrival was not successful + """ + + return self.site_collection.find_one( + { + '_id': ObjectId(id), + 'shared': True + } + ) + + def fork_shared_site(self, site_id, email): + """ + fork_shared_site: this makes a copy of the shared site with a specific + site_id within the user's private library + + Args: + site_id (string): the site_id of the shared site + email (string): the user who wishes to fork the site + + Returns: + The new ID of the forked site, or -1 if unsuccessful + """ + + res = self.site_collection.find_one( + { + '_id': ObjectId(site_id), + 'shared': True + } + ) + + if res: + return self.add_to_sites(email, res['title'], res['site'], res['comments'], res['notes']) + + return -1 + + # }}} + +if __name__ == "__main__": + Test = DBManager('test_db', 'test_user_collection', 'test_site_collection') + print "new_user test" + print Test.new_user("test@email.com", "12345", "Yicheng", "Wang") + print Test.new_user("test@email.com", "dgjsadkfhsa", "Yicheng", "Wang") + print Test.new_user("test2@email.com", "ajdfsadfk", "Yicheng", "Wang") + + print "\nauthentication test" + print Test.authenticate("test@email.com", "12345") + print Test.authenticate("test2@email.com", "12345") + print Test.authenticate("asdf@asdf.asdf", "12435") + + print "\nchange password test" + print Test.update_pwd("test@email.com", "54321") + print Test.update_pwd("asdf@asdf.com", "12345") + + print Test.authenticate("test@email.com", "12345") + print Test.authenticate("test@email.com", "54321") + + print "\nget name test" + print Test.get_name_from_email("test@email.com") + print Test.get_name_from_email('test2@email.com') + print Test.get_name_from_email("noemail@lol.com") + + print "\nadd to sites test" + email = "test@email.com" + id = Test.add_to_sites(email, "123456789", "url_lol") + print id + print Test.add_to_sites("test2@email.com", "fkjhsadgfkvasv", "url_lol") + id2 = Test.add_to_sites(email, "12hsadffghas", "url_lol") + print id2 + print Test.add_to_sites("noemail", "asdjfkh", "asdjkf") + + print Test.get_list_of_sites(email) + print Test.get_list_of_sites("test2@email.com") + + print "\nget site on id test" + print Test.get_site_on_id(email, id) + print Test.get_site_on_id('noemail', id) + + print "\nupdate site test" + print Test.update_site('wrongemail@gmail.com', id, "no, html", "hello", "more stuff") + print Test.update_site(email, id, "no, html", "hello", "more stuff") + print Test.get_site_on_id(email, id) + + print "\nchange permission test and get site for sharing" + print Test.change_site_permission("lol", id) + print Test.change_site_permission(email, id) + print Test.get_site_on_id(email, id) + print Test.get_site_for_sharing(id) + print Test.change_site_permission(email, id) + print Test.get_site_on_id(email, id) + print Test.get_site_for_sharing(id) + + print "\ndelete site test" + print Test.delete_site("noemail", id2) + print Test.delete_site(email, id2) + print Test.get_list_of_sites(email) + Test.user_collection.remove() + Test.site_collection.remove() From d1d0a39fe4d2ee9fe1fb8c43d974cce225a1e49f Mon Sep 17 00:00:00 2001 From: Yicheng Wang Date: Fri, 28 Oct 2016 16:36:17 -0400 Subject: [PATCH 3/3] integrated DBManager into api --- api.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/api.py b/api.py index a514c9d..6ebac9d 100644 --- a/api.py +++ b/api.py @@ -23,7 +23,7 @@ from flask import Flask, request, render_template, session, redirect, url_for from werkzeug.contrib.fixers import ProxyFix -from database import * +from database import DBManager from functools import wraps from hashlib import sha256 import json @@ -38,6 +38,7 @@ import unicodedata app = Flask(__name__) +database = DBManager("marginalia", "users", "sites") def login_required(f): @wraps(f) @@ -126,7 +127,7 @@ def register(): m.update(password) passhash = m.hexdigest() - if new_user(email, passhash, first, last): + if database.new_user(email, passhash, first, last): return render_template("register.html", status = "success") # in register.html redirect them to login else: @@ -145,9 +146,9 @@ def login(): m.update(password) passhash = m.hexdigest() - if (authenticate(email, passhash)): + if (database.authenticate(email, passhash)): session["email"] = email - session['name'] = get_name_from_email(email) + session['name'] = database.get_name_from_email(email) return redirect(url_for("home")) else: return render_template("login.html", err = "Incorrect email/password combination") @@ -169,7 +170,7 @@ def forget_pwd(): m.update(new_pass) passhash = m.hexdigest() - if not update_pwd(email, passhash): + if not database.update_pwd(email, passhash): return render_template("forget_pwd.html", err = "The email you entered is not registered") s = SMTP("smtp.gmail.com", 587) @@ -213,7 +214,7 @@ def change_pwd(): m = sha256() m.update(old_password) passhash = m.hexdigest() - if (authenticate(email,passhash)): + if (database.authenticate(email,passhash)): new_password = request.form['newpass'] confirmed = request.form['confirm'] @@ -223,7 +224,7 @@ def change_pwd(): m = sha256() m.update(new_password) newhashed = m.hexdigest() - changed = update_pwd(email, newhashed) + changed = database.update_pwd(email, newhashed) if changed: return render_template("change_pwd.html", status = "success", name = session['name']) else: @@ -235,14 +236,14 @@ def change_pwd(): @login_required def view_static(): email = session['email'] - list_of_sites = get_list_of_sites(email) + list_of_sites = database.get_list_of_sites(email) return render_template("view.html", sites = list_of_sites, name = session['name']) @app.route("/view/") # grab a specific story based on id @login_required def view_site(id): email = session['email'] - site = get_site_on_id(email, id) + site = database.get_site_on_id(email, id) if (site): return render_template("view_one.html", site = site, name = session['name']) @@ -270,7 +271,7 @@ def logout(): @app.route("/share/") # reders the site if shares, gives out error otherwise def share(id): - site = get_site_for_sharing(id) + site = database.get_site_for_sharing(id) if site: if 'name' in session: @@ -329,7 +330,7 @@ def api_add_site(): htmlsite = '

' + title + "

\n

" + author + '

' + htmlsite + '

' + 'Original Site

' - new_id = add_to_sites(email, title, htmlsite, "", "") + new_id = database.add_to_sites(email, title, htmlsite, "", "") #print new_id if new_id != -1: @@ -346,7 +347,7 @@ def api_update_site(id): new_site = request.form['site'] new_comments = request.form['comment'] new_notes = request.form['note'] - if update_site(email, id, new_site, new_comments, new_notes): + if database.update_site(email, id, new_site, new_comments, new_notes): return json.dumps({"status": 'success', 'msg': 'Your marks have been updated'}) return json.dumps({'status': 'failure', 'msg': "Something went wrong :("}) @@ -359,7 +360,7 @@ def api_change_perm(): email = session['email'] id = int(request.form['id']) - if change_site_permission(email, id): + if database.change_site_permission(email, id): return json.dumps({"status": 'success', 'msg': "The permission of your site has been successfully changed", 'to': request.form['to'], 'id': id}) return json.dumps({'status': 'failure', 'msg': 'Something went wrong :('}) @@ -373,7 +374,7 @@ def api_delete_site(): email = session['email'] id = request.form['id'] - if delete_site(email, id): + if database.delete_site(email, id): return json.dumps({'status': 'success', 'msg': 'Your site has been successfully deleted'}) return json.dumps({'status': 'failure', 'msg': 'Something went wrong :('}) @@ -385,7 +386,7 @@ def fork(): email = session['email'] id = int(request.form['id']) - new_id = fork_shared_site(id, email) + new_id = database.fork_shared_site(id, email) if new_id != -1: return json.dumps({'status': 'success', 'msg':'The site has been successfully added to your own library', 'id': new_id})