From dffe8e8df04ff8ef2a243aa3527bf3b3e6577b44 Mon Sep 17 00:00:00 2001 From: dumbmachine Date: Wed, 2 Sep 2020 08:34:23 +0530 Subject: [PATCH 1/7] initial support for uploading via frontend --- backend/__init__.py | 2 + backend/routes/data.py | 57 +++++++++++++++++++ .../src/containers/forms/uploadDataForm.js | 43 ++++++++++++++ frontend/src/containers/modal.js | 4 ++ frontend/src/pages/admin.js | 19 +++++++ 5 files changed, 125 insertions(+) create mode 100644 frontend/src/containers/forms/uploadDataForm.js diff --git a/backend/__init__.py b/backend/__init__.py index 197d8d0..fb62fd7 100644 --- a/backend/__init__.py +++ b/backend/__init__.py @@ -13,7 +13,9 @@ def create_app(test_config=None): + from flask_cors import CORS #ratin app = Flask(__name__, instance_relative_config=True) + cors = CORS(app) #ratin app.config.from_object(Config) app.logger.info(app.config["UPLOAD_FOLDER"]) diff --git a/backend/routes/data.py b/backend/routes/data.py index 0984c0b..24ba378 100644 --- a/backend/routes/data.py +++ b/backend/routes/data.py @@ -90,3 +90,60 @@ def add_data(): ), 201, ) + + +@api.route("/datazip", methods=["POST"]) +# @jwt_required +def add_datazip(): + # identity = get_jwt_identity() + # request_user = User.query.filter_by(username=identity["username"]).first() + # app.logger.info(f"Current user is: {request_user}") + # is_admin = True if request_user.role.role == "admin" else False + + # if is_admin == False: + # return jsonify(message="Unauthorized access!"), 401 + + # if not request.is_json: + # return jsonify(message="Missing JSON in request"), 400 + + from zipfile import ZipFile + files = request.files.items() + for filename, audio_file in files: + # extract all the files and validate them + with ZipFile(audio_file, "r") as zip_ref: + for cmprsd_file in zip_ref.namelist(): + with zip_ref.open(cmprsd_file) as file_test: + pass + # zip_ref.extractall("test") + + return jsonify( + resp="HAHA" + ) + + +def validate_audio_file(audio_file): + """Validate the audio file before adding saving it + + Args: + audio_file (werkzeug.datastructures.FileStorage): The file to be validated before insertion + """ + original_filename = secure_filename(audio_file.filename) + extension = Path(original_filename).suffix.lower() + + if len(extension) > 1 and extension[1:] not in ALLOWED_EXTENSIONS: + return False + + filename = f"{str(uuid.uuid4().hex)}{extension}" + file_path = Path(app.config["UPLOAD_FOLDER"]).joinpath(filename) + + return file_path.as_posix() + + +def validate_annotation_file(annotation_file): + """[summary] + + Args: + annotation_file ([type]): [description] + """ + + return \ No newline at end of file diff --git a/frontend/src/containers/forms/uploadDataForm.js b/frontend/src/containers/forms/uploadDataForm.js new file mode 100644 index 0000000..bc34dcf --- /dev/null +++ b/frontend/src/containers/forms/uploadDataForm.js @@ -0,0 +1,43 @@ +import React from 'react'; +import { withRouter } from "react-router"; +import Dropzone from 'react-dropzone-uploader'; +import { withStore } from "@spyna/react-store"; +import 'react-dropzone-uploader/dist/styles.css' + +class UploadDataForm extends React.Component { + constructor(props) { + super(props); + + const projectId = this.props.projectId; + + this.initialState = { + projectId, + addDataUrl: `/api/datazip`, + }; + + this.state = Object.assign({}, this.initialState); + console.log(this.props); + } + + getUploadParams = ({ file, meta }) => { + const body = new FormData() + body.append('fileField', file) + body.append('temp', 'ratin') + return { url: this.state.addDataUrl, body } + } + handleChangeStatus = ({ meta, file }, status) => { console.log(status, meta, file) } + + render() { + return ( + + ) + } + +} + +export default withStore(withRouter(UploadDataForm)); diff --git a/frontend/src/containers/modal.js b/frontend/src/containers/modal.js index bab258f..5cb999c 100644 --- a/frontend/src/containers/modal.js +++ b/frontend/src/containers/modal.js @@ -2,6 +2,7 @@ import React from "react"; import Modal from "react-bootstrap/Modal"; import CreateUserForm from "./forms/createUserForm"; +import UploadDataForm from "./forms/uploadDataForm"; import EditUserForm from "./forms/editUserForm"; import CreateProjectForm from "./forms/createProjectForm"; import CreateLabelForm from "./forms/createLabelForm"; @@ -28,6 +29,9 @@ const FormModal = (props) => { {props.formType === "NEW_USER" ? : null} {props.formType === "NEW_PROJECT" ? : null} + {props.formType === "NEW_DATA" ? ( + + ) : null} {props.formType === "EDIT_USER" ? ( ) : null} diff --git a/frontend/src/pages/admin.js b/frontend/src/pages/admin.js index 6d65c13..865be26 100644 --- a/frontend/src/pages/admin.js +++ b/frontend/src/pages/admin.js @@ -9,6 +9,7 @@ import { faUserPlus, faTags, faDownload, + faUpload, } from "@fortawesome/free-solid-svg-icons"; import { IconButton } from "../components/button"; import Loader from "../components/loader"; @@ -165,6 +166,12 @@ class Admin extends React.Component { }); } + handleUploadData(e, projectName, projectId) { + console.log(projectId, projectName); + this.setModalShow(true); + this.setState({ formType: "NEW_DATA", title: "Create New Project", projectId, projectName }); + } + setModalShow(modalShow) { this.setState({ modalShow }); } @@ -260,6 +267,18 @@ class Admin extends React.Component { ) } /> + + this.handleUploadData( + e, + project["name"], + project["project_id"] + ) + } + /> Date: Wed, 2 Sep 2020 12:18:57 +0530 Subject: [PATCH 2/7] uploading and segmentation uploading works now --- backend/routes/data.py | 127 ++++++++++++------ .../src/containers/forms/uploadDataForm.js | 5 +- frontend/src/containers/modal.js | 2 +- frontend/src/pages/admin.js | 7 +- 4 files changed, 96 insertions(+), 45 deletions(-) diff --git a/backend/routes/data.py b/backend/routes/data.py index 3f3d4ea..123f1c9 100644 --- a/backend/routes/data.py +++ b/backend/routes/data.py @@ -1,8 +1,10 @@ +import shutil import json import sqlalchemy as sa import uuid from pathlib import Path +from zipfile import ZipFile from flask import jsonify, flash, redirect, url_for, request, send_from_directory from flask_jwt_extended import jwt_required, get_jwt_identity @@ -15,19 +17,22 @@ from . import api +ALLOWED_COMPRESSED_EXTENSIONS = ["zip"] +ALLOWED_ANNOTATION_EXTENSIONS = ["json"] ALLOWED_EXTENSIONS = ["wav", "mp3", "ogg"] - @api.route("/audio/", methods=["GET"]) @jwt_required def send_audio_file(file_name): return send_from_directory(app.config["UPLOAD_FOLDER"], file_name) -def validate_segmentation(segment): +def validate_segmentation(segment, without_data=False): """Validate the segmentation before accepting the annotation's upload from users """ required_key = {"start_time", "end_time", "transcription"} + if without_data: + required_key.add("filename") # requried to search the datapoint if set(required_key).issubset(segment.keys()): return True @@ -191,7 +196,7 @@ def add_data(): 201, ) - +# TODO: Add authentication @api.route("/datazip", methods=["POST"]) # @jwt_required def add_datazip(): @@ -206,44 +211,88 @@ def add_datazip(): # if not request.is_json: # return jsonify(message="Missing JSON in request"), 400 - from zipfile import ZipFile + pid = request.form.get("projectId", None) + uid = request.form.get("userId", None) + project = Project.query.filter_by(id=pid).first() + user = User.query.filter_by(username=uid).first() + files = request.files.items() - for filename, audio_file in files: - # extract all the files and validate them - with ZipFile(audio_file, "r") as zip_ref: - for cmprsd_file in zip_ref.namelist(): - with zip_ref.open(cmprsd_file) as file_test: - pass - # zip_ref.extractall("test") + + app.logger.info(f"user: {user} / project: {project}") + + for _, audio_file in files: + filename = audio_file.filename + filename_ext = Path(filename).suffix.lower() + if filename_ext[1:] in ALLOWED_COMPRESSED_EXTENSIONS: + app.logger.info(f"Compressed file in consideration: {filename}") + with ZipFile(audio_file, "r") as zip_obj: + for cmprsd_filename in zip_obj.namelist(): + app.logger.info(f"cmpressed files are: {cmprsd_filename}") + cmprsd_extension = Path(cmprsd_filename).suffix.lower() + if cmprsd_extension[1:] in ALLOWED_EXTENSIONS: + zip_obj.extract( + cmprsd_filename, + app.config["UPLOAD_FOLDER"] + ) + tfilename = f"{str(uuid.uuid4().hex)}{cmprsd_extension}" + shutil.move( + Path(app.config["UPLOAD_FOLDER"]).joinpath(cmprsd_filename), Path( + app.config["UPLOAD_FOLDER"]).joinpath(tfilename) + ) + data = Data( + project_id=project.id, + filename=tfilename, + original_filename=cmprsd_filename, + reference_transcription="", + is_marked_for_review=True, + assigned_user_id=user.id, + ) + + + db.session.add(data) + db.session.flush() + db.session.commit() + + elif cmprsd_extension[1:] in ALLOWED_ANNOTATION_EXTENSIONS: + pass + # reading the file, temp store it + temp_loc = Path(app.config["UPLOAD_FOLDER"] + ).joinpath(cmprsd_filename) + zip_obj.extract(cmprsd_filename, + app.config["UPLOAD_FOLDER"]) + segmentations = json.load(open(temp_loc, "r")) + for _segment in segmentations: + validated = validate_segmentation(_segment, without_data=True) + + # if not validated: + # continue # skip this datapoint + + if validated: + try: + data = Data.query.filter_by( + project_id=pid, original_filename=_segment['filename']).first() + + if data.id: + app.logger.info(f"seraching for segments: {data.id}") + + new_segment = generate_segmentation( + data_id=data.id, + project_id=project.id, + end_time=_segment["end_time"], + start_time=_segment["start_time"], + transcription=_segment["transcription"], + annotations=_segment.get("annotations", {}) + ) + + app.logger.info( + f"will try to add data in: {data}") + data.set_segmentations([new_segment]) + app.logger.info(f"new_segment: {new_segment.data_id}") + db.session.refresh(data) + db.session.commit() + except Exception as e: + app.logger.info(f"Error {e} for data: {data.id}") return jsonify( resp="HAHA" ) - - -def validate_audio_file(audio_file): - """Validate the audio file before adding saving it - - Args: - audio_file (werkzeug.datastructures.FileStorage): The file to be validated before insertion - """ - original_filename = secure_filename(audio_file.filename) - extension = Path(original_filename).suffix.lower() - - if len(extension) > 1 and extension[1:] not in ALLOWED_EXTENSIONS: - return False - - filename = f"{str(uuid.uuid4().hex)}{extension}" - file_path = Path(app.config["UPLOAD_FOLDER"]).joinpath(filename) - - return file_path.as_posix() - - -def validate_annotation_file(annotation_file): - """[summary] - - Args: - annotation_file ([type]): [description] - """ - - return \ No newline at end of file diff --git a/frontend/src/containers/forms/uploadDataForm.js b/frontend/src/containers/forms/uploadDataForm.js index bc34dcf..bb4b7ff 100644 --- a/frontend/src/containers/forms/uploadDataForm.js +++ b/frontend/src/containers/forms/uploadDataForm.js @@ -9,8 +9,10 @@ class UploadDataForm extends React.Component { super(props); const projectId = this.props.projectId; + const userId = this.props.userId; this.initialState = { + userId, projectId, addDataUrl: `/api/datazip`, }; @@ -22,7 +24,8 @@ class UploadDataForm extends React.Component { getUploadParams = ({ file, meta }) => { const body = new FormData() body.append('fileField', file) - body.append('temp', 'ratin') + body.append('userId', this.state.userId) + body.append('projectId', this.state.projectId) return { url: this.state.addDataUrl, body } } handleChangeStatus = ({ meta, file }, status) => { console.log(status, meta, file) } diff --git a/frontend/src/containers/modal.js b/frontend/src/containers/modal.js index 5cb999c..e6cb7a5 100644 --- a/frontend/src/containers/modal.js +++ b/frontend/src/containers/modal.js @@ -30,7 +30,7 @@ const FormModal = (props) => { {props.formType === "NEW_USER" ? : null} {props.formType === "NEW_PROJECT" ? : null} {props.formType === "NEW_DATA" ? ( - + ) : null} {props.formType === "EDIT_USER" ? ( diff --git a/frontend/src/pages/admin.js b/frontend/src/pages/admin.js index 865be26..13fe5f8 100644 --- a/frontend/src/pages/admin.js +++ b/frontend/src/pages/admin.js @@ -166,10 +166,9 @@ class Admin extends React.Component { }); } - handleUploadData(e, projectName, projectId) { - console.log(projectId, projectName); + handleUploadData(e, userId, projectId) { this.setModalShow(true); - this.setState({ formType: "NEW_DATA", title: "Create New Project", projectId, projectName }); + this.setState({ formType: "NEW_DATA", title: "Create New Project", userId, projectId }); } setModalShow(modalShow) { @@ -274,7 +273,7 @@ class Admin extends React.Component { onClick={(e) => this.handleUploadData( e, - project["name"], + "admin", project["project_id"] ) } From ffc90d736dd98f50a37f6d18e802ed7c331761b1 Mon Sep 17 00:00:00 2001 From: dumbmachine Date: Wed, 2 Sep 2020 13:08:38 +0530 Subject: [PATCH 3/7] minor logging changes --- backend/routes/data.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/backend/routes/data.py b/backend/routes/data.py index 123f1c9..3b787dc 100644 --- a/backend/routes/data.py +++ b/backend/routes/data.py @@ -212,9 +212,9 @@ def add_datazip(): # return jsonify(message="Missing JSON in request"), 400 pid = request.form.get("projectId", None) - uid = request.form.get("userId", None) + username = request.form.get("userId", None) project = Project.query.filter_by(id=pid).first() - user = User.query.filter_by(username=uid).first() + user = User.query.filter_by(username=username).first() files = request.files.items() @@ -260,12 +260,14 @@ def add_datazip(): ).joinpath(cmprsd_filename) zip_obj.extract(cmprsd_filename, app.config["UPLOAD_FOLDER"]) + segmentations = json.load(open(temp_loc, "r")) + for _segment in segmentations: validated = validate_segmentation(_segment, without_data=True) - # if not validated: - # continue # skip this datapoint + if not validated: + continue # skip this datapoint if validated: try: @@ -273,7 +275,6 @@ def add_datazip(): project_id=pid, original_filename=_segment['filename']).first() if data.id: - app.logger.info(f"seraching for segments: {data.id}") new_segment = generate_segmentation( data_id=data.id, @@ -284,12 +285,10 @@ def add_datazip(): annotations=_segment.get("annotations", {}) ) - app.logger.info( - f"will try to add data in: {data}") data.set_segmentations([new_segment]) app.logger.info(f"new_segment: {new_segment.data_id}") - db.session.refresh(data) db.session.commit() + db.session.refresh(data) except Exception as e: app.logger.info(f"Error {e} for data: {data.id}") From b0beb5b6eb88c958f447001c777a17faebe45c2f Mon Sep 17 00:00:00 2001 From: dumbmachine Date: Wed, 2 Sep 2020 13:18:08 +0530 Subject: [PATCH 4/7] added dependency --- frontend/package.json | 1 + frontend/src/pages/admin.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/package.json b/frontend/package.json index b5ae35c..ea3b912 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -12,6 +12,7 @@ "@testing-library/react": "^9.3.2", "@testing-library/user-event": "^7.1.2", "axios": "^0.19.2", + "react-dropzone-uploader": "^2.11.0", "bootstrap": "^4.4.1", "jquery": "^3.5.0", "popper.js": "^1.16.1", diff --git a/frontend/src/pages/admin.js b/frontend/src/pages/admin.js index 13fe5f8..545ddc8 100644 --- a/frontend/src/pages/admin.js +++ b/frontend/src/pages/admin.js @@ -273,7 +273,7 @@ class Admin extends React.Component { onClick={(e) => this.handleUploadData( e, - "admin", + "admin", // since admin is the uploader project["project_id"] ) } From cfe05291dd93df50bbd5333c8ec0e2e4a2d7da98 Mon Sep 17 00:00:00 2001 From: dumbmachine Date: Wed, 2 Sep 2020 16:04:14 +0530 Subject: [PATCH 5/7] all tests have passed --- backend/routes/data.py | 60 +++++++++---------- .../src/containers/forms/uploadDataForm.js | 6 +- frontend/src/containers/modal.js | 2 +- frontend/src/pages/admin.js | 10 +++- 4 files changed, 39 insertions(+), 39 deletions(-) diff --git a/backend/routes/data.py b/backend/routes/data.py index 3b787dc..9f3f583 100644 --- a/backend/routes/data.py +++ b/backend/routes/data.py @@ -212,7 +212,7 @@ def add_datazip(): # return jsonify(message="Missing JSON in request"), 400 pid = request.form.get("projectId", None) - username = request.form.get("userId", None) + username = request.form.get("username", None) project = Project.query.filter_by(id=pid).first() user = User.query.filter_by(username=username).first() @@ -220,45 +220,44 @@ def add_datazip(): app.logger.info(f"user: {user} / project: {project}") - for _, audio_file in files: - filename = audio_file.filename - filename_ext = Path(filename).suffix.lower() - if filename_ext[1:] in ALLOWED_COMPRESSED_EXTENSIONS: - app.logger.info(f"Compressed file in consideration: {filename}") - with ZipFile(audio_file, "r") as zip_obj: - for cmprsd_filename in zip_obj.namelist(): - app.logger.info(f"cmpressed files are: {cmprsd_filename}") - cmprsd_extension = Path(cmprsd_filename).suffix.lower() - if cmprsd_extension[1:] in ALLOWED_EXTENSIONS: + for _, zip_file in files: + zip_filename = zip_file.filename + zip__ext = Path(zip_filename).suffix.lower() + if zip__ext[1:] in ALLOWED_COMPRESSED_EXTENSIONS: + app.logger.info( + f"File to be decompressed: {zip_filename}") + with ZipFile(zip_file, "r") as zip_obj: + for cfilename in zip_obj.namelist(): + cfile_extension = Path(cfilename).suffix.lower() + app.logger.info(f" compressed file: {cfilename}") + if cfile_extension[1:] in ALLOWED_EXTENSIONS: zip_obj.extract( - cmprsd_filename, + cfilename, app.config["UPLOAD_FOLDER"] ) - tfilename = f"{str(uuid.uuid4().hex)}{cmprsd_extension}" + filename = f"{str(uuid.uuid4().hex)}{cfile_extension}" + shutil.move( - Path(app.config["UPLOAD_FOLDER"]).joinpath(cmprsd_filename), Path( - app.config["UPLOAD_FOLDER"]).joinpath(tfilename) + Path(app.config["UPLOAD_FOLDER"]).joinpath(cfilename), + Path(app.config["UPLOAD_FOLDER"]).joinpath(filename) ) data = Data( project_id=project.id, - filename=tfilename, - original_filename=cmprsd_filename, + filename=filename, + original_filename=cfilename, reference_transcription="", is_marked_for_review=True, assigned_user_id=user.id, ) - - db.session.add(data) + db.session.add(data) db.session.flush() - db.session.commit() + # db.session.commit() - elif cmprsd_extension[1:] in ALLOWED_ANNOTATION_EXTENSIONS: - pass - # reading the file, temp store it + elif cfile_extension[1:] in ALLOWED_ANNOTATION_EXTENSIONS: temp_loc = Path(app.config["UPLOAD_FOLDER"] - ).joinpath(cmprsd_filename) - zip_obj.extract(cmprsd_filename, + ).joinpath(cfilename) + zip_obj.extract(cfilename, app.config["UPLOAD_FOLDER"]) segmentations = json.load(open(temp_loc, "r")) @@ -267,15 +266,14 @@ def add_datazip(): validated = validate_segmentation(_segment, without_data=True) if not validated: - continue # skip this datapoint + continue # raise error and skip transaction if validated: try: data = Data.query.filter_by( project_id=pid, original_filename=_segment['filename']).first() - + if data.id: - new_segment = generate_segmentation( data_id=data.id, project_id=project.id, @@ -284,13 +282,11 @@ def add_datazip(): transcription=_segment["transcription"], annotations=_segment.get("annotations", {}) ) - - data.set_segmentations([new_segment]) - app.logger.info(f"new_segment: {new_segment.data_id}") db.session.commit() - db.session.refresh(data) except Exception as e: app.logger.info(f"Error {e} for data: {data.id}") + + db.session.commit() return jsonify( resp="HAHA" diff --git a/frontend/src/containers/forms/uploadDataForm.js b/frontend/src/containers/forms/uploadDataForm.js index bb4b7ff..8006267 100644 --- a/frontend/src/containers/forms/uploadDataForm.js +++ b/frontend/src/containers/forms/uploadDataForm.js @@ -8,11 +8,11 @@ class UploadDataForm extends React.Component { constructor(props) { super(props); + const userName = this.props.userName; const projectId = this.props.projectId; - const userId = this.props.userId; this.initialState = { - userId, + userName, projectId, addDataUrl: `/api/datazip`, }; @@ -24,7 +24,7 @@ class UploadDataForm extends React.Component { getUploadParams = ({ file, meta }) => { const body = new FormData() body.append('fileField', file) - body.append('userId', this.state.userId) + body.append('username', this.state.userName) body.append('projectId', this.state.projectId) return { url: this.state.addDataUrl, body } } diff --git a/frontend/src/containers/modal.js b/frontend/src/containers/modal.js index e6cb7a5..d76dd85 100644 --- a/frontend/src/containers/modal.js +++ b/frontend/src/containers/modal.js @@ -30,7 +30,7 @@ const FormModal = (props) => { {props.formType === "NEW_USER" ? : null} {props.formType === "NEW_PROJECT" ? : null} {props.formType === "NEW_DATA" ? ( - + ) : null} {props.formType === "EDIT_USER" ? ( diff --git a/frontend/src/pages/admin.js b/frontend/src/pages/admin.js index 545ddc8..51a0bac 100644 --- a/frontend/src/pages/admin.js +++ b/frontend/src/pages/admin.js @@ -166,9 +166,13 @@ class Admin extends React.Component { }); } - handleUploadData(e, userId, projectId) { + handleUploadData(e, userName, projectId) { this.setModalShow(true); - this.setState({ formType: "NEW_DATA", title: "Create New Project", userId, projectId }); + this.setState({ + formType: "NEW_DATA", + title: "Create New Project", + userId: userName, // userName: userName does not work :( + projectId: projectId }); } setModalShow(modalShow) { @@ -273,7 +277,7 @@ class Admin extends React.Component { onClick={(e) => this.handleUploadData( e, - "admin", // since admin is the uploader + "admin", project["project_id"] ) } From 49b4f070dca3a2c7ab50ea043b44d30f981c1b4d Mon Sep 17 00:00:00 2001 From: dumbmachine Date: Sun, 13 Sep 2020 03:42:11 +0530 Subject: [PATCH 6/7] upload better --- backend/routes/data.py | 294 ++++++++++++------ backend/routes/projects.py | 114 ++++++- .../src/containers/forms/uploadDataForm.js | 70 +++-- frontend/src/pages/admin.js | 6 +- 4 files changed, 358 insertions(+), 126 deletions(-) diff --git a/backend/routes/data.py b/backend/routes/data.py index 9f3f583..de7cb73 100644 --- a/backend/routes/data.py +++ b/backend/routes/data.py @@ -1,3 +1,4 @@ +import os import shutil import json import sqlalchemy as sa @@ -196,98 +197,209 @@ def add_data(): 201, ) -# TODO: Add authentication -@api.route("/datazip", methods=["POST"]) -# @jwt_required -def add_datazip(): - # identity = get_jwt_identity() - # request_user = User.query.filter_by(username=identity["username"]).first() - # app.logger.info(f"Current user is: {request_user}") - # is_admin = True if request_user.role.role == "admin" else False - # if is_admin == False: - # return jsonify(message="Unauthorized access!"), 401 +def files_from_zip(zip_file): + """Generator for getting files from the a zip file - # if not request.is_json: - # return jsonify(message="Missing JSON in request"), 400 + Returns: + Generator: if valid, generator of files in the zip + False: if the file is invalid + """ + with ZipFile(zip_file, "r") as zip_obj: + for cfilename in zip_obj.namelist(): + cfile_extension = Path(cfilename).suffix.lower() + if cfile_extension[1:] in ALLOWED_EXTENSIONS: + zip_obj.extract( + cfilename, + Path(app.config["UPLOAD_FOLDER"]) + ) + yield Path(app.config["UPLOAD_FOLDER"]).joinpath(cfilename), "data" + elif cfile_extension[1:] in ALLOWED_ANNOTATION_EXTENSIONS: + zip_obj.extract( + cfilename, + Path(app.config["UPLOAD_FOLDER"]) + ) + yield Path(app.config["UPLOAD_FOLDER"]).joinpath(cfilename), "annotation" + +def file_to_database( + db, + user, + project, + audio_file, + is_marked_for_review, + reference_transcription, + compressed_file=False +): + """Add data to database and save a copy in the /uploads folder - pid = request.form.get("projectId", None) - username = request.form.get("username", None) - project = Project.query.filter_by(id=pid).first() - user = User.query.filter_by(username=username).first() + TODO: + - delete compressed file is there was some error + """ + try: + + if compressed_file: + original_filename = os.path.basename(audio_file) + extension = Path(original_filename).suffix.lower() + filename = f"{str(uuid.uuid4().hex)}{extension}" + from_path = Path(app.config["UPLOAD_FOLDER"]).joinpath(original_filename) + to_path = Path(app.config["UPLOAD_FOLDER"]).joinpath(filename) + shutil.move(from_path, to_path) + else: + original_filename = secure_filename(audio_file.filename) + extension = Path(original_filename).suffix.lower() + filename = f"{str(uuid.uuid4().hex)}{extension}" + file_path = Path(app.config["UPLOAD_FOLDER"]).joinpath(filename) + audio_file.save(file_path.as_posix()) - files = request.files.items() - - app.logger.info(f"user: {user} / project: {project}") - - for _, zip_file in files: - zip_filename = zip_file.filename - zip__ext = Path(zip_filename).suffix.lower() - if zip__ext[1:] in ALLOWED_COMPRESSED_EXTENSIONS: - app.logger.info( - f"File to be decompressed: {zip_filename}") - with ZipFile(zip_file, "r") as zip_obj: - for cfilename in zip_obj.namelist(): - cfile_extension = Path(cfilename).suffix.lower() - app.logger.info(f" compressed file: {cfilename}") - if cfile_extension[1:] in ALLOWED_EXTENSIONS: - zip_obj.extract( - cfilename, - app.config["UPLOAD_FOLDER"] - ) - filename = f"{str(uuid.uuid4().hex)}{cfile_extension}" - - shutil.move( - Path(app.config["UPLOAD_FOLDER"]).joinpath(cfilename), - Path(app.config["UPLOAD_FOLDER"]).joinpath(filename) - ) - data = Data( - project_id=project.id, - filename=filename, - original_filename=cfilename, - reference_transcription="", - is_marked_for_review=True, - assigned_user_id=user.id, - ) - - db.session.add(data) - db.session.flush() - # db.session.commit() - - elif cfile_extension[1:] in ALLOWED_ANNOTATION_EXTENSIONS: - temp_loc = Path(app.config["UPLOAD_FOLDER"] - ).joinpath(cfilename) - zip_obj.extract(cfilename, - app.config["UPLOAD_FOLDER"]) - - segmentations = json.load(open(temp_loc, "r")) - - for _segment in segmentations: - validated = validate_segmentation(_segment, without_data=True) - - if not validated: - continue # raise error and skip transaction - - if validated: - try: - data = Data.query.filter_by( - project_id=pid, original_filename=_segment['filename']).first() - - if data.id: - new_segment = generate_segmentation( - data_id=data.id, - project_id=project.id, - end_time=_segment["end_time"], - start_time=_segment["start_time"], - transcription=_segment["transcription"], - annotations=_segment.get("annotations", {}) - ) - db.session.commit() - except Exception as e: - app.logger.info(f"Error {e} for data: {data.id}") - - db.session.commit() - - return jsonify( - resp="HAHA" - ) + data = Data( + project_id=project.id, + filename=filename, + original_filename=original_filename, + reference_transcription=reference_transcription, + is_marked_for_review=is_marked_for_review, + assigned_user_id=user.id, + ) + db.session.add(data) + db.session.flush() + + return True + except Exception as e: + if compressed_file: + shutil.rmtree(path=from_path, ignore_errors=True) + shutil.rmtree(path=to_path, ignore_errors=True) + app.logger.error(e) + app.logger.info(f"the error was: { e.with_traceback }") + return False + + +def annotation_to_database( + project, + annotation_file +): + """Add segmentation to database from a json + """ + try: + segmentations = json.load(annotation_file) + for _segment in segmentations: + validated = validate_segmentation( + _segment, without_data=True + ) + if validated: + data = Data.query.filter_by( + project_id=project.id, + original_filename=_segment['filename'] + ).first() + + if data: + new_segment = generate_segmentation( + data_id=data.id, + project_id=project.id, + end_time=_segment["end_time"], + start_time=_segment["start_time"], + transcription=_segment["transcription"], + annotations=_segment.get( + "annotations", {}) + ) + + return True + return False + except Exception as e: + app.logger.error(e) + return False + +# @api.route("/datazip", methods=["POST"]) +# @jwt_required +# def add_datazip(): +# identity = get_jwt_identity() +# app.logger.info(f"Current user is: {identity}") +# request_user = User.query.filter_by(username=identity["username"]).first() +# app.logger.info(f"Current user is: {request_user}") +# is_admin = True if request_user.role.role == "admin" else False + +# if is_admin == False: +# return jsonify(message="Unauthorized access!"), 401 + + +# project = Project.query.filter_by(id=pid).first() +# user = User.query.filter_by(username=username).first() + +# files = request.files.items() + +# app.logger.info(f"user: {user} / project: {project}") + +# # for _, zip_file in files: +# # zip_filename = zip_file.filename +# # zip__ext = Path(zip_filename).suffix.lower() +# # if zip__ext[1:] in ALLOWED_COMPRESSED_EXTENSIONS: +# # app.logger.info( +# # f"File to be decompressed: {zip_filename}") +# # with ZipFile(zip_file, "r") as zip_obj: +# # for cfilename in zip_obj.namelist(): +# # cfile_extension = Path(cfilename).suffix.lower() +# # app.logger.info(f" compressed file: {cfilename}") +# # if cfile_extension[1:] in ALLOWED_EXTENSIONS: +# # zip_obj.extract( +# # cfilename, +# # app.config["UPLOAD_FOLDER"] +# # ) +# # filename = f"{str(uuid.uuid4().hex)}{cfile_extension}" + +# # shutil.move( +# # Path(app.config["UPLOAD_FOLDER"] +# # ).joinpath(cfilename), +# # Path(app.config["UPLOAD_FOLDER"] +# # ).joinpath(filename) +# # ) +# # data = Data( +# # project_id=project.id, +# # filename=filename, +# # original_filename=cfilename, +# # reference_transcription="", +# # is_marked_for_review=True, +# # assigned_user_id=user.id, +# # ) + +# # db.session.add(data) +# # db.session.flush() +# # # db.session.commit() + +# # elif cfile_extension[1:] in ALLOWED_ANNOTATION_EXTENSIONS: +# # temp_loc = Path(app.config["UPLOAD_FOLDER"] +# # ).joinpath(cfilename) +# # zip_obj.extract(cfilename, +# # app.config["UPLOAD_FOLDER"]) + +# # segmentations = json.load(open(temp_loc, "r")) + +# # for _segment in segmentations: +# # validated = validate_segmentation( +# # _segment, without_data=True) + +# # if not validated: +# # continue # raise error and skip transaction + +# # if validated: +# # try: +# # data = Data.query.filter_by( +# # project_id=pid, original_filename=_segment['filename']).first() + +# # if data.id: +# # new_segment = generate_segmentation( +# # data_id=data.id, +# # project_id=project.id, +# # end_time=_segment["end_time"], +# # start_time=_segment["start_time"], +# # transcription=_segment["transcription"], +# # annotations=_segment.get( +# # "annotations", {}) +# # ) +# # db.session.commit() +# # except Exception as e: +# # app.logger.info( +# # f"Error {e} for data: {data.id}") + +# # db.session.commit() + +# return jsonify( +# resp="HAHA" +# ) diff --git a/backend/routes/projects.py b/backend/routes/projects.py index f89332a..933f887 100644 --- a/backend/routes/projects.py +++ b/backend/routes/projects.py @@ -1,15 +1,26 @@ import sqlalchemy as sa import uuid +from pathlib import Path from flask import jsonify, flash, redirect, url_for, request from flask_jwt_extended import jwt_required, get_jwt_identity from werkzeug.urls import url_parse +from werkzeug.exceptions import BadRequest from backend import app, db from backend.models import Project, User, Label, Data, Segmentation, LabelValue from . import api -from .data import generate_segmentation + +from .data import ( + files_from_zip, + file_to_database, + ALLOWED_EXTENSIONS, + generate_segmentation, + annotation_to_database, + ALLOWED_ANNOTATION_EXTENSIONS, + ALLOWED_COMPRESSED_EXTENSIONS +) def generate_api_key(): @@ -708,3 +719,104 @@ def get_project_annotations(project_id): ), 200, ) + +@api.route("/projects//data", methods=["POST"]) +@jwt_required +def add_data_to_project(project_id): + """Upload data via zip files ro direct supported formats + + """ + identity = get_jwt_identity() + request_user = User.query.filter_by(username=identity["username"]).first() + app.logger.info(f"Current user is: {request_user}") + is_admin = True if request_user.role.role == "admin" else False + + if is_admin == False: + return jsonify(message="Unauthorized access!"), 401 + + project = Project.query.get(project_id) + files = request.files.items() + + for _, file in files: + filename = file.filename + file_ext = Path(filename).suffix.lower() + + if file_ext[1:] in ALLOWED_EXTENSIONS: + # if its a normal file, simple insertion + commit_flag = file_to_database( + db=db, + user=request_user, + project=project, + audio_file=file, + is_marked_for_review=True, + reference_transcription="False" + ) + + if not commit_flag: + return ( + jsonify( + message="Error during uploading of file", + type="UPLOAD_FAILED", + ), + 500 + ) + + + elif file_ext[1:] in ALLOWED_ANNOTATION_EXTENSIONS: + commit_flag = annotation_to_database( + project=project, + annotation_file=file + ) + + if not commit_flag: + return ( + jsonify( + message="Error during uploading of file", + type="UPLOAD_FAILED", + ), + 500 + ) + + elif file_ext[1:] in ALLOWED_COMPRESSED_EXTENSIONS: + cmprsd_files = files_from_zip( + zip_file=file + ) + for cfilename, filetype in cmprsd_files: + if filetype == "data": + commit_flag = file_to_database( + db=db, + user=request_user, + project=project, + audio_file=cfilename, + is_marked_for_review=True, + reference_transcription="False", + compressed_file=True + ) + else: + cfile = open(cfilename, "r") + commit_flag = annotation_to_database( + project=project, + annotation_file=cfile + ) + + app.logger.info(f"File rpocessed was: {cfilename} \n") + + if not commit_flag: + return ( + jsonify( + message="Error during uploading of file", + type="UPLOAD_FAILED", + ), + 500 + ) + + else: + raise BadRequest(description="File format is not supported") + + app.logger.info("Going to commit the session now") + try: + db.session.commit() + except Exception as e: + app.logger.error(e) + + return jsonify(something="Asd") diff --git a/frontend/src/containers/forms/uploadDataForm.js b/frontend/src/containers/forms/uploadDataForm.js index 8006267..096f78c 100644 --- a/frontend/src/containers/forms/uploadDataForm.js +++ b/frontend/src/containers/forms/uploadDataForm.js @@ -1,8 +1,8 @@ -import React from 'react'; +import React from "react"; import { withRouter } from "react-router"; -import Dropzone from 'react-dropzone-uploader'; +import Dropzone from "react-dropzone-uploader"; import { withStore } from "@spyna/react-store"; -import 'react-dropzone-uploader/dist/styles.css' +import "react-dropzone-uploader/dist/styles.css"; class UploadDataForm extends React.Component { constructor(props) { @@ -14,33 +14,41 @@ class UploadDataForm extends React.Component { this.initialState = { userName, projectId, - addDataUrl: `/api/datazip`, - }; - - this.state = Object.assign({}, this.initialState); - console.log(this.props); - } - - getUploadParams = ({ file, meta }) => { - const body = new FormData() - body.append('fileField', file) - body.append('username', this.state.userName) - body.append('projectId', this.state.projectId) - return { url: this.state.addDataUrl, body } - } - handleChangeStatus = ({ meta, file }, status) => { console.log(status, meta, file) } - - render() { - return ( - - ) - } - + // addDataUrl: `/api/datazip`, + addDataUrl: `/api/projects/${projectId}/data`, + }; + + this.state = Object.assign({}, this.initialState); + console.log(this.props); + } + + getUploadParams = ({ file, meta }) => { + const body = new FormData(); + body.append("fileField", file); + body.append("username", this.state.userName); + body.append("projectId", this.state.projectId); + return { + url: this.state.addDataUrl, + body, + headers: { + Authorization: localStorage.getItem("access_token"), + // Authorization: "Bearer " + localStorage.getItem("access_token") + }, + }; + }; + handleChangeStatus = ({ meta, file }, status) => { + console.log(status, meta, file); + }; + + render() { + return ( + + ); + } } -export default withStore(withRouter(UploadDataForm)); +export default withStore(withRouter(UploadDataForm)); \ No newline at end of file diff --git a/frontend/src/pages/admin.js b/frontend/src/pages/admin.js index 51a0bac..2765a81 100644 --- a/frontend/src/pages/admin.js +++ b/frontend/src/pages/admin.js @@ -168,9 +168,9 @@ class Admin extends React.Component { handleUploadData(e, userName, projectId) { this.setModalShow(true); - this.setState({ - formType: "NEW_DATA", - title: "Create New Project", + this.setState({ + formType: "NEW_DATA", + title: "Create New Project", userId: userName, // userName: userName does not work :( projectId: projectId }); } From 2cd49b13f45ae7d8d352191b08ea488d4ffa3be8 Mon Sep 17 00:00:00 2001 From: dumbmachine Date: Sun, 13 Sep 2020 04:08:35 +0530 Subject: [PATCH 7/7] upload refactor --- backend/routes/data.py | 120 +++--------------- backend/routes/projects.py | 22 ++-- .../src/containers/forms/uploadDataForm.js | 27 ++-- frontend/src/containers/modal.js | 2 +- frontend/src/pages/admin.js | 1 - 5 files changed, 37 insertions(+), 135 deletions(-) diff --git a/backend/routes/data.py b/backend/routes/data.py index de7cb73..8ec4e86 100644 --- a/backend/routes/data.py +++ b/backend/routes/data.py @@ -221,6 +221,7 @@ def files_from_zip(zip_file): ) yield Path(app.config["UPLOAD_FOLDER"]).joinpath(cfilename), "annotation" + def file_to_database( db, user, @@ -241,7 +242,8 @@ def file_to_database( original_filename = os.path.basename(audio_file) extension = Path(original_filename).suffix.lower() filename = f"{str(uuid.uuid4().hex)}{extension}" - from_path = Path(app.config["UPLOAD_FOLDER"]).joinpath(original_filename) + from_path = Path(app.config["UPLOAD_FOLDER"]).joinpath( + original_filename) to_path = Path(app.config["UPLOAD_FOLDER"]).joinpath(filename) shutil.move(from_path, to_path) else: @@ -267,17 +269,15 @@ def file_to_database( if compressed_file: shutil.rmtree(path=from_path, ignore_errors=True) shutil.rmtree(path=to_path, ignore_errors=True) + app.logger.error("Error in adding the data") app.logger.error(e) - app.logger.info(f"the error was: { e.with_traceback }") return False -def annotation_to_database( - project, - annotation_file -): +def annotation_to_database(project, annotation_file): """Add segmentation to database from a json """ + ret_flag = False try: segmentations = json.load(annotation_file) for _segment in segmentations: @@ -300,106 +300,16 @@ def annotation_to_database( annotations=_segment.get( "annotations", {}) ) + ret_flag = True - return True - return False + # delete the annotations file from the disk if exists + if hasattr(annotation_file, "name") and os.path.exists(annotation_file.name): + os.remove(annotation_file.name) + elif hasattr(annotation_file, "filename") and os.path.exists(annotation_file.filename): + os.remove(annotation_file.filename) + + return ret_flag except Exception as e: + app.logger.error("Error in adding the annotations") app.logger.error(e) return False - -# @api.route("/datazip", methods=["POST"]) -# @jwt_required -# def add_datazip(): -# identity = get_jwt_identity() -# app.logger.info(f"Current user is: {identity}") -# request_user = User.query.filter_by(username=identity["username"]).first() -# app.logger.info(f"Current user is: {request_user}") -# is_admin = True if request_user.role.role == "admin" else False - -# if is_admin == False: -# return jsonify(message="Unauthorized access!"), 401 - - -# project = Project.query.filter_by(id=pid).first() -# user = User.query.filter_by(username=username).first() - -# files = request.files.items() - -# app.logger.info(f"user: {user} / project: {project}") - -# # for _, zip_file in files: -# # zip_filename = zip_file.filename -# # zip__ext = Path(zip_filename).suffix.lower() -# # if zip__ext[1:] in ALLOWED_COMPRESSED_EXTENSIONS: -# # app.logger.info( -# # f"File to be decompressed: {zip_filename}") -# # with ZipFile(zip_file, "r") as zip_obj: -# # for cfilename in zip_obj.namelist(): -# # cfile_extension = Path(cfilename).suffix.lower() -# # app.logger.info(f" compressed file: {cfilename}") -# # if cfile_extension[1:] in ALLOWED_EXTENSIONS: -# # zip_obj.extract( -# # cfilename, -# # app.config["UPLOAD_FOLDER"] -# # ) -# # filename = f"{str(uuid.uuid4().hex)}{cfile_extension}" - -# # shutil.move( -# # Path(app.config["UPLOAD_FOLDER"] -# # ).joinpath(cfilename), -# # Path(app.config["UPLOAD_FOLDER"] -# # ).joinpath(filename) -# # ) -# # data = Data( -# # project_id=project.id, -# # filename=filename, -# # original_filename=cfilename, -# # reference_transcription="", -# # is_marked_for_review=True, -# # assigned_user_id=user.id, -# # ) - -# # db.session.add(data) -# # db.session.flush() -# # # db.session.commit() - -# # elif cfile_extension[1:] in ALLOWED_ANNOTATION_EXTENSIONS: -# # temp_loc = Path(app.config["UPLOAD_FOLDER"] -# # ).joinpath(cfilename) -# # zip_obj.extract(cfilename, -# # app.config["UPLOAD_FOLDER"]) - -# # segmentations = json.load(open(temp_loc, "r")) - -# # for _segment in segmentations: -# # validated = validate_segmentation( -# # _segment, without_data=True) - -# # if not validated: -# # continue # raise error and skip transaction - -# # if validated: -# # try: -# # data = Data.query.filter_by( -# # project_id=pid, original_filename=_segment['filename']).first() - -# # if data.id: -# # new_segment = generate_segmentation( -# # data_id=data.id, -# # project_id=project.id, -# # end_time=_segment["end_time"], -# # start_time=_segment["start_time"], -# # transcription=_segment["transcription"], -# # annotations=_segment.get( -# # "annotations", {}) -# # ) -# # db.session.commit() -# # except Exception as e: -# # app.logger.info( -# # f"Error {e} for data: {data.id}") - -# # db.session.commit() - -# return jsonify( -# resp="HAHA" -# ) diff --git a/backend/routes/projects.py b/backend/routes/projects.py index 933f887..5afef9b 100644 --- a/backend/routes/projects.py +++ b/backend/routes/projects.py @@ -720,7 +720,8 @@ def get_project_annotations(project_id): 200, ) -@api.route("/projects//data", methods=["POST"]) + +@api.route("/projects//upload", methods=["POST"]) @jwt_required def add_data_to_project(project_id): """Upload data via zip files ro direct supported formats @@ -742,7 +743,6 @@ def add_data_to_project(project_id): file_ext = Path(filename).suffix.lower() if file_ext[1:] in ALLOWED_EXTENSIONS: - # if its a normal file, simple insertion commit_flag = file_to_database( db=db, user=request_user, @@ -761,7 +761,6 @@ def add_data_to_project(project_id): 500 ) - elif file_ext[1:] in ALLOWED_ANNOTATION_EXTENSIONS: commit_flag = annotation_to_database( project=project, @@ -799,8 +798,6 @@ def add_data_to_project(project_id): annotation_file=cfile ) - app.logger.info(f"File rpocessed was: {cfilename} \n") - if not commit_flag: return ( jsonify( @@ -812,11 +809,12 @@ def add_data_to_project(project_id): else: raise BadRequest(description="File format is not supported") + db.session.commit() - app.logger.info("Going to commit the session now") - try: - db.session.commit() - except Exception as e: - app.logger.error(e) - - return jsonify(something="Asd") + return ( + jsonify( + message="Data uploaded succesfully", + type="UPLOAD_SUCCESS", + ), + 200 + ) diff --git a/frontend/src/containers/forms/uploadDataForm.js b/frontend/src/containers/forms/uploadDataForm.js index 096f78c..9c53bf3 100644 --- a/frontend/src/containers/forms/uploadDataForm.js +++ b/frontend/src/containers/forms/uploadDataForm.js @@ -5,17 +5,14 @@ import { withStore } from "@spyna/react-store"; import "react-dropzone-uploader/dist/styles.css"; class UploadDataForm extends React.Component { - constructor(props) { - super(props); + constructor(props) { + super(props); - const userName = this.props.userName; - const projectId = this.props.projectId; + const projectId = this.props.projectId; - this.initialState = { - userName, - projectId, - // addDataUrl: `/api/datazip`, - addDataUrl: `/api/projects/${projectId}/data`, + this.initialState = { + projectId, + addDataUrl: `/api/projects/${projectId}/upload`, }; this.state = Object.assign({}, this.initialState); @@ -25,30 +22,28 @@ class UploadDataForm extends React.Component { getUploadParams = ({ file, meta }) => { const body = new FormData(); body.append("fileField", file); - body.append("username", this.state.userName); - body.append("projectId", this.state.projectId); return { url: this.state.addDataUrl, body, headers: { Authorization: localStorage.getItem("access_token"), - // Authorization: "Bearer " + localStorage.getItem("access_token") }, }; }; - handleChangeStatus = ({ meta, file }, status) => { - console.log(status, meta, file); + + handleSubmit = (files, allFiles) => { + allFiles.forEach((f) => f.remove()); }; render() { return ( ); } } -export default withStore(withRouter(UploadDataForm)); \ No newline at end of file +export default withStore(withRouter(UploadDataForm)); diff --git a/frontend/src/containers/modal.js b/frontend/src/containers/modal.js index d76dd85..72f7ecd 100644 --- a/frontend/src/containers/modal.js +++ b/frontend/src/containers/modal.js @@ -30,7 +30,7 @@ const FormModal = (props) => { {props.formType === "NEW_USER" ? : null} {props.formType === "NEW_PROJECT" ? : null} {props.formType === "NEW_DATA" ? ( - + ) : null} {props.formType === "EDIT_USER" ? ( diff --git a/frontend/src/pages/admin.js b/frontend/src/pages/admin.js index 2765a81..ca7133c 100644 --- a/frontend/src/pages/admin.js +++ b/frontend/src/pages/admin.js @@ -171,7 +171,6 @@ class Admin extends React.Component { this.setState({ formType: "NEW_DATA", title: "Create New Project", - userId: userName, // userName: userName does not work :( projectId: projectId }); }