diff --git a/backend/routes/data.py b/backend/routes/data.py index ba0c5ec..8ec4e86 100644 --- a/backend/routes/data.py +++ b/backend/routes/data.py @@ -1,8 +1,11 @@ +import os +import shutil import json import sqlalchemy as sa import uuid from pathlib import Path +from zipfile import ZipFile from flask import jsonify, flash, redirect, url_for, request, send_from_directory from flask_jwt_extended import jwt_required, get_jwt_identity @@ -15,19 +18,22 @@ from . import api +ALLOWED_COMPRESSED_EXTENSIONS = ["zip"] +ALLOWED_ANNOTATION_EXTENSIONS = ["json"] ALLOWED_EXTENSIONS = ["wav", "mp3", "ogg"] - @api.route("/audio/", methods=["GET"]) @jwt_required def send_audio_file(file_name): return send_from_directory(app.config["UPLOAD_FOLDER"], file_name) -def validate_segmentation(segment): +def validate_segmentation(segment, without_data=False): """Validate the segmentation before accepting the annotation's upload from users """ required_key = {"start_time", "end_time", "transcription"} + if without_data: + required_key.add("filename") # requried to search the datapoint if set(required_key).issubset(segment.keys()): return True @@ -190,3 +196,120 @@ def add_data(): ), 201, ) + + +def files_from_zip(zip_file): + """Generator for getting files from the a zip file + + Returns: + Generator: if valid, generator of files in the zip + False: if the file is invalid + """ + with ZipFile(zip_file, "r") as zip_obj: + for cfilename in zip_obj.namelist(): + cfile_extension = Path(cfilename).suffix.lower() + if cfile_extension[1:] in ALLOWED_EXTENSIONS: + zip_obj.extract( + cfilename, + Path(app.config["UPLOAD_FOLDER"]) + ) + yield Path(app.config["UPLOAD_FOLDER"]).joinpath(cfilename), "data" + elif cfile_extension[1:] in ALLOWED_ANNOTATION_EXTENSIONS: + zip_obj.extract( + cfilename, + Path(app.config["UPLOAD_FOLDER"]) + ) + yield Path(app.config["UPLOAD_FOLDER"]).joinpath(cfilename), "annotation" + + +def file_to_database( + db, + user, + project, + audio_file, + is_marked_for_review, + reference_transcription, + compressed_file=False +): + """Add data to database and save a copy in the /uploads folder + + TODO: + - delete compressed file is there was some error + """ + try: + + if compressed_file: + original_filename = os.path.basename(audio_file) + extension = Path(original_filename).suffix.lower() + filename = f"{str(uuid.uuid4().hex)}{extension}" + from_path = Path(app.config["UPLOAD_FOLDER"]).joinpath( + original_filename) + to_path = Path(app.config["UPLOAD_FOLDER"]).joinpath(filename) + shutil.move(from_path, to_path) + else: + original_filename = secure_filename(audio_file.filename) + extension = Path(original_filename).suffix.lower() + filename = f"{str(uuid.uuid4().hex)}{extension}" + file_path = Path(app.config["UPLOAD_FOLDER"]).joinpath(filename) + audio_file.save(file_path.as_posix()) + + data = Data( + project_id=project.id, + filename=filename, + original_filename=original_filename, + reference_transcription=reference_transcription, + is_marked_for_review=is_marked_for_review, + assigned_user_id=user.id, + ) + db.session.add(data) + db.session.flush() + + return True + except Exception as e: + if compressed_file: + shutil.rmtree(path=from_path, ignore_errors=True) + shutil.rmtree(path=to_path, ignore_errors=True) + app.logger.error("Error in adding the data") + app.logger.error(e) + return False + + +def annotation_to_database(project, annotation_file): + """Add segmentation to database from a json + """ + ret_flag = False + try: + segmentations = json.load(annotation_file) + for _segment in segmentations: + validated = validate_segmentation( + _segment, without_data=True + ) + if validated: + data = Data.query.filter_by( + project_id=project.id, + original_filename=_segment['filename'] + ).first() + + if data: + new_segment = generate_segmentation( + data_id=data.id, + project_id=project.id, + end_time=_segment["end_time"], + start_time=_segment["start_time"], + transcription=_segment["transcription"], + annotations=_segment.get( + "annotations", {}) + ) + ret_flag = True + + # delete the annotations file from the disk if exists + if hasattr(annotation_file, "name") and os.path.exists(annotation_file.name): + os.remove(annotation_file.name) + elif hasattr(annotation_file, "filename") and os.path.exists(annotation_file.filename): + os.remove(annotation_file.filename) + + return ret_flag + except Exception as e: + app.logger.error("Error in adding the annotations") + app.logger.error(e) + return False diff --git a/backend/routes/projects.py b/backend/routes/projects.py index f89332a..5afef9b 100644 --- a/backend/routes/projects.py +++ b/backend/routes/projects.py @@ -1,15 +1,26 @@ import sqlalchemy as sa import uuid +from pathlib import Path from flask import jsonify, flash, redirect, url_for, request from flask_jwt_extended import jwt_required, get_jwt_identity from werkzeug.urls import url_parse +from werkzeug.exceptions import BadRequest from backend import app, db from backend.models import Project, User, Label, Data, Segmentation, LabelValue from . import api -from .data import generate_segmentation + +from .data import ( + files_from_zip, + file_to_database, + ALLOWED_EXTENSIONS, + generate_segmentation, + annotation_to_database, + ALLOWED_ANNOTATION_EXTENSIONS, + ALLOWED_COMPRESSED_EXTENSIONS +) def generate_api_key(): @@ -708,3 +719,102 @@ def get_project_annotations(project_id): ), 200, ) + + +@api.route("/projects//upload", methods=["POST"]) +@jwt_required +def add_data_to_project(project_id): + """Upload data via zip files ro direct supported formats + + """ + identity = get_jwt_identity() + request_user = User.query.filter_by(username=identity["username"]).first() + app.logger.info(f"Current user is: {request_user}") + is_admin = True if request_user.role.role == "admin" else False + + if is_admin == False: + return jsonify(message="Unauthorized access!"), 401 + + project = Project.query.get(project_id) + files = request.files.items() + + for _, file in files: + filename = file.filename + file_ext = Path(filename).suffix.lower() + + if file_ext[1:] in ALLOWED_EXTENSIONS: + commit_flag = file_to_database( + db=db, + user=request_user, + project=project, + audio_file=file, + is_marked_for_review=True, + reference_transcription="False" + ) + + if not commit_flag: + return ( + jsonify( + message="Error during uploading of file", + type="UPLOAD_FAILED", + ), + 500 + ) + + elif file_ext[1:] in ALLOWED_ANNOTATION_EXTENSIONS: + commit_flag = annotation_to_database( + project=project, + annotation_file=file + ) + + if not commit_flag: + return ( + jsonify( + message="Error during uploading of file", + type="UPLOAD_FAILED", + ), + 500 + ) + + elif file_ext[1:] in ALLOWED_COMPRESSED_EXTENSIONS: + cmprsd_files = files_from_zip( + zip_file=file + ) + for cfilename, filetype in cmprsd_files: + if filetype == "data": + commit_flag = file_to_database( + db=db, + user=request_user, + project=project, + audio_file=cfilename, + is_marked_for_review=True, + reference_transcription="False", + compressed_file=True + ) + else: + cfile = open(cfilename, "r") + commit_flag = annotation_to_database( + project=project, + annotation_file=cfile + ) + + if not commit_flag: + return ( + jsonify( + message="Error during uploading of file", + type="UPLOAD_FAILED", + ), + 500 + ) + + else: + raise BadRequest(description="File format is not supported") + db.session.commit() + + return ( + jsonify( + message="Data uploaded succesfully", + type="UPLOAD_SUCCESS", + ), + 200 + ) diff --git a/frontend/package.json b/frontend/package.json index b5ae35c..ea3b912 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -12,6 +12,7 @@ "@testing-library/react": "^9.3.2", "@testing-library/user-event": "^7.1.2", "axios": "^0.19.2", + "react-dropzone-uploader": "^2.11.0", "bootstrap": "^4.4.1", "jquery": "^3.5.0", "popper.js": "^1.16.1", diff --git a/frontend/src/containers/forms/uploadDataForm.js b/frontend/src/containers/forms/uploadDataForm.js new file mode 100644 index 0000000..9c53bf3 --- /dev/null +++ b/frontend/src/containers/forms/uploadDataForm.js @@ -0,0 +1,49 @@ +import React from "react"; +import { withRouter } from "react-router"; +import Dropzone from "react-dropzone-uploader"; +import { withStore } from "@spyna/react-store"; +import "react-dropzone-uploader/dist/styles.css"; + +class UploadDataForm extends React.Component { + constructor(props) { + super(props); + + const projectId = this.props.projectId; + + this.initialState = { + projectId, + addDataUrl: `/api/projects/${projectId}/upload`, + }; + + this.state = Object.assign({}, this.initialState); + console.log(this.props); + } + + getUploadParams = ({ file, meta }) => { + const body = new FormData(); + body.append("fileField", file); + return { + url: this.state.addDataUrl, + body, + headers: { + Authorization: localStorage.getItem("access_token"), + }, + }; + }; + + handleSubmit = (files, allFiles) => { + allFiles.forEach((f) => f.remove()); + }; + + render() { + return ( + + ); + } +} + +export default withStore(withRouter(UploadDataForm)); diff --git a/frontend/src/containers/modal.js b/frontend/src/containers/modal.js index bab258f..72f7ecd 100644 --- a/frontend/src/containers/modal.js +++ b/frontend/src/containers/modal.js @@ -2,6 +2,7 @@ import React from "react"; import Modal from "react-bootstrap/Modal"; import CreateUserForm from "./forms/createUserForm"; +import UploadDataForm from "./forms/uploadDataForm"; import EditUserForm from "./forms/editUserForm"; import CreateProjectForm from "./forms/createProjectForm"; import CreateLabelForm from "./forms/createLabelForm"; @@ -28,6 +29,9 @@ const FormModal = (props) => { {props.formType === "NEW_USER" ? : null} {props.formType === "NEW_PROJECT" ? : null} + {props.formType === "NEW_DATA" ? ( + + ) : null} {props.formType === "EDIT_USER" ? ( ) : null} diff --git a/frontend/src/pages/admin.js b/frontend/src/pages/admin.js index 6d65c13..ca7133c 100644 --- a/frontend/src/pages/admin.js +++ b/frontend/src/pages/admin.js @@ -9,6 +9,7 @@ import { faUserPlus, faTags, faDownload, + faUpload, } from "@fortawesome/free-solid-svg-icons"; import { IconButton } from "../components/button"; import Loader from "../components/loader"; @@ -165,6 +166,14 @@ class Admin extends React.Component { }); } + handleUploadData(e, userName, projectId) { + this.setModalShow(true); + this.setState({ + formType: "NEW_DATA", + title: "Create New Project", + projectId: projectId }); + } + setModalShow(modalShow) { this.setState({ modalShow }); } @@ -260,6 +269,18 @@ class Admin extends React.Component { ) } /> + + this.handleUploadData( + e, + "admin", + project["project_id"] + ) + } + />