diff --git a/embedFigureBinary.js b/embedFigureBinary.js new file mode 100755 index 0000000000..0aebbc76f7 --- /dev/null +++ b/embedFigureBinary.js @@ -0,0 +1,56 @@ +#!/usr/bin/env node +/* +Parse Obojobo json files and embed images into them for uploading +EX: embedFigureBinary.js ../chen-obojobo-modules/*.json +You can pass it a single file, several files, or a wild card that your shell expands into multiple files +The script will look in the directory and any child directories that the .json file is in for images with the same filename +The script will write out the the same file name with .embedded.json added to the end + +*/ +const { execSync } = require('child_process') +const fs = require('fs') +const path = require('path') + +function traverse(o, func) { + for (let i in o) { + func.apply(this, [i, o[i], o]); + if (o[i] !== null && typeof (o[i]) === "object" ) { + //going one step down in the object tree!! + traverse(o[i], func); + } + } +} + +const embedBinaryDataIntoFigures = (draftJson, searchInDir, fileName) => { + traverse(draftJson, (key, val, obj) => { + if(key === 'type' && val ==='ObojoboDraft.Chunks.Figure'){ + if(obj.content.filename){ + const output = execSync(`find . -name "${obj.content.filename}"`, {cwd: searchInDir}) + const filePath = output.toString().split(/\r?\n/)[0] + if(!filePath){ + console.error(`Error: unable to find file '${obj.content.filename}' used in chunk: '${obj.id}' in ${path.basename(fileName)}`) + if(obj?.content?.url?.startsWith('http')) console.error(` ^ though this figure uses a url ${obj.content.url}`) + return + } + const combinedPath = searchInDir + filePath.substring(1) + const imageBinary = fs.readFileSync(combinedPath, {encoding: 'base64'}); + obj.content.imageBinary = imageBinary + } else if (obj?.content?.url?.startsWith('http') ){ + // @TODO: maybe we should download the file and embed it? + } + } + }) + return draftJson +} + + +const args = process.argv.slice(2); + +args.forEach((filePath) => { + const rawJSON = fs.readFileSync(filePath) + const data = JSON.parse(rawJSON) + const searchInDir = path.dirname(filePath); + const updatedJSON = embedBinaryDataIntoFigures(data, searchInDir, filePath) + fs.writeFileSync(filePath+'.embedded.json', JSON.stringify(updatedJSON, null, 4)) +}) + diff --git a/packages/app/obojobo-express/package.json b/packages/app/obojobo-express/package.json index 0eb7945a81..41bf13be0e 100644 --- a/packages/app/obojobo-express/package.json +++ b/packages/app/obojobo-express/package.json @@ -58,6 +58,7 @@ "glob": "^7.1.6", "is-svg": "^4.3.1", "json-inflector": "^1.1.0", + "mime": "^3.0.0", "moment": "^2.29.1", "morgan": "~1.10.0", "multer": "^1.4.2", diff --git a/packages/app/obojobo-express/server/models/media.js b/packages/app/obojobo-express/server/models/media.js index 2ec7bf642e..47a70607e0 100644 --- a/packages/app/obojobo-express/server/models/media.js +++ b/packages/app/obojobo-express/server/models/media.js @@ -256,7 +256,7 @@ class Media { let mediaBinaryId = null if (!binary || !size || !mimetype || !dimensions || !mode) { - throw new Error('One or more required arguments not provided.') + throw new Error('Inserting an image, but one or more required arguments not provided.') } if (mode === MODE_INSERT_ORIGINAL_IMAGE && !userId) { @@ -376,7 +376,13 @@ class Media { let file try { - file = await fs.readFile(fileInfo.path) + if (fileInfo.buffer) { + // allow fileInfo to already contain a loaded buffer + file = fileInfo.buffer + } else { + // load the file from disk + file = await fs.readFile(fileInfo.path) + } } catch (error) { // calling methods expect a thenable object to be returned logger.logError('Error Reading media file', error) @@ -387,7 +393,7 @@ class Media { if (!isValid) { // Delete the temporary media stored by Multer - await fs.unlink(fileInfo.path) + if (fileInfo.path) await fs.unlink(fileInfo.path) throw new Error( `File upload only supports the following filetypes: ${mediaConfig.allowedMimeTypesRegex .split('|') @@ -408,7 +414,7 @@ class Media { }) // Delete the temporary media stored by Multer - await fs.unlink(fileInfo.path) + if (fileInfo.path) await fs.unlink(fileInfo.path) oboEvents.emit(Media.EVENT_IMAGE_CREATED, { userId, diff --git a/packages/app/obojobo-express/server/routes/api/drafts.js b/packages/app/obojobo-express/server/routes/api/drafts.js index 0e99a142be..188bb22faf 100644 --- a/packages/app/obojobo-express/server/routes/api/drafts.js +++ b/packages/app/obojobo-express/server/routes/api/drafts.js @@ -18,6 +18,8 @@ const { requireCanDeleteDrafts, checkContentId } = oboRequire('server/express_validators') +const Media = oboRequire('server/models/media') +const mime = require('mime') const isNoDataFromQueryError = e => { return ( @@ -102,12 +104,53 @@ router } }) +async function traverse(o, func) { + for (let i in o) { + await func.apply(this, [i, o[i], o]) + if (o[i] !== null && typeof o[i] === 'object') { + //going one step down in the object tree!! + await traverse(o[i], func) + } + } +} + +const extractAndUploadEmbeddedImages = async (draftJson, userId) => { + await traverse(draftJson, async (key, val, obj) => { + if ( + key === 'type' && + val === 'ObojoboDraft.Chunks.Figure' && + typeof obj?.content?.imageBinary === 'string' + ) { + // use the filename to determine the mimetype + const mimetype = mime.getType(obj.content.filename) + // load the base64 data into a buffer + const buf = Buffer.from(obj.content.imageBinary, 'base64') + + // mock the fileInfo needed by Media + const mockFileInfo = { + originalname: obj.content.filename, + mimetype, + size: buf.length, + buffer: buf + } + + // save the media asset to the db + const mediaRecord = await Media.createAndSave(userId, mockFileInfo) + + // update the json + obj.content.url = mediaRecord.media_id + delete obj.content.imageBinary + } + }) + return draftJson +} + // Create a Draft // mounted as /api/drafts/new router .route('/new') .post(requireCanCreateDrafts) - .post((req, res, next) => { + .post(async (req, res, next) => { const content = req.body.content const format = req.body.format @@ -115,7 +158,13 @@ router let draftXml = !format ? draftTemplateXML : null if (format === 'application/json') { - draftJson = content + try { + const jsonContent = typeof content === 'string' ? JSON.parse(content) : content + draftJson = await extractAndUploadEmbeddedImages(jsonContent, req.currentUser.id) + } catch (e){ + logger.error('Parse JSON Failed:', e, content) + return res.unexpected(e) + } } else if (format === 'application/xml') { draftXml = content try { @@ -132,12 +181,13 @@ router } } - return DraftModel.createWithContent(req.currentUser.id, draftJson, draftXml) - .then(draft => { - res.set('Obo-DraftContentId', draft.content.id) - res.success({ id: draft.id, contentId: draft.content.id }) - }) - .catch(res.unexpected) + try { + const draft = await DraftModel.createWithContent(req.currentUser.id, draftJson, draftXml) + res.set('Obo-DraftContentId', draft.content.id) + res.success({ id: draft.id, contentId: draft.content.id }) + } catch (error) { + res.unexpected(error) + } }) // Create an editable tutorial document // mounted as /api/drafts/tutorial diff --git a/yarn.lock b/yarn.lock index 81f20c8fb4..c237ab873b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9550,6 +9550,11 @@ mime@^2.4.4, mime@^2.4.6: resolved "https://registry.yarnpkg.com/mime/-/mime-2.5.2.tgz#6e3dc6cc2b9510643830e5f19d5cb753da5eeabe" integrity sha512-tqkh47FzKeCPD2PUiPB6pkbMzsCasjxAfC62/Wap5qrUWcb+sFasXUC5I3gYM5iBM8v/Qpn4UK0x+j0iHyFPDg== +mime@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/mime/-/mime-3.0.0.tgz#b374550dca3a0c18443b0c950a6a58f1931cf7a7" + integrity sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A== + mimic-fn@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" @@ -13559,18 +13564,6 @@ tar@^6.0.2, tar@^6.1.0: mkdirp "^1.0.3" yallist "^4.0.0" -tar@^6.1.2: - version "6.1.11" - resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.11.tgz#6760a38f003afa1b2ffd0ffe9e9abbd0eab3d621" - integrity sha512-an/KZQzQUkZCkuoAA64hM92X0Urb6VpRhAFllDzz44U2mcD5scmT3zBc4VgVpkugF580+DQn8eAFSyoQt0tznA== - dependencies: - chownr "^2.0.0" - fs-minipass "^2.0.0" - minipass "^3.0.0" - minizlib "^2.1.1" - mkdirp "^1.0.3" - yallist "^4.0.0" - temp-dir@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/temp-dir/-/temp-dir-1.0.0.tgz#0a7c0ea26d3a39afa7e0ebea9c1fc0bc4daa011d"