From a963a0757387b12584d71cc04ce0b792630de84d Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Thu, 31 Oct 2024 15:29:58 +0200 Subject: [PATCH 1/5] add support for encrypted mailboxes --- lib/api/mailboxes.js | 10 ++++++++-- lib/api/messages.js | 3 ++- lib/schemas/response/mailboxes-schemas.js | 1 + 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/api/mailboxes.js b/lib/api/mailboxes.js index 327c37c5..767f8e66 100644 --- a/lib/api/mailboxes.js +++ b/lib/api/mailboxes.js @@ -214,7 +214,8 @@ module.exports = (db, server, mailboxHandler) => { specialUse: mailboxData.specialUse, modifyIndex: mailboxData.modifyIndex, subscribed: mailboxData.subscribed, - hidden: !!mailboxData.hidden + hidden: !!mailboxData.hidden, + encryptMessages: !!mailboxData.encryptMessages }; if (mailboxData.retention) { @@ -291,6 +292,7 @@ module.exports = (db, server, mailboxHandler) => { .min(0) .description('Retention policy for the created Mailbox. Milliseconds after a message added to mailbox expires. Set to 0 to disable.'), sess: sessSchema, + encryptMessages: booleanSchema.default(false).description('If true then messages in this mailbox are encrypted'), ip: sessIPSchema }, queryParams: {}, @@ -344,7 +346,8 @@ module.exports = (db, server, mailboxHandler) => { let opts = { subscribed: true, - hidden: !!result.value.hidden + hidden: !!result.value.hidden, + encryptMessages: !!result.value.encryptMessages }; if (retention) { @@ -399,6 +402,7 @@ module.exports = (db, server, mailboxHandler) => { modifyIndex: Joi.number().required().description('Modification sequence number. Incremented on every change in the mailbox.'), subscribed: booleanSchema.required().description('Mailbox subscription status. IMAP clients may unsubscribe from a folder.'), hidden: booleanSchema.required().description('Is the folder hidden or not'), + encryptMessages: booleanSchema.default(false).required().description('If true then messages in this mailbox are encrypted'), total: Joi.number().required().description('How many messages are stored in this mailbox'), unseen: Joi.number().required().description('How many unseen messages are stored in this mailbox') }) @@ -527,6 +531,7 @@ module.exports = (db, server, mailboxHandler) => { modifyIndex: mailboxData.modifyIndex, subscribed: mailboxData.subscribed, hidden: !!mailboxData.hidden, + encryptMessages: !!mailboxData.encryptMessages, total, unseen }); @@ -552,6 +557,7 @@ module.exports = (db, server, mailboxHandler) => { 'Retention policy for the Mailbox (in ms). Changing retention value only affects messages added to this folder after the change' ), subscribed: booleanSchema.description('Change Mailbox subscription state'), + encryptMessages: booleanSchema.default(false).description('If true then messages in this mailbox are encrypted'), hidden: booleanSchema.description('Is the folder hidden or not. Hidden folders can not be opened in IMAP.'), sess: sessSchema, ip: sessIPSchema diff --git a/lib/api/messages.js b/lib/api/messages.js index 41fdddd5..8b5057e2 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -2402,7 +2402,8 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti }); } - if (userData.encryptMessages && !result.value.draft) { + if ((userData.encryptMessages || mailboxData.encryptMessages) && !result.value.draft) { + // encrypt message if global encryption ON or encrypted target mailbox try { let encrypted = await encryptMessage(userData.pubKey, raw); if (encrypted) { diff --git a/lib/schemas/response/mailboxes-schemas.js b/lib/schemas/response/mailboxes-schemas.js index 3b3eac9a..bc0cf7a9 100644 --- a/lib/schemas/response/mailboxes-schemas.js +++ b/lib/schemas/response/mailboxes-schemas.js @@ -15,6 +15,7 @@ const GetMailboxesResult = Joi.object({ 'Default retention policy for this mailbox (in ms). If set then messages added to this mailbox will be automatically deleted after retention time.' ), hidden: booleanSchema.required().description('Is the folder hidden or not'), + encryptMessages: booleanSchema.default(false).required().description('If true then messages in this mailbox are encrypted'), total: Joi.number().required().description('How many messages are stored in this mailbox'), unseen: Joi.number().required().description('How many unseen messages are stored in this mailbox'), size: Joi.number().description('Total size of mailbox in bytes.') From e1fd1e40b39149ad596e1848e6829b549d5dc2c1 Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Mon, 4 Nov 2024 12:45:31 +0200 Subject: [PATCH 2/5] mailboxes.js remove unnecessary default value --- lib/api/mailboxes.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/api/mailboxes.js b/lib/api/mailboxes.js index 767f8e66..179684c9 100644 --- a/lib/api/mailboxes.js +++ b/lib/api/mailboxes.js @@ -402,7 +402,7 @@ module.exports = (db, server, mailboxHandler) => { modifyIndex: Joi.number().required().description('Modification sequence number. Incremented on every change in the mailbox.'), subscribed: booleanSchema.required().description('Mailbox subscription status. IMAP clients may unsubscribe from a folder.'), hidden: booleanSchema.required().description('Is the folder hidden or not'), - encryptMessages: booleanSchema.default(false).required().description('If true then messages in this mailbox are encrypted'), + encryptMessages: booleanSchema.required().description('If true then messages in this mailbox are encrypted'), total: Joi.number().required().description('How many messages are stored in this mailbox'), unseen: Joi.number().required().description('How many unseen messages are stored in this mailbox') }) @@ -557,7 +557,7 @@ module.exports = (db, server, mailboxHandler) => { 'Retention policy for the Mailbox (in ms). Changing retention value only affects messages added to this folder after the change' ), subscribed: booleanSchema.description('Change Mailbox subscription state'), - encryptMessages: booleanSchema.default(false).description('If true then messages in this mailbox are encrypted'), + encryptMessages: booleanSchema.description('If true then messages in this mailbox are encrypted'), hidden: booleanSchema.description('Is the folder hidden or not. Hidden folders can not be opened in IMAP.'), sess: sessSchema, ip: sessIPSchema From 076f2c6996ab1ab861dcd03d249dc3e7e7eca4e4 Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Mon, 11 Nov 2024 09:26:30 +0200 Subject: [PATCH 3/5] filtering-handler, add raw to call to addmessage. Feature: Encrypted mailboxes added --- lib/filter-handler.js | 4 +- lib/message-handler.js | 948 ++++++++++++++++++++++++----------------- 2 files changed, 562 insertions(+), 390 deletions(-) diff --git a/lib/filter-handler.js b/lib/filter-handler.js index ad9621c8..3bda7899 100644 --- a/lib/filter-handler.js +++ b/lib/filter-handler.js @@ -143,6 +143,8 @@ class FilterHandler { let prepared; + let raw = Buffer.concat(chunks, chunklen); + if (options.mimeTree) { if (options.mimeTree && options.mimeTree.header) { // remove old headers @@ -157,7 +159,6 @@ class FilterHandler { mimeTree: options.mimeTree }); } else { - let raw = Buffer.concat(chunks, chunklen); prepared = await this.prepareMessage({ raw }); @@ -661,6 +662,7 @@ class FilterHandler { date: false, flags, + raw, rawchunks }; diff --git a/lib/message-handler.js b/lib/message-handler.js index 887ab62a..a9d97efb 100644 --- a/lib/message-handler.js +++ b/lib/message-handler.js @@ -189,421 +189,488 @@ class MessageHandler { return setImmediate(() => callback(new Error('Message size ' + options.raw.length + ' bytes is too large'))); } - this.prepareMessage(options, (err, prepared) => { + this.getMailbox(options, (err, mailboxData) => { if (err) { return callback(err); } - let id = prepared.id; - let mimeTree = prepared.mimeTree; - let size = prepared.size; - let bodystructure = prepared.bodystructure; - let envelope = prepared.envelope; - let idate = prepared.idate; - let hdate = prepared.hdate; - let msgid = prepared.msgid; - let subject = prepared.subject; - let headers = prepared.headers; - - let flags = Array.isArray(options.flags) ? options.flags : [].concat(options.flags || []); - let maildata = options.maildata || this.indexer.getMaildata(mimeTree); - - this.getMailbox(options, (err, mailboxData) => { + // get target mailbox data + + options.targetMailboxEncrypted = !!mailboxData.encryptMessages; + + this.users.collection('users').findOne({ _id: options.user }, (err, userData) => { if (err) { return callback(err); } - let cleanup = (...args) => { - if (!args[0]) { - return callback(...args); - } - - let attachmentIds = Object.keys(mimeTree.attachmentMap || {}).map(key => mimeTree.attachmentMap[key]); - if (!attachmentIds.length) { - return callback(...args); - } + // get target user data - this.attachmentStorage.deleteMany(attachmentIds, maildata.magic, () => callback(...args)); - }; - - this.indexer.storeNodeBodies(maildata, mimeTree, err => { + this.prepareMessage(options, (err, prepared) => { if (err) { - return cleanup(err); + return callback(err); } - // prepare message object - let messageData = { - _id: id, + // check if already encrypted + let alreadyEncrypted = false; - // should be kept when COPY'ing or MOVE'ing - root: id, + // message already prepared, check if encrypted + const parsedHeader = (prepared.mimeTree && prepared.mimeTree?.parsedHeader) || {}; + const parsedContentType = parsedHeader['content-type']; - v: consts.SCHEMA_VERSION, + if (parsedContentType && parsedContentType.subtype === 'encrypted') { + alreadyEncrypted = true; + } - // if true then expires after rdate + retention - exp: !!mailboxData.retention, - rdate: Date.now() + (mailboxData.retention || 0), + let flags = Array.isArray(options.flags) ? options.flags : [].concat(options.flags || []); + + let addMessage = () => { + let id = prepared.id; + let mimeTree = prepared.mimeTree; + let size = prepared.size; + let bodystructure = prepared.bodystructure; + let envelope = prepared.envelope; + let idate = prepared.idate; + let hdate = prepared.hdate; + let msgid = prepared.msgid; + let subject = prepared.subject; + let headers = prepared.headers; + + let maildata = options.maildata || this.indexer.getMaildata(mimeTree); + + let cleanup = (...args) => { + if (!args[0]) { + return callback(...args); + } - // make sure the field exists. it is set to true when user is deleted - userDeleted: false, + let attachmentIds = Object.keys(mimeTree.attachmentMap || {}).map(key => mimeTree.attachmentMap[key]); + if (!attachmentIds.length) { + return callback(...args); + } - idate, - hdate, - flags, - size, + this.attachmentStorage.deleteMany(attachmentIds, maildata.magic, () => callback(...args)); + }; - // some custom metadata about the delivery - meta: options.meta || {}, + this.indexer.storeNodeBodies(maildata, mimeTree, err => { + if (err) { + return cleanup(err); + } - // list filter IDs that matched this message - filters: Array.isArray(options.filters) ? options.filters : [].concat(options.filters || []), + // prepare message object + let messageData = { + _id: id, - headers, - mimeTree, - envelope, - bodystructure, - msgid, + // should be kept when COPY'ing or MOVE'ing + root: id, - // use boolean for more commonly used (and searched for) flags - unseen: !flags.includes('\\Seen'), - flagged: flags.includes('\\Flagged'), - undeleted: !flags.includes('\\Deleted'), - draft: flags.includes('\\Draft'), + v: consts.SCHEMA_VERSION, - magic: maildata.magic, + // if true then expires after rdate + retention + exp: !!mailboxData.retention, + rdate: Date.now() + (mailboxData.retention || 0), - subject, + // make sure the field exists. it is set to true when user is deleted + userDeleted: false, - // do not archive deleted messages that have been copied - copied: false - }; + idate, + hdate, + flags, + size, - if (options.verificationResults) { - messageData.verificationResults = options.verificationResults; - } + // some custom metadata about the delivery + meta: options.meta || {}, - if (options.outbound) { - messageData.outbound = [].concat(options.outbound || []); - } + // list filter IDs that matched this message + filters: Array.isArray(options.filters) ? options.filters : [].concat(options.filters || []), - if (options.forwardTargets) { - messageData.forwardTargets = [].concat(options.forwardTargets || []); - } + headers, + mimeTree, + envelope, + bodystructure, + msgid, - if (maildata.attachments && maildata.attachments.length) { - messageData.attachments = maildata.attachments; - messageData.ha = maildata.attachments.some(a => !a.related); - } else { - messageData.ha = false; - } + // use boolean for more commonly used (and searched for) flags + unseen: !flags.includes('\\Seen'), + flagged: flags.includes('\\Flagged'), + undeleted: !flags.includes('\\Deleted'), + draft: flags.includes('\\Draft'), - if (maildata.text) { - messageData.text = maildata.text.replace(/\r\n/g, '\n').trim(); + magic: maildata.magic, - // text is indexed with a fulltext index, so only store the beginning of it - if (messageData.text.length > consts.MAX_PLAINTEXT_INDEXED) { - messageData.textFooter = messageData.text.substr(consts.MAX_PLAINTEXT_INDEXED); - messageData.text = messageData.text.substr(0, consts.MAX_PLAINTEXT_INDEXED); + subject, - // truncate remaining text if total length exceeds maximum allowed - if ( - consts.MAX_PLAINTEXT_CONTENT > consts.MAX_PLAINTEXT_INDEXED && - messageData.textFooter.length > consts.MAX_PLAINTEXT_CONTENT - consts.MAX_PLAINTEXT_INDEXED - ) { - messageData.textFooter = messageData.textFooter.substr(0, consts.MAX_PLAINTEXT_CONTENT - consts.MAX_PLAINTEXT_INDEXED); + // do not archive deleted messages that have been copied + copied: false + }; + + if (options.verificationResults) { + messageData.verificationResults = options.verificationResults; } - } - messageData.text = - messageData.text.length <= consts.MAX_PLAINTEXT_CONTENT - ? messageData.text - : messageData.text.substr(0, consts.MAX_PLAINTEXT_CONTENT); - messageData.intro = this.createIntro(messageData.text); - } + if (options.outbound) { + messageData.outbound = [].concat(options.outbound || []); + } - if (maildata.html && maildata.html.length) { - let htmlSize = 0; - messageData.html = maildata.html - .map(html => { - if (htmlSize >= consts.MAX_HTML_CONTENT || !html) { - return ''; - } + if (options.forwardTargets) { + messageData.forwardTargets = [].concat(options.forwardTargets || []); + } - if (htmlSize + Buffer.byteLength(html) <= consts.MAX_HTML_CONTENT) { - htmlSize += Buffer.byteLength(html); - return html; - } + if (maildata.attachments && maildata.attachments.length) { + messageData.attachments = maildata.attachments; + messageData.ha = maildata.attachments.some(a => !a.related); + } else { + messageData.ha = false; + } - html = html.substr(0, consts.MAX_HTML_CONTENT); - htmlSize += Buffer.byteLength(html); - return html; - }) - .filter(html => html); + if (maildata.text) { + messageData.text = maildata.text.replace(/\r\n/g, '\n').trim(); - // if message has HTML content use it instead of text/plain content for intro - messageData.intro = this.createIntro(htmlToText(messageData.html.join(''))); - } + // text is indexed with a fulltext index, so only store the beginning of it + if (messageData.text.length > consts.MAX_PLAINTEXT_INDEXED) { + messageData.textFooter = messageData.text.substr(consts.MAX_PLAINTEXT_INDEXED); + messageData.text = messageData.text.substr(0, consts.MAX_PLAINTEXT_INDEXED); - this.users.collection('users').findOneAndUpdate( - { - _id: mailboxData.user - }, - { - $inc: { - storageUsed: size - } - }, - { - returnDocument: 'after', - projection: { - storageUsed: true - } - }, - (err, r) => { - if (err) { - return cleanup(err); - } + // truncate remaining text if total length exceeds maximum allowed + if ( + consts.MAX_PLAINTEXT_CONTENT > consts.MAX_PLAINTEXT_INDEXED && + messageData.textFooter.length > consts.MAX_PLAINTEXT_CONTENT - consts.MAX_PLAINTEXT_INDEXED + ) { + messageData.textFooter = messageData.textFooter.substr(0, consts.MAX_PLAINTEXT_CONTENT - consts.MAX_PLAINTEXT_INDEXED); + } + } + messageData.text = + messageData.text.length <= consts.MAX_PLAINTEXT_CONTENT + ? messageData.text + : messageData.text.substr(0, consts.MAX_PLAINTEXT_CONTENT); - if (r && r.value) { - this.loggelf({ - short_message: '[QUOTA] +', - _mail_action: 'quota', - _user: mailboxData.user, - _inc: size, - _storage_used: r.value.storageUsed, - _sess: options.session && options.session.id, - _mailbox: mailboxData._id - }); + messageData.intro = this.createIntro(messageData.text); } - let rollback = err => { - this.users.collection('users').findOneAndUpdate( - { - _id: mailboxData.user - }, - { - $inc: { - storageUsed: -size - } - }, - { - returnDocument: 'after', - projection: { - storageUsed: true + if (maildata.html && maildata.html.length) { + let htmlSize = 0; + messageData.html = maildata.html + .map(html => { + if (htmlSize >= consts.MAX_HTML_CONTENT || !html) { + return ''; } - }, - (...args) => { - let r = args && args[1]; - - if (r && r.value) { - this.loggelf({ - short_message: '[QUOTA] -', - _mail_action: 'quota', - _user: mailboxData.user, - _inc: -size, - _storage_used: r.value.storageUsed, - _sess: options.session && options.session.id, - _mailbox: mailboxData._id, - _rollback: 'yes', - _error: err.message, - _code: err.code - }); + + if (htmlSize + Buffer.byteLength(html) <= consts.MAX_HTML_CONTENT) { + htmlSize += Buffer.byteLength(html); + return html; } - cleanup(err); - } - ); - }; + html = html.substr(0, consts.MAX_HTML_CONTENT); + htmlSize += Buffer.byteLength(html); + return html; + }) + .filter(html => html); + + // if message has HTML content use it instead of text/plain content for intro + messageData.intro = this.createIntro(htmlToText(messageData.html.join(''))); + } - // acquire new UID+MODSEQ - this.database.collection('mailboxes').findOneAndUpdate( + this.users.collection('users').findOneAndUpdate( { - _id: mailboxData._id + _id: mailboxData.user }, { $inc: { - // allocate bot UID and MODSEQ values so when journal is later sorted by - // modseq then UIDs are always in ascending order - uidNext: 1, - modifyIndex: 1 + storageUsed: size } }, { - // use original value to get correct UIDNext - returnDocument: 'before' + returnDocument: 'after', + projection: { + storageUsed: true + } }, - (err, item) => { + (err, r) => { if (err) { - return rollback(err); - } - - if (!item || !item.value) { - // was not able to acquire a lock - let err = new Error('Mailbox is missing'); - err.imapResponse = 'TRYCREATE'; - return rollback(err); - } - - let mailboxData = item.value; - - // updated message object by setting mailbox specific values - messageData.mailbox = mailboxData._id; - messageData.user = mailboxData.user; - messageData.uid = mailboxData.uidNext; - messageData.modseq = mailboxData.modifyIndex + 1; - - if (!flags.includes('\\Deleted')) { - messageData.searchable = true; + return cleanup(err); } - if (mailboxData.specialUse === '\\Junk') { - messageData.junk = true; + if (r && r.value) { + this.loggelf({ + short_message: '[QUOTA] +', + _mail_action: 'quota', + _user: mailboxData.user, + _inc: size, + _storage_used: r.value.storageUsed, + _sess: options.session && options.session.id, + _mailbox: mailboxData._id + }); } - this.getThreadId(mailboxData.user, subject, mimeTree, (err, thread) => { - if (err) { - return rollback(err); - } + let rollback = err => { + this.users.collection('users').findOneAndUpdate( + { + _id: mailboxData.user + }, + { + $inc: { + storageUsed: -size + } + }, + { + returnDocument: 'after', + projection: { + storageUsed: true + } + }, + (...args) => { + let r = args && args[1]; + + if (r && r.value) { + this.loggelf({ + short_message: '[QUOTA] -', + _mail_action: 'quota', + _user: mailboxData.user, + _inc: -size, + _storage_used: r.value.storageUsed, + _sess: options.session && options.session.id, + _mailbox: mailboxData._id, + _rollback: 'yes', + _error: err.message, + _code: err.code + }); + } - messageData.thread = thread; + cleanup(err); + } + ); + }; - this.database.collection('messages').insertOne(messageData, { writeConcern: 'majority' }, (err, r) => { + // acquire new UID+MODSEQ + this.database.collection('mailboxes').findOneAndUpdate( + { + _id: mailboxData._id + }, + { + $inc: { + // allocate bot UID and MODSEQ values so when journal is later sorted by + // modseq then UIDs are always in ascending order + uidNext: 1, + modifyIndex: 1 + } + }, + { + // use original value to get correct UIDNext + returnDocument: 'before' + }, + (err, item) => { if (err) { return rollback(err); } - if (!r || !r.acknowledged) { - let err = new Error('Failed to store message [1]'); - err.responseCode = 500; - err.code = 'StoreError'; + if (!item || !item.value) { + // was not able to acquire a lock + let err = new Error('Mailbox is missing'); + err.imapResponse = 'TRYCREATE'; return rollback(err); } - let logTime = messageData.meta.time || new Date(); - if (typeof logTime === 'number') { - logTime = new Date(logTime); - } + let mailboxData = item.value; - let uidValidity = mailboxData.uidValidity; - let uid = messageData.uid; + // updated message object by setting mailbox specific values + messageData.mailbox = mailboxData._id; + messageData.user = mailboxData.user; + messageData.uid = mailboxData.uidNext; + messageData.modseq = mailboxData.modifyIndex + 1; - if ( - options.session && - options.session.selected && - options.session.selected.mailbox && - options.session.selected.mailbox.toString() === mailboxData._id.toString() - ) { - options.session.writeStream.write(options.session.formatResponse('EXISTS', messageData.uid)); + if (!flags.includes('\\Deleted')) { + messageData.searchable = true; } - let updateAddressRegister = next => { - let addresses = []; + if (mailboxData.specialUse === '\\Junk') { + messageData.junk = true; + } - if (messageData.junk || flags.includes('\\Draft')) { - // skip junk and draft messages - return next(); + this.getThreadId(mailboxData.user, subject, mimeTree, (err, thread) => { + if (err) { + return rollback(err); } - let parsed = messageData.mimeTree && messageData.mimeTree.parsedHeader; + messageData.thread = thread; - if (parsed) { - let keyList = mailboxData.specialUse === '\\Sent' ? ['to', 'cc', 'bcc'] : ['from']; + this.database.collection('messages').insertOne(messageData, { writeConcern: 'majority' }, (err, r) => { + if (err) { + return rollback(err); + } - for (const disallowedHeader of DISALLOWED_HEADERS_FOR_ADDRESS_REGISTER) { - // if email contains headers that we do not want, - // don't add any emails to address register - if (parsed[disallowedHeader]) { + if (!r || !r.acknowledged) { + let err = new Error('Failed to store message [1]'); + err.responseCode = 500; + err.code = 'StoreError'; + return rollback(err); + } + + let logTime = messageData.meta.time || new Date(); + if (typeof logTime === 'number') { + logTime = new Date(logTime); + } + + let uidValidity = mailboxData.uidValidity; + let uid = messageData.uid; + + if ( + options.session && + options.session.selected && + options.session.selected.mailbox && + options.session.selected.mailbox.toString() === mailboxData._id.toString() + ) { + options.session.writeStream.write(options.session.formatResponse('EXISTS', messageData.uid)); + } + + let updateAddressRegister = next => { + let addresses = []; + + if (messageData.junk || flags.includes('\\Draft')) { + // skip junk and draft messages return next(); } - } - for (let key of keyList) { - if (parsed[key] && parsed[key].length) { - for (let addr of parsed[key]) { - if (/no-?reply/i.test(addr.address)) { - continue; + let parsed = messageData.mimeTree && messageData.mimeTree.parsedHeader; + + if (parsed) { + let keyList = mailboxData.specialUse === '\\Sent' ? ['to', 'cc', 'bcc'] : ['from']; + + for (const disallowedHeader of DISALLOWED_HEADERS_FOR_ADDRESS_REGISTER) { + // if email contains headers that we do not want, + // don't add any emails to address register + if (parsed[disallowedHeader]) { + return next(); } - if (!addresses.some(a => a.address === addr.address)) { - addresses.push(addr); + } + + for (let key of keyList) { + if (parsed[key] && parsed[key].length) { + for (let addr of parsed[key]) { + if (/no-?reply/i.test(addr.address)) { + continue; + } + if (!addresses.some(a => a.address === addr.address)) { + addresses.push(addr); + } + } } } } - } - } - - if (!addresses.length) { - return next(); - } - this.updateAddressRegister(mailboxData.user, addresses) - .then(() => next()) - .catch(err => next(err)); - }; + if (!addresses.length) { + return next(); + } - updateAddressRegister(() => { - this.notifier.addEntries( - mailboxData, - { - command: 'EXISTS', - uid: messageData.uid, - ignore: options.session && options.session.id, - message: messageData._id, - modseq: messageData.modseq, - unseen: messageData.unseen, - idate: messageData.idate, - thread: messageData.thread - }, - () => { - this.notifier.fire(mailboxData.user); - - let raw = options.rawchunks || options.raw; - let processAudits = async () => { - let audits = await this.database - .collection('audits') - .find({ user: mailboxData.user, expires: { $gt: new Date() } }) - .toArray(); - - let now = new Date(); - for (let auditData of audits) { - if ((auditData.start && auditData.start > now) || (auditData.end && auditData.end < now)) { - // audit not active - continue; - } - await this.auditHandler.store(auditData._id, raw, { - date: messageData.idate, - msgid: messageData.msgid, - header: messageData.mimeTree && messageData.mimeTree.parsedHeader, - ha: messageData.ha, - mailbox: mailboxData._id, - mailboxPath: mailboxData.path, - info: Object.assign({ queueId: messageData.outbound }, messageData.meta) - }); - } - }; + this.updateAddressRegister(mailboxData.user, addresses) + .then(() => next()) + .catch(err => next(err)); + }; - let next = () => { - cleanup(null, true, { - uidValidity, - uid, - id: messageData._id.toString(), - mailbox: mailboxData._id.toString(), - mailboxPath: mailboxData.path, - size, - status: 'new' - }); - }; + updateAddressRegister(() => { + this.notifier.addEntries( + mailboxData, + { + command: 'EXISTS', + uid: messageData.uid, + ignore: options.session && options.session.id, + message: messageData._id, + modseq: messageData.modseq, + unseen: messageData.unseen, + idate: messageData.idate, + thread: messageData.thread + }, + () => { + this.notifier.fire(mailboxData.user); - // do not use more suitable .finally() as it is not supported in Node v8 - return processAudits().then(next).catch(next); - } - ); + let raw = options.rawchunks || options.raw; + let processAudits = async () => { + let audits = await this.database + .collection('audits') + .find({ user: mailboxData.user, expires: { $gt: new Date() } }) + .toArray(); + + let now = new Date(); + for (let auditData of audits) { + if ( + (auditData.start && auditData.start > now) || + (auditData.end && auditData.end < now) + ) { + // audit not active + continue; + } + await this.auditHandler.store(auditData._id, raw, { + date: messageData.idate, + msgid: messageData.msgid, + header: messageData.mimeTree && messageData.mimeTree.parsedHeader, + ha: messageData.ha, + mailbox: mailboxData._id, + mailboxPath: mailboxData.path, + info: Object.assign({ queueId: messageData.outbound }, messageData.meta) + }); + } + }; + + let next = () => { + cleanup(null, true, { + uidValidity, + uid, + id: messageData._id.toString(), + mailbox: mailboxData._id.toString(), + mailboxPath: mailboxData.path, + size, + status: 'new' + }); + }; + + // do not use more suitable .finally() as it is not supported in Node v8 + return processAudits().then(next).catch(next); + } + ); + }); + }); }); - }); - }); + } + ); } ); + }); + }; + + if (!alreadyEncrypted) { + // not already encrypted, check if user has encryption on or target mailbox is encrypted + if ((userData.encryptMessages || !!mailboxData.encryptMessages) && userData.pubKey && !flags.includes('\\Draft')) { + // user has encryption on or target mailbox encrypted, encrypt message and prepare again + // do not encrypt drafts + this.encryptMessage(userData.pubKey, options.raw, (err, res) => { + if (err) { + return callback(err); + } + + if (res) { + // new encrypted raw available + options.raw = res; + } + + delete options.prepared; // delete any existing prepared as new will be generated + this.prepareMessage(options, (err, newPrepared) => { + if (err) { + return callback(err); + } + + newPrepared.id = prepared.id; // retain original + + options.prepared = newPrepared; // new prepared in options just in case + prepared = newPrepared; // overwrite top-level original prepared + options.maildata = this.indexer.getMaildata(newPrepared.mimeTree); // get new maildata of encrypted message + addMessage(); + }); + }); + } else { + // not already encrypted and no need to + addMessage(); } - ); + } else { + // message already encrypted + addMessage(); + } }); }); }); @@ -1043,82 +1110,185 @@ class MessageHandler { } } - this.database.collection('messages').insertOne(message, { writeConcern: 'majority' }, (err, r) => { - if (err) { - return cursor.close(() => done(err)); - } + const updateMessage = () => { + this.database.collection('messages').insertOne(message, { writeConcern: 'majority' }, (err, r) => { + if (err) { + return cursor.close(() => done(err)); + } - if (!r || !r.acknowledged) { - let err = new Error('Failed to store message [2]'); - err.responseCode = 500; - err.code = 'StoreError'; - return cursor.close(() => done(err)); - } + if (!r || !r.acknowledged) { + let err = new Error('Failed to store message [2]'); + err.responseCode = 500; + err.code = 'StoreError'; + return cursor.close(() => done(err)); + } - let insertId = r.insertedId; - - // delete old message - this.database.collection('messages').deleteOne( - { - _id: messageId, - mailbox: mailboxData._id, - uid: messageUid - }, - { writeConcern: 'majority' }, - (err, r) => { - if (err) { - return cursor.close(() => done(err)); - } + let insertId = r.insertedId; - if (r && r.deletedCount) { - if (options.session) { - options.session.writeStream.write(options.session.formatResponse('EXPUNGE', sourceUid)); + // delete old message + this.database.collection('messages').deleteOne( + { + _id: messageId, + mailbox: mailboxData._id, + uid: messageUid + }, + { writeConcern: 'majority' }, + (err, r) => { + if (err) { + return cursor.close(() => done(err)); } - removeEntries.push({ - command: 'EXPUNGE', - ignore: options.session && options.session.id, - uid: messageUid, - message: messageId, - unseen, - // modseq is needed to avoid updating mailbox entry - modseq: newModseq - }); + if (r && r.deletedCount) { + if (options.session) { + options.session.writeStream.write(options.session.formatResponse('EXPUNGE', sourceUid)); + } + + removeEntries.push({ + command: 'EXPUNGE', + ignore: options.session && options.session.id, + uid: messageUid, + message: messageId, + unseen, + // modseq is needed to avoid updating mailbox entry + modseq: newModseq + }); + + if (options.showExpunged) { + options.session.writeStream.write(options.session.formatResponse('EXPUNGE', messageUid)); + } + } - if (options.showExpunged) { - options.session.writeStream.write(options.session.formatResponse('EXPUNGE', messageUid)); + let entry = { + command: 'EXISTS', + uid: uidNext, + message: insertId, + unseen: message.unseen, + idate: message.idate, + thread: message.thread + }; + if (junk) { + entry.junk = junk; + } + existsEntries.push(entry); + + if (existsEntries.length >= consts.BULK_BATCH_SIZE) { + // mark messages as deleted from old mailbox + return this.notifier.addEntries(mailboxData, removeEntries, () => { + // mark messages as added to new mailbox + this.notifier.addEntries(targetData, existsEntries, () => { + removeEntries = []; + existsEntries = []; + this.notifier.fire(mailboxData.user); + processNext(); + }); + }); } + processNext(); } - - let entry = { - command: 'EXISTS', - uid: uidNext, - message: insertId, - unseen: message.unseen, - idate: message.idate, - thread: message.thread - }; - if (junk) { - entry.junk = junk; + ); + }); + }; + + if (targetData.encryptMessages) { + // move target mailbox is encrypted + const parsedHeader = (message.mimeTree && message.mimeTree.parsedHeader) || {}; + const parsedContentType = parsedHeader['content-type']; + + if (parsedContentType && parsedContentType.subtype === 'encrypted') { + // message already encrypted, just continue move + updateMessage(); + } else { + // not yet encrypted + this.users.collection('users').findOne({ _id: mailboxData.user }, (err, res) => { + if (err) { + return done(err); } - existsEntries.push(entry); - - if (existsEntries.length >= consts.BULK_BATCH_SIZE) { - // mark messages as deleted from old mailbox - return this.notifier.addEntries(mailboxData, removeEntries, () => { - // mark messages as added to new mailbox - this.notifier.addEntries(targetData, existsEntries, () => { - removeEntries = []; - existsEntries = []; - this.notifier.fire(mailboxData.user); - processNext(); + // get user data + + // get raw from existing mimetree + const outputStream = this.indexer.rebuild(message.mimeTree).value; // get raw rebuilder stream + let raw = Buffer.from([], 'binary'); // set initial raw + outputStream + .on('data', data => { + raw = Buffer.concat([raw, data]); + }) + .on('end', () => { + // when done rebuilding + this.encryptMessage(res.pubKey || '', raw, (err, res) => { + if (err) { + return done(err); + } + + // encrypt rebuilt raw + + if (res) { + // encrypted + this.prepareMessage({ raw: res }, (err, prepared) => { + if (err) { + return done(err); + } + // prepare new message structure from encrypted raw + + prepared.id = message.id; // reuse existing id + + const maildata = this.indexer.getMaildata(prepared.mimeTree); // get new maildata + + // add attachments of encrypted messages + if (maildata.attachments && maildata.attachments.length) { + message.attachments = maildata.attachments; + message.ha = maildata.attachments.some(a => !a.related); + } else { + message.ha = false; + } + + // remove fields that may leak data in FE or DB + delete message.text; + delete message.html; + message.intro = ''; + + this.indexer.storeNodeBodies(maildata, prepared.mimeTree, err => { + // store new attachments + let cleanup = (...args) => { + if (!args[0]) { + return callback(...args); + } + + let attachmentIds = Object.keys(prepared.mimeTree.attachmentMap || {}).map( + key => prepared.mimeTree.attachmentMap[key] + ); + if (!attachmentIds.length) { + return callback(...args); + } + + this.attachmentStorage.deleteMany(attachmentIds, maildata.magic, () => + callback(...args) + ); + }; + + if (err) { + return cleanup(err); + } + + // overwrite required values of existing message with new values + message.mimeTree = prepared.mimeTree; + message.size = prepared.size; + message.bodystructure = prepared.bodystructure; + message.envelope = prepared.envelope; + message.headers = prepared.headers; + updateMessage(); + }); + }); + } else { + updateMessage(); + } }); }); - } - processNext(); - } - ); - }); + }); + } + } else { + // move target is not encrypted so proceed + updateMessage(); + } } ); }); From fa849f2fb4b4e66d6a5abf4a6ce0e4064f42e1fa Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Mon, 11 Nov 2024 10:36:00 +0200 Subject: [PATCH 4/5] encrypt messages copied into encrypted mailbox --- lib/handlers/on-copy.js | 90 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/lib/handlers/on-copy.js b/lib/handlers/on-copy.js index a692396c..25a319c8 100644 --- a/lib/handlers/on-copy.js +++ b/lib/handlers/on-copy.js @@ -132,6 +132,12 @@ async function copyHandler(server, messageHandler, connection, mailbox, update, notifyLongRunning(); + let targetMailboxEncrypted = false; + + if (targetData.encryptMessages) { + targetMailboxEncrypted = true; + } + try { while ((messageData = await cursor.next())) { tools.checkSocket(socket); // do we even have to copy anything? @@ -141,6 +147,12 @@ async function copyHandler(server, messageHandler, connection, mailbox, update, uid: messageData.uid, _id: messageData._id }; + + const parsedHeader = (messageData.mimeTree && messageData.mimeTree.parsedHeader) || {}; + const parsedContentType = parsedHeader['content-type']; + + const isMessageEncrypted = parsedContentType ? parsedContentType.subtype === 'encrypted' : false; + // Copying is not done in bulk to minimize risk of going out of sync with incremental UIDs sourceUid.unshift(messageData.uid); let item = await db.database.collection('mailboxes').findOneAndUpdate( @@ -218,6 +230,84 @@ async function copyHandler(server, messageHandler, connection, mailbox, update, { writeConcern: 'majority' } ); + const newPrepared = await new Promise((resolve, reject) => { + if (targetMailboxEncrypted && !isMessageEncrypted && userData.pubKey) { + // encrypt message + const outputStream = messageHandler.indexer.rebuild(messageData.mimeTree).value; // get raw rebuilder stream + let raw = Buffer.from([], 'binary'); // set initial raw + + outputStream + .on('data', data => { + raw = Buffer.concat([raw, data]); + }) + .on('end', () => { + messageHandler.encryptMessages(userData.pubKey || '', raw, (err, res) => { + if (err) { + return reject(err); + } + + // encrypted rebuilt raw + + if (res) { + messageHandler.prepareMessage({ raw: res }, (err, prepared) => { + if (err) { + return reject(err); + } + // prepared new message structure from encrypted raw + + const maildata = messageHandler.indexer.getMaildata(prepared.mimeTree); + + // add attachments of encrypted messages + if (maildata.attachments && maildata.attachments.length) { + messageData.attachments = maildata.attachments; + messageData.ha = maildata.attachments.some(a => !a.related); + } else { + messageData.ha = false; + } + + // remove fields that may leak data in FE or DB + delete messageData.text; + delete messageData.html; + messageData.intro = ''; + + messageHandler.indexer.storeNodeBodies(maildata, prepared.mimeTree, err => { + // store new attachments + let cleanup = () => { + let attachmentIds = Object.keys(prepared.mimeTree.attachmentMap || {}).map( + key => prepared.mimeTree.attachmentMap[key] + ); + + messageHandler.attachmentStorage.deleteMany(attachmentIds, maildata.magic); + + if (err) { + return reject(err); + } + }; + + if (err) { + return cleanup(err); + } + + return resolve(prepared); + }); + }); + } + }); + }); + } else { + resolve(false); + } + }); + + // replace fields + if (newPrepared) { + messageData.mimeTree = newPrepared.mimeTree; + messageData.size = newPrepared.size; + messageData.bodystructure = newPrepared.bodystructure; + messageData.envelope = newPrepared.envelope; + messageData.headers = newPrepared.headers; + } + let r = await db.database.collection('messages').insertOne(messageData, { writeConcern: 'majority' }); if (!r || !r.acknowledged) { From d1c67e6264f7f01f52646404ad1b38fef7cdbd3f Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Wed, 4 Dec 2024 21:39:08 +0200 Subject: [PATCH 5/5] fix streams in on-copy and message-handler. message-handler optimizations, filter-handler optimizations --- lib/filter-handler.js | 15 +- lib/handlers/on-copy.js | 22 +- lib/message-handler.js | 803 +++++++++++++++++++++------------------- 3 files changed, 453 insertions(+), 387 deletions(-) diff --git a/lib/filter-handler.js b/lib/filter-handler.js index 3bda7899..ab5ba65d 100644 --- a/lib/filter-handler.js +++ b/lib/filter-handler.js @@ -141,10 +141,9 @@ class FilterHandler { let rawchunks = chunks; - let prepared; - - let raw = Buffer.concat(chunks, chunklen); + let raw; + let prepared; if (options.mimeTree) { if (options.mimeTree && options.mimeTree.header) { // remove old headers @@ -159,6 +158,7 @@ class FilterHandler { mimeTree: options.mimeTree }); } else { + raw = Buffer.concat(chunks, chunklen); prepared = await this.prepareMessage({ raw }); @@ -661,11 +661,14 @@ class FilterHandler { date: false, flags, - - raw, - rawchunks + rawchunks, + chunklen }; + if (raw) { + messageOpts.raw = raw; + } + if (options.verificationResults) { messageOpts.verificationResults = options.verificationResults; } diff --git a/lib/handlers/on-copy.js b/lib/handlers/on-copy.js index 25a319c8..8e802358 100644 --- a/lib/handlers/on-copy.js +++ b/lib/handlers/on-copy.js @@ -233,15 +233,27 @@ async function copyHandler(server, messageHandler, connection, mailbox, update, const newPrepared = await new Promise((resolve, reject) => { if (targetMailboxEncrypted && !isMessageEncrypted && userData.pubKey) { // encrypt message - const outputStream = messageHandler.indexer.rebuild(messageData.mimeTree).value; // get raw rebuilder stream - let raw = Buffer.from([], 'binary'); // set initial raw + // get raw from existing mimetree + let outputStream = messageHandler.indexer.rebuild(messageData.mimeTree); // get raw rebuilder response obj (.value is the stream) + if (!outputStream || outputStream.type !== 'stream' || !outputStream.value) { + return reject(new Error('Cannot fetch message')); + } + outputStream = outputStream.value; // set stream to actual stream object (.value) + + let chunks = []; + let chunklen = 0; outputStream - .on('data', data => { - raw = Buffer.concat([raw, data]); + .on('readable', () => { + let chunk; + while ((chunk = outputStream.read()) !== null) { + chunks.push(chunk); + chunklen += chunk.length; + } }) .on('end', () => { - messageHandler.encryptMessages(userData.pubKey || '', raw, (err, res) => { + const raw = Buffer.concat(chunks, chunklen); + messageHandler.encryptMessages(userData.pubKey, raw, (err, res) => { if (err) { return reject(err); } diff --git a/lib/message-handler.js b/lib/message-handler.js index a9d97efb..e9544217 100644 --- a/lib/message-handler.js +++ b/lib/message-handler.js @@ -204,474 +204,510 @@ class MessageHandler { } // get target user data + let prepared = options.prepared; // might be undefined - this.prepareMessage(options, (err, prepared) => { - if (err) { - return callback(err); - } + // check if already encrypted + let alreadyEncrypted = false; - // check if already encrypted - let alreadyEncrypted = false; - - // message already prepared, check if encrypted + // message already prepared, check if encrypted + if (prepared) { + // got prepared const parsedHeader = (prepared.mimeTree && prepared.mimeTree?.parsedHeader) || {}; const parsedContentType = parsedHeader['content-type']; if (parsedContentType && parsedContentType.subtype === 'encrypted') { alreadyEncrypted = true; } + } else { + // no prepared, use raw + if (options.rawchunks && !options.raw) { + // got rawchunks instead of raw + if (options.chunklen) { + options.raw = Buffer.concat(options.rawchunks, options.chunklen); + } else { + options.raw = Buffer.concat(options.rawchunks); + } + } - let flags = Array.isArray(options.flags) ? options.flags : [].concat(options.flags || []); - - let addMessage = () => { - let id = prepared.id; - let mimeTree = prepared.mimeTree; - let size = prepared.size; - let bodystructure = prepared.bodystructure; - let envelope = prepared.envelope; - let idate = prepared.idate; - let hdate = prepared.hdate; - let msgid = prepared.msgid; - let subject = prepared.subject; - let headers = prepared.headers; - - let maildata = options.maildata || this.indexer.getMaildata(mimeTree); - - let cleanup = (...args) => { - if (!args[0]) { - return callback(...args); - } + const rawString = options.raw.toString('binary'); // get string from the raw bytes of message + const regex = /Content-Type:\s*multipart\/encrypted/gim; - let attachmentIds = Object.keys(mimeTree.attachmentMap || {}).map(key => mimeTree.attachmentMap[key]); - if (!attachmentIds.length) { - return callback(...args); - } + if (regex.test(rawString)) { + // if there is encrypted content-type then message already encrypted, no need to re-encrypt it + alreadyEncrypted = true; + } + } - this.attachmentStorage.deleteMany(attachmentIds, maildata.magic, () => callback(...args)); - }; + let flags = Array.isArray(options.flags) ? options.flags : [].concat(options.flags || []); + + let addMessage = () => { + let id = prepared.id; + let mimeTree = prepared.mimeTree; + let size = prepared.size; + let bodystructure = prepared.bodystructure; + let envelope = prepared.envelope; + let idate = prepared.idate; + let hdate = prepared.hdate; + let msgid = prepared.msgid; + let subject = prepared.subject; + let headers = prepared.headers; + + let maildata = options.maildata || this.indexer.getMaildata(mimeTree); + + let cleanup = (...args) => { + if (!args[0]) { + return callback(...args); + } - this.indexer.storeNodeBodies(maildata, mimeTree, err => { - if (err) { - return cleanup(err); - } + let attachmentIds = Object.keys(mimeTree.attachmentMap || {}).map(key => mimeTree.attachmentMap[key]); + if (!attachmentIds.length) { + return callback(...args); + } - // prepare message object - let messageData = { - _id: id, + this.attachmentStorage.deleteMany(attachmentIds, maildata.magic, () => callback(...args)); + }; - // should be kept when COPY'ing or MOVE'ing - root: id, + this.indexer.storeNodeBodies(maildata, mimeTree, err => { + if (err) { + return cleanup(err); + } - v: consts.SCHEMA_VERSION, + // prepare message object + let messageData = { + _id: id, - // if true then expires after rdate + retention - exp: !!mailboxData.retention, - rdate: Date.now() + (mailboxData.retention || 0), + // should be kept when COPY'ing or MOVE'ing + root: id, - // make sure the field exists. it is set to true when user is deleted - userDeleted: false, + v: consts.SCHEMA_VERSION, - idate, - hdate, - flags, - size, + // if true then expires after rdate + retention + exp: !!mailboxData.retention, + rdate: Date.now() + (mailboxData.retention || 0), - // some custom metadata about the delivery - meta: options.meta || {}, + // make sure the field exists. it is set to true when user is deleted + userDeleted: false, - // list filter IDs that matched this message - filters: Array.isArray(options.filters) ? options.filters : [].concat(options.filters || []), + idate, + hdate, + flags, + size, - headers, - mimeTree, - envelope, - bodystructure, - msgid, + // some custom metadata about the delivery + meta: options.meta || {}, - // use boolean for more commonly used (and searched for) flags - unseen: !flags.includes('\\Seen'), - flagged: flags.includes('\\Flagged'), - undeleted: !flags.includes('\\Deleted'), - draft: flags.includes('\\Draft'), + // list filter IDs that matched this message + filters: Array.isArray(options.filters) ? options.filters : [].concat(options.filters || []), - magic: maildata.magic, + headers, + mimeTree, + envelope, + bodystructure, + msgid, - subject, + // use boolean for more commonly used (and searched for) flags + unseen: !flags.includes('\\Seen'), + flagged: flags.includes('\\Flagged'), + undeleted: !flags.includes('\\Deleted'), + draft: flags.includes('\\Draft'), - // do not archive deleted messages that have been copied - copied: false - }; + magic: maildata.magic, - if (options.verificationResults) { - messageData.verificationResults = options.verificationResults; - } + subject, - if (options.outbound) { - messageData.outbound = [].concat(options.outbound || []); - } + // do not archive deleted messages that have been copied + copied: false + }; - if (options.forwardTargets) { - messageData.forwardTargets = [].concat(options.forwardTargets || []); - } + if (options.verificationResults) { + messageData.verificationResults = options.verificationResults; + } - if (maildata.attachments && maildata.attachments.length) { - messageData.attachments = maildata.attachments; - messageData.ha = maildata.attachments.some(a => !a.related); - } else { - messageData.ha = false; - } + if (options.outbound) { + messageData.outbound = [].concat(options.outbound || []); + } - if (maildata.text) { - messageData.text = maildata.text.replace(/\r\n/g, '\n').trim(); + if (options.forwardTargets) { + messageData.forwardTargets = [].concat(options.forwardTargets || []); + } - // text is indexed with a fulltext index, so only store the beginning of it - if (messageData.text.length > consts.MAX_PLAINTEXT_INDEXED) { - messageData.textFooter = messageData.text.substr(consts.MAX_PLAINTEXT_INDEXED); - messageData.text = messageData.text.substr(0, consts.MAX_PLAINTEXT_INDEXED); + if (maildata.attachments && maildata.attachments.length) { + messageData.attachments = maildata.attachments; + messageData.ha = maildata.attachments.some(a => !a.related); + } else { + messageData.ha = false; + } - // truncate remaining text if total length exceeds maximum allowed - if ( - consts.MAX_PLAINTEXT_CONTENT > consts.MAX_PLAINTEXT_INDEXED && - messageData.textFooter.length > consts.MAX_PLAINTEXT_CONTENT - consts.MAX_PLAINTEXT_INDEXED - ) { - messageData.textFooter = messageData.textFooter.substr(0, consts.MAX_PLAINTEXT_CONTENT - consts.MAX_PLAINTEXT_INDEXED); - } - } - messageData.text = - messageData.text.length <= consts.MAX_PLAINTEXT_CONTENT - ? messageData.text - : messageData.text.substr(0, consts.MAX_PLAINTEXT_CONTENT); + if (maildata.text) { + messageData.text = maildata.text.replace(/\r\n/g, '\n').trim(); + + // text is indexed with a fulltext index, so only store the beginning of it + if (messageData.text.length > consts.MAX_PLAINTEXT_INDEXED) { + messageData.textFooter = messageData.text.substr(consts.MAX_PLAINTEXT_INDEXED); + messageData.text = messageData.text.substr(0, consts.MAX_PLAINTEXT_INDEXED); - messageData.intro = this.createIntro(messageData.text); + // truncate remaining text if total length exceeds maximum allowed + if ( + consts.MAX_PLAINTEXT_CONTENT > consts.MAX_PLAINTEXT_INDEXED && + messageData.textFooter.length > consts.MAX_PLAINTEXT_CONTENT - consts.MAX_PLAINTEXT_INDEXED + ) { + messageData.textFooter = messageData.textFooter.substr(0, consts.MAX_PLAINTEXT_CONTENT - consts.MAX_PLAINTEXT_INDEXED); + } } + messageData.text = + messageData.text.length <= consts.MAX_PLAINTEXT_CONTENT + ? messageData.text + : messageData.text.substr(0, consts.MAX_PLAINTEXT_CONTENT); - if (maildata.html && maildata.html.length) { - let htmlSize = 0; - messageData.html = maildata.html - .map(html => { - if (htmlSize >= consts.MAX_HTML_CONTENT || !html) { - return ''; - } + messageData.intro = this.createIntro(messageData.text); + } - if (htmlSize + Buffer.byteLength(html) <= consts.MAX_HTML_CONTENT) { - htmlSize += Buffer.byteLength(html); - return html; - } + if (maildata.html && maildata.html.length) { + let htmlSize = 0; + messageData.html = maildata.html + .map(html => { + if (htmlSize >= consts.MAX_HTML_CONTENT || !html) { + return ''; + } - html = html.substr(0, consts.MAX_HTML_CONTENT); + if (htmlSize + Buffer.byteLength(html) <= consts.MAX_HTML_CONTENT) { htmlSize += Buffer.byteLength(html); return html; - }) - .filter(html => html); - - // if message has HTML content use it instead of text/plain content for intro - messageData.intro = this.createIntro(htmlToText(messageData.html.join(''))); - } - - this.users.collection('users').findOneAndUpdate( - { - _id: mailboxData.user - }, - { - $inc: { - storageUsed: size - } - }, - { - returnDocument: 'after', - projection: { - storageUsed: true - } - }, - (err, r) => { - if (err) { - return cleanup(err); } - if (r && r.value) { - this.loggelf({ - short_message: '[QUOTA] +', - _mail_action: 'quota', - _user: mailboxData.user, - _inc: size, - _storage_used: r.value.storageUsed, - _sess: options.session && options.session.id, - _mailbox: mailboxData._id - }); - } + html = html.substr(0, consts.MAX_HTML_CONTENT); + htmlSize += Buffer.byteLength(html); + return html; + }) + .filter(html => html); - let rollback = err => { - this.users.collection('users').findOneAndUpdate( - { - _id: mailboxData.user - }, - { - $inc: { - storageUsed: -size - } - }, - { - returnDocument: 'after', - projection: { - storageUsed: true - } - }, - (...args) => { - let r = args && args[1]; - - if (r && r.value) { - this.loggelf({ - short_message: '[QUOTA] -', - _mail_action: 'quota', - _user: mailboxData.user, - _inc: -size, - _storage_used: r.value.storageUsed, - _sess: options.session && options.session.id, - _mailbox: mailboxData._id, - _rollback: 'yes', - _error: err.message, - _code: err.code - }); - } + // if message has HTML content use it instead of text/plain content for intro + messageData.intro = this.createIntro(htmlToText(messageData.html.join(''))); + } - cleanup(err); - } - ); - }; + this.users.collection('users').findOneAndUpdate( + { + _id: mailboxData.user + }, + { + $inc: { + storageUsed: size + } + }, + { + returnDocument: 'after', + projection: { + storageUsed: true + } + }, + (err, r) => { + if (err) { + return cleanup(err); + } - // acquire new UID+MODSEQ - this.database.collection('mailboxes').findOneAndUpdate( + if (r && r.value) { + this.loggelf({ + short_message: '[QUOTA] +', + _mail_action: 'quota', + _user: mailboxData.user, + _inc: size, + _storage_used: r.value.storageUsed, + _sess: options.session && options.session.id, + _mailbox: mailboxData._id + }); + } + + let rollback = err => { + this.users.collection('users').findOneAndUpdate( { - _id: mailboxData._id + _id: mailboxData.user }, { $inc: { - // allocate bot UID and MODSEQ values so when journal is later sorted by - // modseq then UIDs are always in ascending order - uidNext: 1, - modifyIndex: 1 + storageUsed: -size } }, { - // use original value to get correct UIDNext - returnDocument: 'before' + returnDocument: 'after', + projection: { + storageUsed: true + } }, - (err, item) => { - if (err) { - return rollback(err); + (...args) => { + let r = args && args[1]; + + if (r && r.value) { + this.loggelf({ + short_message: '[QUOTA] -', + _mail_action: 'quota', + _user: mailboxData.user, + _inc: -size, + _storage_used: r.value.storageUsed, + _sess: options.session && options.session.id, + _mailbox: mailboxData._id, + _rollback: 'yes', + _error: err.message, + _code: err.code + }); } - if (!item || !item.value) { - // was not able to acquire a lock - let err = new Error('Mailbox is missing'); - err.imapResponse = 'TRYCREATE'; - return rollback(err); - } + cleanup(err); + } + ); + }; + + // acquire new UID+MODSEQ + this.database.collection('mailboxes').findOneAndUpdate( + { + _id: mailboxData._id + }, + { + $inc: { + // allocate bot UID and MODSEQ values so when journal is later sorted by + // modseq then UIDs are always in ascending order + uidNext: 1, + modifyIndex: 1 + } + }, + { + // use original value to get correct UIDNext + returnDocument: 'before' + }, + (err, item) => { + if (err) { + return rollback(err); + } + + if (!item || !item.value) { + // was not able to acquire a lock + let err = new Error('Mailbox is missing'); + err.imapResponse = 'TRYCREATE'; + return rollback(err); + } - let mailboxData = item.value; + let mailboxData = item.value; - // updated message object by setting mailbox specific values - messageData.mailbox = mailboxData._id; - messageData.user = mailboxData.user; - messageData.uid = mailboxData.uidNext; - messageData.modseq = mailboxData.modifyIndex + 1; + // updated message object by setting mailbox specific values + messageData.mailbox = mailboxData._id; + messageData.user = mailboxData.user; + messageData.uid = mailboxData.uidNext; + messageData.modseq = mailboxData.modifyIndex + 1; - if (!flags.includes('\\Deleted')) { - messageData.searchable = true; - } + if (!flags.includes('\\Deleted')) { + messageData.searchable = true; + } + + if (mailboxData.specialUse === '\\Junk') { + messageData.junk = true; + } - if (mailboxData.specialUse === '\\Junk') { - messageData.junk = true; + this.getThreadId(mailboxData.user, subject, mimeTree, (err, thread) => { + if (err) { + return rollback(err); } - this.getThreadId(mailboxData.user, subject, mimeTree, (err, thread) => { + messageData.thread = thread; + + this.database.collection('messages').insertOne(messageData, { writeConcern: 'majority' }, (err, r) => { if (err) { return rollback(err); } - messageData.thread = thread; + if (!r || !r.acknowledged) { + let err = new Error('Failed to store message [1]'); + err.responseCode = 500; + err.code = 'StoreError'; + return rollback(err); + } - this.database.collection('messages').insertOne(messageData, { writeConcern: 'majority' }, (err, r) => { - if (err) { - return rollback(err); - } + let logTime = messageData.meta.time || new Date(); + if (typeof logTime === 'number') { + logTime = new Date(logTime); + } - if (!r || !r.acknowledged) { - let err = new Error('Failed to store message [1]'); - err.responseCode = 500; - err.code = 'StoreError'; - return rollback(err); - } + let uidValidity = mailboxData.uidValidity; + let uid = messageData.uid; - let logTime = messageData.meta.time || new Date(); - if (typeof logTime === 'number') { - logTime = new Date(logTime); - } + if ( + options.session && + options.session.selected && + options.session.selected.mailbox && + options.session.selected.mailbox.toString() === mailboxData._id.toString() + ) { + options.session.writeStream.write(options.session.formatResponse('EXISTS', messageData.uid)); + } - let uidValidity = mailboxData.uidValidity; - let uid = messageData.uid; + let updateAddressRegister = next => { + let addresses = []; - if ( - options.session && - options.session.selected && - options.session.selected.mailbox && - options.session.selected.mailbox.toString() === mailboxData._id.toString() - ) { - options.session.writeStream.write(options.session.formatResponse('EXISTS', messageData.uid)); + if (messageData.junk || flags.includes('\\Draft')) { + // skip junk and draft messages + return next(); } - let updateAddressRegister = next => { - let addresses = []; + let parsed = messageData.mimeTree && messageData.mimeTree.parsedHeader; - if (messageData.junk || flags.includes('\\Draft')) { - // skip junk and draft messages - return next(); - } - - let parsed = messageData.mimeTree && messageData.mimeTree.parsedHeader; - - if (parsed) { - let keyList = mailboxData.specialUse === '\\Sent' ? ['to', 'cc', 'bcc'] : ['from']; + if (parsed) { + let keyList = mailboxData.specialUse === '\\Sent' ? ['to', 'cc', 'bcc'] : ['from']; - for (const disallowedHeader of DISALLOWED_HEADERS_FOR_ADDRESS_REGISTER) { - // if email contains headers that we do not want, - // don't add any emails to address register - if (parsed[disallowedHeader]) { - return next(); - } + for (const disallowedHeader of DISALLOWED_HEADERS_FOR_ADDRESS_REGISTER) { + // if email contains headers that we do not want, + // don't add any emails to address register + if (parsed[disallowedHeader]) { + return next(); } + } - for (let key of keyList) { - if (parsed[key] && parsed[key].length) { - for (let addr of parsed[key]) { - if (/no-?reply/i.test(addr.address)) { - continue; - } - if (!addresses.some(a => a.address === addr.address)) { - addresses.push(addr); - } + for (let key of keyList) { + if (parsed[key] && parsed[key].length) { + for (let addr of parsed[key]) { + if (/no-?reply/i.test(addr.address)) { + continue; + } + if (!addresses.some(a => a.address === addr.address)) { + addresses.push(addr); } } } } + } - if (!addresses.length) { - return next(); - } + if (!addresses.length) { + return next(); + } - this.updateAddressRegister(mailboxData.user, addresses) - .then(() => next()) - .catch(err => next(err)); - }; - - updateAddressRegister(() => { - this.notifier.addEntries( - mailboxData, - { - command: 'EXISTS', - uid: messageData.uid, - ignore: options.session && options.session.id, - message: messageData._id, - modseq: messageData.modseq, - unseen: messageData.unseen, - idate: messageData.idate, - thread: messageData.thread - }, - () => { - this.notifier.fire(mailboxData.user); - - let raw = options.rawchunks || options.raw; - let processAudits = async () => { - let audits = await this.database - .collection('audits') - .find({ user: mailboxData.user, expires: { $gt: new Date() } }) - .toArray(); - - let now = new Date(); - for (let auditData of audits) { - if ( - (auditData.start && auditData.start > now) || - (auditData.end && auditData.end < now) - ) { - // audit not active - continue; - } - await this.auditHandler.store(auditData._id, raw, { - date: messageData.idate, - msgid: messageData.msgid, - header: messageData.mimeTree && messageData.mimeTree.parsedHeader, - ha: messageData.ha, - mailbox: mailboxData._id, - mailboxPath: mailboxData.path, - info: Object.assign({ queueId: messageData.outbound }, messageData.meta) - }); + this.updateAddressRegister(mailboxData.user, addresses) + .then(() => next()) + .catch(err => next(err)); + }; + + updateAddressRegister(() => { + this.notifier.addEntries( + mailboxData, + { + command: 'EXISTS', + uid: messageData.uid, + ignore: options.session && options.session.id, + message: messageData._id, + modseq: messageData.modseq, + unseen: messageData.unseen, + idate: messageData.idate, + thread: messageData.thread + }, + () => { + this.notifier.fire(mailboxData.user); + + let raw = options.rawchunks || options.raw; + let processAudits = async () => { + let audits = await this.database + .collection('audits') + .find({ user: mailboxData.user, expires: { $gt: new Date() } }) + .toArray(); + + let now = new Date(); + for (let auditData of audits) { + if ((auditData.start && auditData.start > now) || (auditData.end && auditData.end < now)) { + // audit not active + continue; } - }; - - let next = () => { - cleanup(null, true, { - uidValidity, - uid, - id: messageData._id.toString(), - mailbox: mailboxData._id.toString(), + await this.auditHandler.store(auditData._id, raw, { + date: messageData.idate, + msgid: messageData.msgid, + header: messageData.mimeTree && messageData.mimeTree.parsedHeader, + ha: messageData.ha, + mailbox: mailboxData._id, mailboxPath: mailboxData.path, - size, - status: 'new' + info: Object.assign({ queueId: messageData.outbound }, messageData.meta) }); - }; + } + }; + + let next = () => { + cleanup(null, true, { + uidValidity, + uid, + id: messageData._id.toString(), + mailbox: mailboxData._id.toString(), + mailboxPath: mailboxData.path, + size, + status: 'new' + }); + }; - // do not use more suitable .finally() as it is not supported in Node v8 - return processAudits().then(next).catch(next); - } - ); - }); + // do not use more suitable .finally() as it is not supported in Node v8 + return processAudits().then(next).catch(next); + } + ); }); }); - } - ); - } - ); - }); - }; + }); + } + ); + } + ); + }); + }; - if (!alreadyEncrypted) { - // not already encrypted, check if user has encryption on or target mailbox is encrypted - if ((userData.encryptMessages || !!mailboxData.encryptMessages) && userData.pubKey && !flags.includes('\\Draft')) { - // user has encryption on or target mailbox encrypted, encrypt message and prepare again - // do not encrypt drafts - this.encryptMessage(userData.pubKey, options.raw, (err, res) => { + if (!alreadyEncrypted) { + // not already encrypted, check if user has encryption on or target mailbox is encrypted + if ((userData.encryptMessages || !!mailboxData.encryptMessages) && userData.pubKey && !flags.includes('\\Draft')) { + if (options.rawchunks && !options.raw) { + // got rawchunks instead of raw + if (options.chunklen) { + options.raw = Buffer.concat(options.rawchunks, options.chunklen); + } else { + options.raw = Buffer.concat(options.rawchunks); + } + } + // user has encryption on or target mailbox encrypted, encrypt message and prepare again + // do not encrypt drafts + // may have a situation where we got prepared and no options.raw but options.rawchunks instead, concat them + this.encryptMessage(userData.pubKey, options.raw, (err, res) => { + if (err) { + return callback(err); + } + + if (res) { + // new encrypted raw available + options.raw = res; + } + + delete options.prepared; // delete any existing prepared as new will be generated + this.prepareMessage(options, (err, newPrepared) => { if (err) { return callback(err); } - if (res) { - // new encrypted raw available - options.raw = res; - } - - delete options.prepared; // delete any existing prepared as new will be generated - this.prepareMessage(options, (err, newPrepared) => { - if (err) { - return callback(err); - } + newPrepared.id = prepared.id; // retain original - newPrepared.id = prepared.id; // retain original - - options.prepared = newPrepared; // new prepared in options just in case - prepared = newPrepared; // overwrite top-level original prepared - options.maildata = this.indexer.getMaildata(newPrepared.mimeTree); // get new maildata of encrypted message - addMessage(); - }); + options.prepared = newPrepared; // new prepared in options just in case + prepared = newPrepared; // overwrite top-level original prepared + options.maildata = this.indexer.getMaildata(newPrepared.mimeTree); // get new maildata of encrypted message + addMessage(); }); - } else { - // not already encrypted and no need to + }); + } else { + // not already encrypted and no need to + this.prepareMessage(options, (err, newPrepared) => { + if (err) { + return callback(err); + } + + prepared = newPrepared; addMessage(); + }); + } + } else { + // message already encrypted + this.prepareMessage(options, (err, newPrepared) => { + if (err) { + return callback(err); } - } else { - // message already encrypted + + prepared = newPrepared; addMessage(); - } - }); + }); + } }); }); } @@ -1204,17 +1240,32 @@ class MessageHandler { return done(err); } // get user data + if (!res.pubKey) { + return updateMessage(); + } // get raw from existing mimetree - const outputStream = this.indexer.rebuild(message.mimeTree).value; // get raw rebuilder stream - let raw = Buffer.from([], 'binary'); // set initial raw + let outputStream = this.indexer.rebuild(message.mimeTree); // get raw rebuilder response obj (.value is the stream) + + if (!outputStream || outputStream.type !== 'stream' || !outputStream.value) { + return done(new Error('Cannot fetch message')); + } + outputStream = outputStream.value; // set stream to actual stream object (.value) + + let chunks = []; + let chunklen = 0; outputStream - .on('data', data => { - raw = Buffer.concat([raw, data]); + .on('readable', () => { + let chunk; + while ((chunk = outputStream.read()) !== null) { + chunks.push(chunk); + chunklen += chunk.length; + } }) .on('end', () => { // when done rebuilding - this.encryptMessage(res.pubKey || '', raw, (err, res) => { + const raw = Buffer.concat(chunks, chunklen); + this.encryptMessage(res.pubKey, raw, (err, res) => { if (err) { return done(err); }