From d45027738f645a1ae0f96bd1ceb67255b524f9ec Mon Sep 17 00:00:00 2001 From: Pierre-Gilles Leymarie Date: Thu, 13 Jun 2024 09:57:02 +0200 Subject: [PATCH] Gladys Plus : Send Telegram message when backup failed (#2099) --- server/config/brain/backup/answers.en.json | 8 + server/config/brain/backup/answers.fr.json | 8 + server/lib/gateway/gateway.backup.js | 197 ++++++++++-------- .../test/lib/gateway/gateway.backup.test.js | 22 +- 4 files changed, 142 insertions(+), 93 deletions(-) create mode 100644 server/config/brain/backup/answers.en.json create mode 100644 server/config/brain/backup/answers.fr.json diff --git a/server/config/brain/backup/answers.en.json b/server/config/brain/backup/answers.en.json new file mode 100644 index 0000000000..2be7321fc6 --- /dev/null +++ b/server/config/brain/backup/answers.en.json @@ -0,0 +1,8 @@ +[ + { + "label": "backup.fail", + "answers": [ + "The Gladys Plus backup failed. For more information, go to the \"Settings\" => \"Tasks\" tab. The error message is {{ errorMessage }}" + ] + } +] diff --git a/server/config/brain/backup/answers.fr.json b/server/config/brain/backup/answers.fr.json new file mode 100644 index 0000000000..9c2e09139c --- /dev/null +++ b/server/config/brain/backup/answers.fr.json @@ -0,0 +1,8 @@ +[ + { + "label": "backup.fail", + "answers": [ + "La sauvegarde Gladys Plus a échoué. Pour en savoir plus, rendez-vous sur l'onglet \"Paramètres\" => \"Tâches\". Le message d'erreur est {{ errorMessage }}" + ] + } +] diff --git a/server/lib/gateway/gateway.backup.js b/server/lib/gateway/gateway.backup.js index cfe7b4b8a5..ae4e9fa27f 100644 --- a/server/lib/gateway/gateway.backup.js +++ b/server/lib/gateway/gateway.backup.js @@ -9,6 +9,7 @@ const logger = require('../../utils/logger'); const { exec } = require('../../utils/childProcess'); const { readChunk } = require('../../utils/readChunk'); const { NotFoundError } = require('../../utils/coreErrors'); +const { USER_ROLE } = require('../../utils/constants'); const BACKUP_NAME_BASE = 'gladys-db-backup'; @@ -32,107 +33,119 @@ const UPLOAD_ONE_CHUNK_RETRY_OPTIONS = { * backup(); */ async function backup(jobId) { - const encryptKey = await this.variable.getValue('GLADYS_GATEWAY_BACKUP_KEY'); - if (encryptKey === null) { - throw new NotFoundError('GLADYS_GATEWAY_BACKUP_KEY_NOT_FOUND'); - } - const systemInfos = await this.system.getInfos(); - const now = new Date(); - const date = `${now.getFullYear()}-${now.getMonth() + - 1}-${now.getDate()}-${now.getHours()}-${now.getMinutes()}-${now.getSeconds()}`; - const backupFileName = `${BACKUP_NAME_BASE}-${date}.db`; - const backupFilePath = path.join(this.config.backupsFolder, backupFileName); - const compressedBackupFilePath = `${backupFilePath}.gz`; - const encryptedBackupFilePath = `${compressedBackupFilePath}.enc`; - // we ensure the backup folder exists - await fse.ensureDir(this.config.backupsFolder); - // we lock the database - logger.info(`Gateway backup: Locking Database`); - // It's possible to get "Cannot start a transaction within a transaction" errors - // So we might want to retry this part a few times - await retry(async (bail, attempt) => { - await db.sequelize.transaction({ type: Sequelize.Transaction.TYPES.IMMEDIATE }, async () => { - logger.info(`Backup attempt n°${attempt} : Cleaning backup folder`); - // we delete old backups - await fse.emptyDir(this.config.backupsFolder); - // We backup database - logger.info(`Starting Gateway backup in folder ${backupFilePath}`); - await exec(`sqlite3 ${this.config.storage} ".backup '${backupFilePath}'"`); - logger.info(`Gateway backup: Unlocking Database`); - }); - }, SQLITE_BACKUP_RETRY_OPTIONS); - await this.job.updateProgress(jobId, 10); - const fileInfos = await fsPromise.stat(backupFilePath); - const fileSizeMB = Math.round(fileInfos.size / 1024 / 1024); - logger.info(`Gateway backup : Success! File size is ${fileSizeMB}mb.`); - // compress backup - logger.info(`Gateway backup: Gzipping backup`); - await exec(`gzip ${backupFilePath}`); - await this.job.updateProgress(jobId, 20); - // encrypt backup - logger.info(`Gateway backup: Encrypting backup`); - await exec( - `openssl enc -aes-256-cbc -pass pass:${encryptKey} -in ${compressedBackupFilePath} -out ${encryptedBackupFilePath}`, - ); - await this.job.updateProgress(jobId, 30); - // Upload file to the Gladys Gateway - const encryptedFileInfos = await fsPromise.stat(encryptedBackupFilePath); - logger.info( - `Gateway backup: Uploading backup, size of encrypted backup = ${Math.round( - encryptedFileInfos.size / 1024 / 1024, - )}mb.`, - ); - const initializeBackupResponse = await this.gladysGatewayClient.initializeMultiPartBackup({ - file_size: encryptedFileInfos.size, - }); try { - const totalOfChunksToUpload = initializeBackupResponse.parts.length; - - const partsUploaded = await Promise.mapSeries(initializeBackupResponse.parts, async (part, index) => { - const startPosition = index * initializeBackupResponse.chunk_size; - const chunk = await readChunk(encryptedBackupFilePath, { - length: initializeBackupResponse.chunk_size, - startPosition, + const encryptKey = await this.variable.getValue('GLADYS_GATEWAY_BACKUP_KEY'); + if (encryptKey === null) { + throw new NotFoundError('GLADYS_GATEWAY_BACKUP_KEY_NOT_FOUND'); + } + const systemInfos = await this.system.getInfos(); + const now = new Date(); + const date = `${now.getFullYear()}-${now.getMonth() + + 1}-${now.getDate()}-${now.getHours()}-${now.getMinutes()}-${now.getSeconds()}`; + const backupFileName = `${BACKUP_NAME_BASE}-${date}.db`; + const backupFilePath = path.join(this.config.backupsFolder, backupFileName); + const compressedBackupFilePath = `${backupFilePath}.gz`; + const encryptedBackupFilePath = `${compressedBackupFilePath}.enc`; + // we ensure the backup folder exists + await fse.ensureDir(this.config.backupsFolder); + // we lock the database + logger.info(`Gateway backup: Locking Database`); + // It's possible to get "Cannot start a transaction within a transaction" errors + // So we might want to retry this part a few times + await retry(async (bail, attempt) => { + await db.sequelize.transaction({ type: Sequelize.Transaction.TYPES.IMMEDIATE }, async () => { + logger.info(`Backup attempt n°${attempt} : Cleaning backup folder`); + // we delete old backups + await fse.emptyDir(this.config.backupsFolder); + // We backup database + logger.info(`Starting Gateway backup in folder ${backupFilePath}`); + await exec(`sqlite3 ${this.config.storage} ".backup '${backupFilePath}'"`); + logger.info(`Gateway backup: Unlocking Database`); }); + }, SQLITE_BACKUP_RETRY_OPTIONS); + await this.job.updateProgress(jobId, 10); + const fileInfos = await fsPromise.stat(backupFilePath); + const fileSizeMB = Math.round(fileInfos.size / 1024 / 1024); + logger.info(`Gateway backup : Success! File size is ${fileSizeMB}mb.`); + // compress backup + logger.info(`Gateway backup: Gzipping backup`); + await exec(`gzip ${backupFilePath}`); + await this.job.updateProgress(jobId, 20); + // encrypt backup + logger.info(`Gateway backup: Encrypting backup`); + await exec( + `openssl enc -aes-256-cbc -pass pass:${encryptKey} -in ${compressedBackupFilePath} -out ${encryptedBackupFilePath}`, + ); + await this.job.updateProgress(jobId, 30); + // Upload file to the Gladys Gateway + const encryptedFileInfos = await fsPromise.stat(encryptedBackupFilePath); + logger.info( + `Gateway backup: Uploading backup, size of encrypted backup = ${Math.round( + encryptedFileInfos.size / 1024 / 1024, + )}mb.`, + ); + const initializeBackupResponse = await this.gladysGatewayClient.initializeMultiPartBackup({ + file_size: encryptedFileInfos.size, + }); + try { + const totalOfChunksToUpload = initializeBackupResponse.parts.length; - // each chunk is retried - const partUploaded = await retry(async () => { - const { headers } = await this.gladysGatewayClient.uploadOneBackupChunk( - part.signed_url, - chunk, - systemInfos.gladys_version, - ); - return { - PartNumber: part.part_number, - ETag: headers.etag.replace(/"/g, ''), - }; - }, UPLOAD_ONE_CHUNK_RETRY_OPTIONS); + const partsUploaded = await Promise.mapSeries(initializeBackupResponse.parts, async (part, index) => { + const startPosition = index * initializeBackupResponse.chunk_size; + const chunk = await readChunk(encryptedBackupFilePath, { + length: initializeBackupResponse.chunk_size, + startPosition, + }); - const percent = Math.round(30 + (((index + 1) * 100) / totalOfChunksToUpload) * 0.7); - await this.job.updateProgress(jobId, percent); + // each chunk is retried + const partUploaded = await retry(async () => { + const { headers } = await this.gladysGatewayClient.uploadOneBackupChunk( + part.signed_url, + chunk, + systemInfos.gladys_version, + ); + return { + PartNumber: part.part_number, + ETag: headers.etag.replace(/"/g, ''), + }; + }, UPLOAD_ONE_CHUNK_RETRY_OPTIONS); - return partUploaded; - }); - await this.gladysGatewayClient.finalizeMultiPartBackup({ - file_key: initializeBackupResponse.file_key, - file_id: initializeBackupResponse.file_id, - parts: partsUploaded, - backup_id: initializeBackupResponse.backup_id, - }); - await this.job.updateProgress(jobId, 100); - // done! - logger.info(`Gladys backup uploaded with success to Gladys Gateway.`); + const percent = Math.round(30 + (((index + 1) * 100) / totalOfChunksToUpload) * 0.7); + await this.job.updateProgress(jobId, percent); + + return partUploaded; + }); + await this.gladysGatewayClient.finalizeMultiPartBackup({ + file_key: initializeBackupResponse.file_key, + file_id: initializeBackupResponse.file_id, + parts: partsUploaded, + backup_id: initializeBackupResponse.backup_id, + }); + await this.job.updateProgress(jobId, 100); + // done! + logger.info(`Gladys backup uploaded with success to Gladys Gateway.`); + } catch (e) { + await this.gladysGatewayClient.abortMultiPartBackup({ + file_key: initializeBackupResponse.file_key, + file_id: initializeBackupResponse.file_id, + backup_id: initializeBackupResponse.backup_id, + }); + throw e; + } + return { + encryptedBackupFilePath, + }; } catch (e) { - await this.gladysGatewayClient.abortMultiPartBackup({ - file_key: initializeBackupResponse.file_key, - file_id: initializeBackupResponse.file_id, - backup_id: initializeBackupResponse.backup_id, + // If the backup fails, we need to warn the admins of this installation + const admins = await this.user.getByRole(USER_ROLE.ADMIN); + admins.forEach((admin) => { + const message = this.brain.getReply(admin.language, 'backup.fail', { + errorMessage: e.toString(), + }); + this.message.sendToUser(admin.selector, message); }); throw e; } - return { - encryptedBackupFilePath, - }; } module.exports = { diff --git a/server/test/lib/gateway/gateway.backup.test.js b/server/test/lib/gateway/gateway.backup.test.js index 8c916f1c3c..a3f2aeb767 100644 --- a/server/test/lib/gateway/gateway.backup.test.js +++ b/server/test/lib/gateway/gateway.backup.test.js @@ -17,6 +17,9 @@ describe('gateway.backup', async function describe() { const variable = {}; const event = {}; + const user = {}; + const brain = {}; + const message = {}; let gateway; @@ -63,7 +66,15 @@ describe('gateway.backup', async function describe() { shutdown: fake.resolves(true), }; - gateway = new Gateway(variable, event, system, sequelize, config, {}, {}, {}, job, scheduler); + user.getByRole = fake.resolves([ + { language: 'fr', selector: 'toto-fr' }, + { language: 'en', selector: 'toto-en' }, + ]); + + message.sendToUser = fake.returns(null); + brain.getReply = fake.returns('Backup failed!'); + + gateway = new Gateway(variable, event, system, sequelize, config, user, {}, {}, job, scheduler, message, brain); }); afterEach(() => { @@ -92,6 +103,15 @@ describe('gateway.backup', async function describe() { assert.calledOnce(gateway.gladysGatewayClient.initializeMultiPartBackup); assert.calledOnce(gateway.gladysGatewayClient.abortMultiPartBackup); + assert.calledOnce(user.getByRole); + assert.calledWith(brain.getReply, 'fr', 'backup.fail', { + errorMessage: 'Error: error', + }); + assert.calledWith(brain.getReply, 'en', 'backup.fail', { + errorMessage: 'Error: error', + }); + assert.calledWith(message.sendToUser, 'toto-fr', 'Backup failed!'); + assert.calledWith(message.sendToUser, 'toto-en', 'Backup failed!'); }); it('should backup gladys with lots of insert at the same time', async () => {