From ee9212a27a7cb827d64824c57207c6b350e58a55 Mon Sep 17 00:00:00 2001 From: Florent Beauchamp Date: Fri, 20 Sep 2024 09:05:23 +0000 Subject: [PATCH] feat(backups): implement long terme retention long term retention sometimes called GFS ( Grand Father / Father / Son) is a way to promote some backup to be kept on a long time that way , the user can use the find the best equilibrium between storage and security This commit add the code mechanics to indentify bakcup that can be deleted safely. It is intended to use with a form that ask the suer for the number of day, week, month, and year for which XO will keep the most recent It extends the actual system of keeping the n most recent backup Keep in mind that the backup oromoted by week and month can be decaled --- @xen-orchestra/backups/_getOldEntries.mjs | 86 +++++++++++- .../backups/_getOldEntries.test.mjs | 132 ++++++++++++++++++ 2 files changed, 216 insertions(+), 2 deletions(-) create mode 100644 @xen-orchestra/backups/_getOldEntries.test.mjs diff --git a/@xen-orchestra/backups/_getOldEntries.mjs b/@xen-orchestra/backups/_getOldEntries.mjs index 43656d225f1..b4f4907642f 100644 --- a/@xen-orchestra/backups/_getOldEntries.mjs +++ b/@xen-orchestra/backups/_getOldEntries.mjs @@ -1,4 +1,86 @@ +const LTR_DEFINITIONS = { + daily: { + makeDateFormatter: ({ firstHourOfTheDay = 0 } = {}) => { + return date => { + const copy = new Date(date) + copy.setHours(copy.getHours() - firstHourOfTheDay) + return `${copy.getFullYear()}-${copy.getMonth()}-${copy.getDate()}` + } + }, + }, + weekly: { + makeDateFormatter: ({ firstDayOfWeek = 1 /* sunday is 0 , let's use monday as default instead */ } = {}) => { + return date => { + const copy = new Date(date) + copy.setDate(date.getDate() - ((date.getDay() + 7 - firstDayOfWeek) % 7)) + return `${copy.getFullYear()}-${copy.getMonth()}-${copy.getDate()}` + } + }, + ancestor: 'daily', + }, + monthly: { + makeDateFormatter: ({ firstDayOfMonth = 0 } = {}) => { + return date => { + const copy = new Date(date) + copy.setDate(copy.getDate() - firstDayOfMonth) + return `${copy.getFullYear()}-${copy.getMonth()}` + } + }, + ancestor: 'weekly', + }, + yearly: { + makeDateFormatter: () => { + return date => `${date.getFullYear()}` + }, + ancestor: 'monthly', + }, +} + // returns all entries but the last retention-th -export function getOldEntries(retention, entries) { - return entries === undefined ? [] : retention > 0 ? entries.slice(0, -retention) : entries +/** + * return the entries too old to be kept + * if multiple entries are i the same time bucket : keep only the most recent one + * if an entry is valid in any of the bucket OR the minRetentionCount : keep it + * if a bucket is cmpletly empty : it does not count as one, thus it may extend the retention + * @returns Array + */ +export function getOldEntries(minRetentionCount, entries, { longTermRetention = {} } = {}) { + const dateBuckets = {} + for (const [duration, { retention, settings }] of Object.entries(longTermRetention)) { + if (LTR_DEFINITIONS[duration] === undefined) { + throw new Error(`Retention of type ${retention} is invalid`) + } + dateBuckets[duration] = { + remaining: retention, + lastMatchingBucket: null, + formatter: LTR_DEFINITIONS[duration].makeDateFormatter(settings), + } + } + const nb = entries.length + const oldEntries = [] + + for (let i = nb - 1; i >= 0; i--) { + const entry = entries[i] + const entryDate = new Date(entry.timestamp) + let shouldBeKept = false + for (const [duration, { remaining, lastMatchingBucket, formatter }] of Object.entries(dateBuckets)) { + if (remaining === 0) { + continue + } + const bucket = formatter(entryDate) + if (lastMatchingBucket !== bucket) { + shouldBeKept = true + dateBuckets[duration].remaining -= 1 + dateBuckets[duration].lastMatchingBucket = bucket + } + } + if (i >= nb - minRetentionCount) { + shouldBeKept = true + } + if (!shouldBeKept) { + oldEntries.push(entry) + } + } + // we expect the entries to be in the right order + return oldEntries.reverse() } diff --git a/@xen-orchestra/backups/_getOldEntries.test.mjs b/@xen-orchestra/backups/_getOldEntries.test.mjs new file mode 100644 index 00000000000..3840bb419ce --- /dev/null +++ b/@xen-orchestra/backups/_getOldEntries.test.mjs @@ -0,0 +1,132 @@ +import { describe, it } from 'node:test' +import assert from 'node:assert/strict' + +import { getOldEntries } from './_getOldEntries.mjs' + +describe('_getOldEntries() should succeed', () => { + const tests = [ + { + args: [ + 1, + [ + { timestamp: 1, id: 1 }, + { timestamp: 3, id: 2 }, + { timestamp: 2, id: 3 }, + ], + ], + expectedIds: [1, 2], + testLabel: 'should handle number based retention ', + }, + + { + args: [ + 0, + [ + { timestamp: +new Date('2024-09-01 00:01:00'), id: 1 }, // too old + { timestamp: +new Date('2024-09-01 00:00:00'), id: 2 }, // too old + { timestamp: +new Date('2024-09-02 00:09:00'), id: 3 }, // too old in same day + { timestamp: +new Date('2024-09-02 00:10:00'), id: 4 }, + { timestamp: +new Date('2024-09-03 00:09:00'), id: 5 }, + { timestamp: +new Date('2024-09-04 00:09:00'), id: 6 }, // too old in same day + { timestamp: +new Date('2024-09-04 00:10:00'), id: 7 }, + ], + { + longTermRetention: { + daily: { retention: 3 }, + }, + }, + ], + expectedIds: [1, 2, 3, 6], + testLabel: 'should handle day based retention ', + }, + { + args: [ + 0, + [ + { timestamp: +new Date('2024-09-01 00:01:00'), id: 1 }, // week n-3 too old + { timestamp: +new Date('2024-09-02 00:00:00'), id: 2 }, // week n-3 too old + { timestamp: +new Date('2024-09-03 00:09:00'), id: 3 }, // week n-2 + { timestamp: +new Date('2024-09-04 00:09:00'), id: 4 }, // week n-2 + { timestamp: +new Date('2024-09-05 00:09:00'), id: 5 }, // week n-2 + { timestamp: +new Date('2024-09-06 00:09:00'), id: 6 }, // week n-2 + { timestamp: +new Date('2024-09-07 00:09:00'), id: 7 }, // week n-2 + { timestamp: +new Date('2024-09-09 00:09:00'), id: 8 }, // week n-2 , most recent kept + { timestamp: +new Date('2024-09-15 00:09:00'), id: 9 }, // week n-1 kept + { timestamp: +new Date('2024-09-22 00:09:00'), id: 10 }, // week n kept + ], + { + longTermRetention: { + weekly: { retention: 3 }, + }, + }, + ], + expectedIds: [1, 2, 3, 4, 5, 6, 8], + testLabel: 'should handle week based retention ', + }, + { + args: [ + 0, + [ + { timestamp: +new Date('2024-06-22 00:09:00'), id: 1 }, // too old + { timestamp: +new Date('2024-07-31 00:09:00'), id: 2 }, // first of july + { timestamp: +new Date('2024-08-01 00:09:00'), id: 3 }, // older of august + { timestamp: +new Date('2024-08-05 00:09:00'), id: 4 }, // older of august + { timestamp: +new Date('2024-08-07 00:09:00'), id: 5 }, // most recent of august + { timestamp: +new Date('2024-09-09 00:09:00'), id: 6 }, // older of september + { timestamp: +new Date('2024-09-15 00:09:00'), id: 7 }, // older of september + { timestamp: +new Date('2024-09-22 00:09:00'), id: 8 }, // most recent of september + ], + { + longTermRetention: { + weekly: { retention: 3 }, + }, + }, + ], + expectedIds: [1, 2, 3, 4, 6], + testLabel: 'should handle month based retention', + }, + { + args: [ + 0, + [ + { timestamp: +new Date('2023-05-18 00:09:00'), id: 1 }, // too old + { timestamp: +new Date('2024-06-15 00:09:00'), id: 2 }, // too old in same year + { timestamp: +new Date('2024-07-04 00:09:00'), id: 3 }, + { timestamp: +new Date('2024-08-12 00:09:00'), id: 4 }, + { timestamp: +new Date('2024-09-05 00:09:00'), id: 5 }, + { timestamp: +new Date('2024-10-02 00:09:00'), id: 6 }, + { timestamp: +new Date('2024-11-01 00:09:00'), id: 7 }, + { timestamp: +new Date('2024-12-17 00:09:00'), id: 8 }, + { timestamp: +new Date('2024-12-24 00:09:00'), id: 10 }, + { timestamp: +new Date('2025-12-31 00:09:00'), id: 11 }, // same day/week/month/year + { timestamp: +new Date('2025-12-31 00:09:00'), id: 12 }, // new month /year + { timestamp: +new Date('2025-01-01 00:09:00'), id: 13 }, // same day/week/month/year + { timestamp: +new Date('2025-01-01 00:10:00'), id: 14 }, // new year / + ], + { + longTermRetention: { + daily: { retention: 2 }, + weekly: { retention: 4 }, + monthly: { retention: 8 }, + yearly: { retention: 2 }, + }, + }, + ], + expectedIds: [1, 2, 11, 13], + testLabel: 'complete test ', + }, + ] + + for (const { args, expectedIds, testLabel } of tests) { + it(testLabel, () => { + const oldEntries = getOldEntries.apply(null, args) + assert.strictEqual(oldEntries.length, expectedIds.length, 'different length') + for (let i = 0; i < expectedIds.length; i++) { + assert.strictEqual(oldEntries[i].id, expectedIds[i]) + } + }) + } +}) + +describe('_getOldEntries() should fail when called incorrectly', () => {}) +describe('_getOldEntries() should handle picking specific backup to promote', () => {})