+
-
diff --git a/server/.eslintrc.json b/server/.eslintrc.json
index 7caff0e7ec..332e2989a4 100644
--- a/server/.eslintrc.json
+++ b/server/.eslintrc.json
@@ -2,7 +2,7 @@
"plugins": ["jsdoc", "require-jsdoc", "no-call", "mocha", "promise"],
"extends": ["airbnb-base", "plugin:jsdoc/recommended", "prettier"],
"parserOptions": {
- "ecmaVersion": 2018
+ "ecmaVersion": 2023
},
"env": {
"node": true
diff --git a/server/api/controllers/device.controller.js b/server/api/controllers/device.controller.js
index 15687d6988..b04c096e11 100644
--- a/server/api/controllers/device.controller.js
+++ b/server/api/controllers/device.controller.js
@@ -1,5 +1,5 @@
const asyncMiddleware = require('../middlewares/asyncMiddleware');
-const { EVENTS, ACTIONS, ACTIONS_STATUS } = require('../../utils/constants');
+const { EVENTS, ACTIONS, ACTIONS_STATUS, SYSTEM_VARIABLE_NAMES } = require('../../utils/constants');
module.exports = function DeviceController(gladys) {
/**
@@ -106,6 +106,37 @@ module.exports = function DeviceController(gladys) {
res.json(states);
}
+ /**
+ * @api {post} /api/v1/device/purge_all_sqlite_state purgeAllSqliteStates
+ * @apiName purgeAllSqliteStates
+ * @apiGroup Device
+ */
+ async function purgeAllSqliteStates(req, res) {
+ gladys.event.emit(EVENTS.DEVICE.PURGE_ALL_SQLITE_STATES);
+ res.json({ success: true });
+ }
+
+ /**
+ * @api {post} /api/v1/device/migrate_from_sqlite_to_duckdb migrateFromSQLiteToDuckDb
+ * @apiName migrateFromSQLiteToDuckDb
+ * @apiGroup Device
+ */
+ async function migrateFromSQLiteToDuckDb(req, res) {
+ await gladys.variable.destroy(SYSTEM_VARIABLE_NAMES.DUCKDB_MIGRATED);
+ gladys.event.emit(EVENTS.DEVICE.MIGRATE_FROM_SQLITE_TO_DUCKDB);
+ res.json({ success: true });
+ }
+
+ /**
+ * @api {get} /api/v1/device/duckdb_migration_state getDuckDbMigrationState
+ * @apiName getDuckDbMigrationState
+ * @apiGroup Device
+ */
+ async function getDuckDbMigrationState(req, res) {
+ const migrationState = await gladys.device.getDuckDbMigrationState();
+ res.json(migrationState);
+ }
+
return Object.freeze({
create: asyncMiddleware(create),
get: asyncMiddleware(get),
@@ -115,5 +146,8 @@ module.exports = function DeviceController(gladys) {
setValue: asyncMiddleware(setValue),
setValueFeature: asyncMiddleware(setValueFeature),
getDeviceFeaturesAggregated: asyncMiddleware(getDeviceFeaturesAggregated),
+ purgeAllSqliteStates: asyncMiddleware(purgeAllSqliteStates),
+ getDuckDbMigrationState: asyncMiddleware(getDuckDbMigrationState),
+ migrateFromSQLiteToDuckDb: asyncMiddleware(migrateFromSQLiteToDuckDb),
});
};
diff --git a/server/api/routes.js b/server/api/routes.js
index f7dc82f14a..6a8561054f 100644
--- a/server/api/routes.js
+++ b/server/api/routes.js
@@ -199,6 +199,18 @@ function getRoutes(gladys) {
authenticated: true,
controller: deviceController.get,
},
+ 'get /api/v1/device/duckdb_migration_state': {
+ authenticated: true,
+ controller: deviceController.getDuckDbMigrationState,
+ },
+ 'post /api/v1/device/purge_all_sqlite_state': {
+ authenticated: true,
+ controller: deviceController.purgeAllSqliteStates,
+ },
+ 'post /api/v1/device/migrate_from_sqlite_to_duckdb': {
+ authenticated: true,
+ controller: deviceController.migrateFromSQLiteToDuckDb,
+ },
'get /api/v1/service/:service_name/device': {
authenticated: true,
controller: deviceController.getDevicesByService,
diff --git a/server/config/scheduler-jobs.js b/server/config/scheduler-jobs.js
index a5727d78df..0823d4683e 100644
--- a/server/config/scheduler-jobs.js
+++ b/server/config/scheduler-jobs.js
@@ -11,11 +11,6 @@ const jobs = [
rule: '0 0 4 * * *', // At 4 AM every day
event: EVENTS.DEVICE.PURGE_STATES,
},
- {
- name: 'hourly-device-state-aggregate',
- rule: '0 0 * * * *', // every hour
- event: EVENTS.DEVICE.CALCULATE_HOURLY_AGGREGATE,
- },
{
name: 'daily-purge-of-old-jobs',
rule: '0 0 22 * * *', // every day at 22:00
diff --git a/server/index.js b/server/index.js
index b9c3b526d1..89dc38ed23 100644
--- a/server/index.js
+++ b/server/index.js
@@ -21,6 +21,26 @@ process.on('uncaughtException', (error, promise) => {
logger.error(error);
});
+const closeSQLite = async () => {
+ try {
+ await db.sequelize.close();
+ logger.info('SQLite closed.');
+ } catch (e) {
+ logger.info('SQLite database is probably already closed');
+ logger.warn(e);
+ }
+};
+
+const closeDuckDB = async () => {
+ try {
+ await db.duckDb.close();
+ logger.info('DuckDB closed.');
+ } catch (e) {
+ logger.info('DuckDB database is probably already closed');
+ logger.warn(e);
+ }
+};
+
const shutdown = async (signal) => {
logger.info(`${signal} received.`);
// We give Gladys 10 seconds to properly shutdown, otherwise we do it
@@ -28,13 +48,8 @@ const shutdown = async (signal) => {
logger.info('Timeout to shutdown expired, forcing shut down.');
process.exit();
}, 10 * 1000);
- logger.info('Closing database connection.');
- try {
- await db.sequelize.close();
- } catch (e) {
- logger.info('Database is probably already closed');
- logger.warn(e);
- }
+ logger.info('Closing database connections.');
+ await Promise.all([closeSQLite(), closeDuckDB()]);
process.exit();
};
@@ -45,7 +60,6 @@ process.on('SIGINT', () => shutdown('SIGINT'));
// create Gladys object
const gladys = Gladys({
jwtSecret: process.env.JWT_SECRET,
- disableDeviceStateAggregation: true,
});
// start Gladys
diff --git a/server/jsconfig.json b/server/jsconfig.json
index 517e6ff384..c7cae7bd66 100644
--- a/server/jsconfig.json
+++ b/server/jsconfig.json
@@ -1,8 +1,10 @@
{
"compilerOptions": {
"checkJs": true,
- "resolveJsonModule": true,
- "lib": ["es5", "es2018"]
+ "moduleResolution": "NodeNext",
+ "lib": ["ES2022"],
+ "target": "ES2022"
},
+
"exclude": ["node_modules", "**/node_modules/*"]
}
diff --git a/server/lib/device/device.calculateAggregate.js b/server/lib/device/device.calculateAggregate.js
deleted file mode 100644
index 6f036c435f..0000000000
--- a/server/lib/device/device.calculateAggregate.js
+++ /dev/null
@@ -1,118 +0,0 @@
-const Promise = require('bluebird');
-const path = require('path');
-const { spawn } = require('child_process');
-const logger = require('../../utils/logger');
-
-const {
- DEVICE_FEATURE_STATE_AGGREGATE_TYPES,
- SYSTEM_VARIABLE_NAMES,
- DEFAULT_AGGREGATES_POLICY_IN_DAYS,
-} = require('../../utils/constants');
-
-const LAST_AGGREGATE_ATTRIBUTES = {
- [DEVICE_FEATURE_STATE_AGGREGATE_TYPES.HOURLY]: 'last_hourly_aggregate',
- [DEVICE_FEATURE_STATE_AGGREGATE_TYPES.DAILY]: 'last_daily_aggregate',
- [DEVICE_FEATURE_STATE_AGGREGATE_TYPES.MONTHLY]: 'last_monthly_aggregate',
-};
-
-const AGGREGATES_POLICY_RETENTION_VARIABLES = {
- [DEVICE_FEATURE_STATE_AGGREGATE_TYPES.HOURLY]: SYSTEM_VARIABLE_NAMES.DEVICE_STATE_HOURLY_AGGREGATES_RETENTION_IN_DAYS,
- [DEVICE_FEATURE_STATE_AGGREGATE_TYPES.DAILY]: SYSTEM_VARIABLE_NAMES.DEVICE_STATE_DAILY_AGGREGATES_RETENTION_IN_DAYS,
- [DEVICE_FEATURE_STATE_AGGREGATE_TYPES.MONTHLY]:
- SYSTEM_VARIABLE_NAMES.DEVICE_STATE_MONTHLY_AGGREGATES_RETENTION_IN_DAYS,
-};
-
-const AGGREGATE_STATES_PER_INTERVAL = 100;
-
-const SCRIPT_PATH = path.join(__dirname, 'device.calculcateAggregateChildProcess.js');
-
-/**
- * @description Calculate Aggregates.
- * @param {string} [type] - Type of the aggregate.
- * @param {string} [jobId] - Id of the job in db.
- * @returns {Promise} - Resolve when finished.
- * @example
- * await calculateAggregate('monthly');
- */
-async function calculateAggregate(type, jobId) {
- logger.info(`Calculating aggregates device feature state for interval ${type}`);
- // First we get the retention policy of this aggregates type
- let retentionPolicyInDays = await this.variable.getValue(AGGREGATES_POLICY_RETENTION_VARIABLES[type]);
-
- // if the setting exist, we parse it
- // otherwise, we take the default value
- if (retentionPolicyInDays) {
- retentionPolicyInDays = parseInt(retentionPolicyInDays, 10);
- } else {
- retentionPolicyInDays = DEFAULT_AGGREGATES_POLICY_IN_DAYS[type];
- }
-
- logger.debug(`Aggregates device feature state policy = ${retentionPolicyInDays} days`);
-
- const now = new Date();
- // the aggregate should be from this date at most
- const minStartFrom = new Date(new Date().setDate(now.getDate() - retentionPolicyInDays));
-
- let endAt;
-
- if (type === DEVICE_FEATURE_STATE_AGGREGATE_TYPES.MONTHLY) {
- endAt = new Date(now.getFullYear(), now.getMonth(), 1);
- } else if (type === DEVICE_FEATURE_STATE_AGGREGATE_TYPES.DAILY) {
- endAt = new Date(now.getFullYear(), now.getMonth(), now.getDate(), 0);
- } else if (type === DEVICE_FEATURE_STATE_AGGREGATE_TYPES.HOURLY) {
- endAt = new Date(now.getFullYear(), now.getMonth(), now.getDate(), now.getHours(), 0);
- }
-
- const params = {
- AGGREGATE_STATES_PER_INTERVAL,
- DEVICE_FEATURE_STATE_AGGREGATE_TYPES,
- LAST_AGGREGATE_ATTRIBUTES,
- type,
- minStartFrom,
- endAt,
- jobId,
- };
-
- const promise = new Promise((resolve, reject) => {
- let err = '';
- const childProcess = spawn('node', [SCRIPT_PATH, JSON.stringify(params)]);
-
- childProcess.stdout.on('data', async (data) => {
- const text = data.toString();
- logger.debug(`device.calculateAggregate stdout: ${data}`);
- if (text && text.indexOf('updateProgress:') !== -1) {
- const position = text.indexOf('updateProgress:');
- const before = text.substr(position + 15);
- const splitted = before.split(':');
- const progress = parseInt(splitted[0], 10);
- if (!Number.isNaN(progress)) {
- await this.job.updateProgress(jobId, progress);
- }
- }
- });
-
- childProcess.stderr.on('data', (data) => {
- logger.warn(`device.calculateAggregate stderr: ${data}`);
- err += data;
- });
-
- childProcess.on('close', (code) => {
- if (code !== 0) {
- logger.warn(`device.calculateAggregate: Exiting child process with code ${code}`);
- const error = new Error(err);
- reject(error);
- } else {
- logger.info(`device.calculateAggregate: Finishing processing for interval ${type} `);
- resolve();
- }
- });
- });
-
- await promise;
-
- return null;
-}
-
-module.exports = {
- calculateAggregate,
-};
diff --git a/server/lib/device/device.calculcateAggregateChildProcess.js b/server/lib/device/device.calculcateAggregateChildProcess.js
deleted file mode 100644
index b5828a5c33..0000000000
--- a/server/lib/device/device.calculcateAggregateChildProcess.js
+++ /dev/null
@@ -1,183 +0,0 @@
-// we allow console.log here because as it's a child process, we'll use
-// logger on the parent instance, not here in this child process
-/* eslint-disable no-console */
-const Promise = require('bluebird');
-const { LTTB } = require('downsample');
-const { Op } = require('sequelize');
-const uuid = require('uuid');
-const db = require('../../models');
-const { chunk } = require('../../utils/chunks');
-
-/**
- * @description This function calculate aggregate device values from a child process.
- * @param {object} params - Parameters.
- * @returns {Promise} - Resolve when finished.
- * @example
- * await calculateAggregateChildProcess({});
- */
-async function calculateAggregateChildProcess(params) {
- const {
- AGGREGATE_STATES_PER_INTERVAL,
- DEVICE_FEATURE_STATE_AGGREGATE_TYPES,
- LAST_AGGREGATE_ATTRIBUTES,
- type,
- jobId,
- } = params;
-
- const minStartFrom = new Date(params.minStartFrom);
- const endAt = new Date(params.endAt);
-
- // first we get all device features
- const deviceFeatures = await db.DeviceFeature.findAll({
- raw: true,
- });
-
- let previousProgress;
-
- // foreach device feature
- // we use Promise.each to do it one by one to avoid overloading Gladys
- await Promise.each(deviceFeatures, async (deviceFeature, index) => {
- console.log(`Calculate aggregates values for device feature ${deviceFeature.selector}.`);
-
- const lastAggregate = deviceFeature[LAST_AGGREGATE_ATTRIBUTES[type]];
- const lastAggregateDate = lastAggregate ? new Date(lastAggregate) : null;
- let startFrom;
- // if there was an aggregate and it's not older than
- // what the retention policy allow
- if (lastAggregateDate && lastAggregateDate < minStartFrom) {
- console.log(`Choosing minStartFrom, ${lastAggregateDate}, ${minStartFrom}`);
- startFrom = minStartFrom;
- } else if (lastAggregateDate && lastAggregateDate >= minStartFrom) {
- console.log(`Choosing lastAggregate, ${lastAggregateDate}, ${minStartFrom}`);
- startFrom = lastAggregateDate;
- } else {
- console.log(`Choosing Default, ${lastAggregateDate}, ${minStartFrom}`);
- startFrom = minStartFrom;
- }
-
- // we get all the data from the last aggregate to beginning of current interval
- const queryParams = {
- raw: true,
- where: {
- device_feature_id: deviceFeature.id,
- created_at: {
- [Op.between]: [startFrom, endAt],
- },
- },
- attributes: ['value', 'created_at'],
- order: [['created_at', 'ASC']],
- };
-
- console.log(`Aggregate: Getting data from ${startFrom} to ${endAt}.`);
-
- const deviceFeatureStates = await db.DeviceFeatureState.findAll(queryParams);
-
- console.log(`Aggregate: Received ${deviceFeatureStates.length} device feature states.`);
-
- const deviceFeatureStatePerInterval = new Map();
-
- // Group each deviceFeature state by interval (same month, same day, same hour)
- deviceFeatureStates.forEach((deviceFeatureState) => {
- let options;
- if (type === DEVICE_FEATURE_STATE_AGGREGATE_TYPES.MONTHLY) {
- options = {
- year: 'numeric',
- month: '2-digit',
- };
- } else if (type === DEVICE_FEATURE_STATE_AGGREGATE_TYPES.DAILY) {
- options = {
- year: 'numeric',
- month: '2-digit',
- day: '2-digit',
- };
- } else if (type === DEVICE_FEATURE_STATE_AGGREGATE_TYPES.HOURLY) {
- options = {
- year: 'numeric',
- month: '2-digit',
- day: '2-digit',
- hour: '2-digit',
- };
- }
- // @ts-ignore
- const key = new Date(deviceFeatureState.created_at).toLocaleDateString('en-US', options);
- if (!deviceFeatureStatePerInterval.has(key)) {
- deviceFeatureStatePerInterval.set(key, []);
- }
- deviceFeatureStatePerInterval.get(key).push(deviceFeatureState);
- });
-
- let deviceFeatureStateAggregatesToInsert = [];
-
- deviceFeatureStatePerInterval.forEach((oneIntervalArray, key) => {
- const dataForDownsampling = oneIntervalArray.map((deviceFeatureState) => {
- return [new Date(deviceFeatureState.created_at), deviceFeatureState.value];
- });
-
- // console.log(`Aggregate: On this interval (${key}), ${oneIntervalArray.length} events found.`);
-
- // we downsample the data
- const downsampled = LTTB(dataForDownsampling, AGGREGATE_STATES_PER_INTERVAL);
-
- // then we format the data to insert it in the DB
- deviceFeatureStateAggregatesToInsert = deviceFeatureStateAggregatesToInsert.concat(
- // @ts-ignore
- downsampled.map((d) => {
- return {
- id: uuid.v4(),
- type,
- device_feature_id: deviceFeature.id,
- value: d[1],
- created_at: d[0],
- updated_at: d[0],
- };
- }),
- );
- });
-
- console.log(`Aggregates: Inserting ${deviceFeatureStateAggregatesToInsert.length} events in database`);
-
- // we bulk insert the data
- if (deviceFeatureStateAggregatesToInsert.length) {
- const queryInterface = db.sequelize.getQueryInterface();
- await queryInterface.bulkDelete('t_device_feature_state_aggregate', {
- type,
- device_feature_id: deviceFeature.id,
- created_at: { [Op.between]: [startFrom, endAt] },
- });
- const chunks = chunk(deviceFeatureStateAggregatesToInsert, 500);
- console.log(`Aggregates: Inserting the data in ${chunks.length} chunks.`);
- await Promise.each(chunks, async (deviceStatesToInsert) => {
- await queryInterface.bulkInsert('t_device_feature_state_aggregate', deviceStatesToInsert);
- });
- }
- await db.DeviceFeature.update(
- { [LAST_AGGREGATE_ATTRIBUTES[type]]: endAt },
- {
- where: {
- id: deviceFeature.id,
- },
- },
- );
- const progress = Math.ceil((index * 100) / deviceFeatures.length);
- if (previousProgress !== progress && jobId) {
- // we need to console.log to give the new progress
- // to the main process
- console.log(`updateProgress:${progress}:updateProgress`);
- previousProgress = progress;
- }
- });
-}
-
-const params = JSON.parse(process.argv[2]);
-
-(async () => {
- try {
- await db.sequelize.query('PRAGMA journal_mode=WAL;');
- await calculateAggregateChildProcess(params);
- await db.sequelize.close();
- process.exit(0);
- } catch (e) {
- console.error(e);
- process.exit(1);
- }
-})();
diff --git a/server/lib/device/device.destroy.js b/server/lib/device/device.destroy.js
index 3d2bc80a0c..32d49c5016 100644
--- a/server/lib/device/device.destroy.js
+++ b/server/lib/device/device.destroy.js
@@ -1,4 +1,5 @@
const { QueryTypes } = require('sequelize');
+const Promise = require('bluebird');
const { NotFoundError, BadParameters } = require('../../utils/coreErrors');
const { DEVICE_POLL_FREQUENCIES, EVENTS } = require('../../utils/constants');
const db = require('../../models');
@@ -68,6 +69,14 @@ async function destroy(selector) {
throw new BadParameters(`${totalNumberOfStates} states in DB. Too much states!`);
}
+ // Delete states from DuckDB
+ await Promise.each(device.features, async (feature) => {
+ await db.duckDbWriteConnectionAllAsync(
+ 'DELETE FROM t_device_feature_state WHERE device_feature_id = ?',
+ feature.id,
+ );
+ });
+
await device.destroy();
// removing from ram cache
diff --git a/server/lib/device/device.getDeviceFeaturesAggregates.js b/server/lib/device/device.getDeviceFeaturesAggregates.js
index 616ad45d0c..fc6d6e0ac7 100644
--- a/server/lib/device/device.getDeviceFeaturesAggregates.js
+++ b/server/lib/device/device.getDeviceFeaturesAggregates.js
@@ -1,13 +1,6 @@
-const { Op, fn, col, literal } = require('sequelize');
-const { LTTB } = require('downsample');
-const dayjs = require('dayjs');
-const utc = require('dayjs/plugin/utc');
-
const db = require('../../models');
const { NotFoundError } = require('../../utils/coreErrors');
-dayjs.extend(utc);
-
/**
* @description Get all features states aggregates.
* @param {string} selector - Device selector.
@@ -26,76 +19,33 @@ async function getDeviceFeaturesAggregates(selector, intervalInMinutes, maxState
const now = new Date();
const intervalDate = new Date(now.getTime() - intervalInMinutes * 60 * 1000);
- const fiveDaysAgo = new Date(now.getTime() - 5 * 24 * 60 * 60 * 1000);
- const thirthyHoursAgo = new Date(now.getTime() - 30 * 60 * 60 * 1000);
- const tenHoursAgo = new Date(now.getTime() - 10 * 60 * 60 * 1000);
- const sixMonthsAgo = new Date(now.getTime() - 6 * 30 * 24 * 60 * 60 * 1000);
-
- let type;
- let groupByFunction;
-
- if (intervalDate < sixMonthsAgo) {
- type = 'monthly';
- groupByFunction = fn('date', col('created_at'));
- } else if (intervalDate < fiveDaysAgo) {
- type = 'daily';
- groupByFunction = fn('date', col('created_at'));
- } else if (intervalDate < thirthyHoursAgo) {
- type = 'hourly';
- groupByFunction = fn('strftime', '%Y-%m-%d %H:00:00', col('created_at'));
- } else if (intervalDate < tenHoursAgo) {
- type = 'hourly';
- // this will extract date rounded to the 5 minutes
- // So if the user queries 24h, he'll get 24 * 12 = 288 items
- groupByFunction = literal(`datetime(strftime('%s', created_at) - strftime('%s', created_at) % 300, 'unixepoch')`);
- } else {
- type = 'live';
- }
-
- let rows;
-
- if (type === 'live') {
- rows = await db.DeviceFeatureState.findAll({
- raw: true,
- attributes: ['created_at', 'value'],
- where: {
- device_feature_id: deviceFeature.id,
- created_at: {
- [Op.gte]: intervalDate,
- },
- },
- });
- } else {
- rows = await db.DeviceFeatureStateAggregate.findAll({
- raw: true,
- attributes: [
- [groupByFunction, 'created_at'],
- [fn('round', fn('avg', col('value')), 2), 'value'],
- ],
- group: [groupByFunction],
- where: {
- device_feature_id: deviceFeature.id,
- type,
- created_at: {
- [Op.gte]: intervalDate,
- },
- },
- });
- }
-
- const dataForDownsampling = rows.map((deviceFeatureState) => {
- return [dayjs.utc(deviceFeatureState.created_at), deviceFeatureState.value];
- });
-
- const downsampled = LTTB(dataForDownsampling, maxStates);
- // @ts-ignore
- const values = downsampled.map((e) => {
- return {
- created_at: e[0],
- value: e[1],
- };
- });
+ const values = await db.duckDbReadConnectionAllAsync(
+ `
+ WITH intervals AS (
+ SELECT
+ created_at,
+ value,
+ NTILE(?) OVER (ORDER BY created_at) AS interval
+ FROM
+ t_device_feature_state
+ WHERE device_feature_id = ?
+ AND created_at > ?
+ )
+ SELECT
+ MIN(created_at) AS created_at,
+ AVG(value) AS value
+ FROM
+ intervals
+ GROUP BY
+ interval
+ ORDER BY
+ created_at;
+ `,
+ maxStates,
+ deviceFeature.id,
+ intervalDate,
+ );
return {
device: {
diff --git a/server/lib/device/device.getDuckDbMigrationState.js b/server/lib/device/device.getDuckDbMigrationState.js
new file mode 100644
index 0000000000..1b50c23921
--- /dev/null
+++ b/server/lib/device/device.getDuckDbMigrationState.js
@@ -0,0 +1,25 @@
+const db = require('../../models');
+const { SYSTEM_VARIABLE_NAMES } = require('../../utils/constants');
+
+/**
+ * @description GetDuckDbMigrationState.
+ * @example await getDuckDbMigrationState();
+ * @returns {Promise} Resolve with current migration state.
+ */
+async function getDuckDbMigrationState() {
+ const isDuckDbMigrated = await this.variable.getValue(SYSTEM_VARIABLE_NAMES.DUCKDB_MIGRATED);
+ const [{ count: duckDbDeviceStateCount }] = await db.duckDbReadConnectionAllAsync(`
+ SELECT COUNT(value) as count FROM t_device_feature_state;
+ `);
+ const sqliteDeviceStateCount = await db.DeviceFeatureState.count();
+
+ return {
+ is_duck_db_migrated: isDuckDbMigrated === 'true',
+ duck_db_device_count: Number(duckDbDeviceStateCount),
+ sqlite_db_device_state_count: sqliteDeviceStateCount,
+ };
+}
+
+module.exports = {
+ getDuckDbMigrationState,
+};
diff --git a/server/lib/device/device.init.js b/server/lib/device/device.init.js
index c4a113e8d4..f67b3190a3 100644
--- a/server/lib/device/device.init.js
+++ b/server/lib/device/device.init.js
@@ -1,15 +1,14 @@
const db = require('../../models');
-const { EVENTS } = require('../../utils/constants');
const logger = require('../../utils/logger');
/**
* @description Init devices in local RAM.
- * @param {boolean} startDeviceStateAggregate - Start the device aggregate task.
+ * @param {boolean} startDuckDbMigration - Should start DuckDB migration.
* @returns {Promise} Resolve with inserted devices.
* @example
* gladys.device.init();
*/
-async function init(startDeviceStateAggregate = true) {
+async function init(startDuckDbMigration = true) {
// load all devices in RAM
const devices = await db.Device.findAll({
include: [
@@ -38,12 +37,11 @@ async function init(startDeviceStateAggregate = true) {
this.brain.addNamedEntity('device', plainDevice.selector, plainDevice.name);
return plainDevice;
});
- if (startDeviceStateAggregate) {
- // calculate aggregate data for device states
- this.eventManager.emit(EVENTS.DEVICE.CALCULATE_HOURLY_AGGREGATE);
- }
// setup polling for device who need polling
this.setupPoll();
+ if (startDuckDbMigration) {
+ this.migrateFromSQLiteToDuckDb();
+ }
return plainDevices;
}
diff --git a/server/lib/device/device.migrateFromSQLiteToDuckDb.js b/server/lib/device/device.migrateFromSQLiteToDuckDb.js
new file mode 100644
index 0000000000..36969b6314
--- /dev/null
+++ b/server/lib/device/device.migrateFromSQLiteToDuckDb.js
@@ -0,0 +1,85 @@
+const Promise = require('bluebird');
+const { Op } = require('sequelize');
+const db = require('../../models');
+const logger = require('../../utils/logger');
+const { SYSTEM_VARIABLE_NAMES } = require('../../utils/constants');
+
+const migrateStateRecursive = async (deviceFeatureId, condition) => {
+ logger.info(`DuckDB : Migrating device feature = ${deviceFeatureId}, offset = ${condition.offset}`);
+ // Get all device feature state in SQLite that match the condition
+ const states = await db.DeviceFeatureState.findAll(condition);
+ logger.info(`DuckDB : Device feature = ${deviceFeatureId} has ${states.length} states to migrate.`);
+ if (states.length === 0) {
+ return null;
+ }
+ await db.duckDbBatchInsertState(deviceFeatureId, states);
+ const newCondition = {
+ ...condition,
+ offset: condition.offset + condition.limit,
+ };
+ // We see if there are other states
+ await migrateStateRecursive(deviceFeatureId, newCondition);
+ return null;
+};
+
+/**
+ * @description Migrate all states from SQLite to DuckDB.
+ * @param {string} jobId - ID of the job.
+ * @param {number} duckDbMigrationPageLimit - Number of rows to migrate in one shot.
+ * @returns {Promise} Resolve when finished.
+ * @example migrateFromSQLiteToDuckDb();
+ */
+async function migrateFromSQLiteToDuckDb(jobId, duckDbMigrationPageLimit = 40000) {
+ // Check if migration was already executed
+ const isDuckDbMigrated = await this.variable.getValue(SYSTEM_VARIABLE_NAMES.DUCKDB_MIGRATED);
+ if (isDuckDbMigrated === 'true') {
+ logger.info('DuckDB : Already migrated from SQLite. Not migrating.');
+ return null;
+ }
+ logger.info('DuckDB: Migrating data from SQLite');
+ const oldestStateInDuckDBPerDeviceFeatureId = await db.duckDbReadConnectionAllAsync(`
+ SELECT
+ MIN(created_at) as created_at,
+ device_feature_id
+ FROM t_device_feature_state
+ GROUP BY device_feature_id;
+ `);
+ logger.info(
+ `DuckDB: Found ${oldestStateInDuckDBPerDeviceFeatureId.length} already migrated device features in DuckDB.`,
+ );
+ const deviceFeatures = await db.DeviceFeature.findAll();
+ logger.info(`DuckDB: Migrating ${deviceFeatures.length} device features`);
+ await Promise.mapSeries(deviceFeatures, async (deviceFeature, deviceFeatureIndex) => {
+ const currentDeviceFeatureOldestStateInDuckDB = oldestStateInDuckDBPerDeviceFeatureId.find(
+ (s) => s.device_feature_id === deviceFeature.id,
+ );
+ const condition = {
+ raw: true,
+ attributes: ['value', 'created_at'],
+ where: {
+ device_feature_id: deviceFeature.id,
+ },
+ order: [['created_at', 'DESC']],
+ limit: duckDbMigrationPageLimit,
+ offset: 0,
+ };
+ if (currentDeviceFeatureOldestStateInDuckDB) {
+ condition.where.created_at = {
+ [Op.lt]: currentDeviceFeatureOldestStateInDuckDB.created_at,
+ };
+ }
+ await migrateStateRecursive(deviceFeature.id, condition);
+ const newProgressPercent = Math.round((deviceFeatureIndex * 100) / deviceFeatures.length);
+ await this.job.updateProgress(jobId, newProgressPercent);
+ });
+
+ logger.info(`DuckDB: Finished migrating DuckDB.`);
+
+ // Save that the migration was executed
+ await this.variable.setValue(SYSTEM_VARIABLE_NAMES.DUCKDB_MIGRATED, 'true');
+ return null;
+}
+
+module.exports = {
+ migrateFromSQLiteToDuckDb,
+};
diff --git a/server/lib/device/device.onHourlyDeviceAggregateEvent.js b/server/lib/device/device.onHourlyDeviceAggregateEvent.js
deleted file mode 100644
index 24818e182b..0000000000
--- a/server/lib/device/device.onHourlyDeviceAggregateEvent.js
+++ /dev/null
@@ -1,38 +0,0 @@
-const { JOB_TYPES } = require('../../utils/constants');
-const logger = require('../../utils/logger');
-
-/**
- * @description It's time to do the daily device state aggregate.
- * @example
- * onHourlyDeviceAggregateEvent()
- */
-async function onHourlyDeviceAggregateEvent() {
- const startHourlyAggregate = this.job.wrapper(JOB_TYPES.HOURLY_DEVICE_STATE_AGGREGATE, async (jobId) => {
- await this.calculateAggregate('hourly', jobId);
- });
- const startDailyAggregate = this.job.wrapper(JOB_TYPES.DAILY_DEVICE_STATE_AGGREGATE, async (jobId) => {
- await this.calculateAggregate('daily', jobId);
- });
- const startMonthlyAggregate = this.job.wrapper(JOB_TYPES.MONTHLY_DEVICE_STATE_AGGREGATE, async (jobId) => {
- await this.calculateAggregate('monthly', jobId);
- });
- try {
- await startHourlyAggregate();
- } catch (e) {
- logger.error(e);
- }
- try {
- await startDailyAggregate();
- } catch (e) {
- logger.error(e);
- }
- try {
- await startMonthlyAggregate();
- } catch (e) {
- logger.error(e);
- }
-}
-
-module.exports = {
- onHourlyDeviceAggregateEvent,
-};
diff --git a/server/lib/device/device.onPurgeStatesEvent.js b/server/lib/device/device.onPurgeStatesEvent.js
index 17cb296b67..87ed7bb0af 100644
--- a/server/lib/device/device.onPurgeStatesEvent.js
+++ b/server/lib/device/device.onPurgeStatesEvent.js
@@ -8,7 +8,6 @@ const { JOB_TYPES } = require('../../utils/constants');
async function onPurgeStatesEvent() {
const purgeAllStates = this.job.wrapper(JOB_TYPES.DEVICE_STATES_PURGE, async () => {
await this.purgeStates();
- await this.purgeAggregateStates();
});
await purgeAllStates();
}
diff --git a/server/lib/device/device.purgeAggregateStates.js b/server/lib/device/device.purgeAggregateStates.js
deleted file mode 100644
index e3313db178..0000000000
--- a/server/lib/device/device.purgeAggregateStates.js
+++ /dev/null
@@ -1,44 +0,0 @@
-const { Op } = require('sequelize');
-const db = require('../../models');
-const logger = require('../../utils/logger');
-const { SYSTEM_VARIABLE_NAMES } = require('../../utils/constants');
-
-/**
- * @description Purge device aggregate states.
- * @returns {Promise} Resolve when finished.
- * @example
- * device.purgeAggregateStates();
- */
-async function purgeAggregateStates() {
- logger.debug('Purging device feature aggregate states...');
- const deviceAggregateStateHistoryInDays = await this.variable.getValue(
- SYSTEM_VARIABLE_NAMES.DEVICE_AGGREGATE_STATE_HISTORY_IN_DAYS,
- );
- const deviceAggregateStateHistoryInDaysInt = parseInt(deviceAggregateStateHistoryInDays, 10);
- if (Number.isNaN(deviceAggregateStateHistoryInDaysInt)) {
- logger.debug('Not purging device feature aggregate states, deviceAggregateStateHistoryInDays is not an integer.');
- return Promise.resolve(false);
- }
- if (deviceAggregateStateHistoryInDaysInt === -1) {
- logger.debug('Not purging device feature aggregate states, deviceAggregateStateHistoryInDays = -1');
- return Promise.resolve(false);
- }
- const queryInterface = db.sequelize.getQueryInterface();
- const now = new Date().getTime();
- // all date before this timestamp will be removed
- const timstampLimit = now - deviceAggregateStateHistoryInDaysInt * 24 * 60 * 60 * 1000;
- const dateLimit = new Date(timstampLimit);
- logger.info(
- `Purging device feature states of the last ${deviceAggregateStateHistoryInDaysInt} days. States older than ${dateLimit} will be purged.`,
- );
- await queryInterface.bulkDelete('t_device_feature_state_aggregate', {
- created_at: {
- [Op.lte]: dateLimit,
- },
- });
- return true;
-}
-
-module.exports = {
- purgeAggregateStates,
-};
diff --git a/server/lib/device/device.purgeAllSqliteStates.js b/server/lib/device/device.purgeAllSqliteStates.js
new file mode 100644
index 0000000000..72b05bef14
--- /dev/null
+++ b/server/lib/device/device.purgeAllSqliteStates.js
@@ -0,0 +1,104 @@
+const { QueryTypes } = require('sequelize');
+const Promise = require('bluebird');
+const db = require('../../models');
+const logger = require('../../utils/logger');
+
+/**
+ * @description Purge all SQLite states.
+ * @param {string} jobId - Id of the job.
+ * @returns {Promise} Resolve when finished.
+ * @example
+ * device.purgeAllSqliteStates('d47b481b-a7be-4224-9850-313cdb8a4065');
+ */
+async function purgeAllSqliteStates(jobId) {
+ if (this.purgeAllSQliteStatesInProgress) {
+ logger.info(`Not purging all SQlite states, a purge is already in progress`);
+ return null;
+ }
+ logger.info(`Purging all SQlite states`);
+ this.purgeAllSQliteStatesInProgress = true;
+
+ try {
+ const numberOfDeviceFeatureStateToDelete = await db.DeviceFeatureState.count();
+ const numberOfDeviceFeatureStateAggregateToDelete = await db.DeviceFeatureStateAggregate.count();
+
+ logger.info(
+ `Purging All SQLite states: ${numberOfDeviceFeatureStateToDelete} states & ${numberOfDeviceFeatureStateAggregateToDelete} aggregates to delete.`,
+ );
+
+ const numberOfIterationsStates = Math.ceil(
+ numberOfDeviceFeatureStateToDelete / this.STATES_TO_PURGE_PER_DEVICE_FEATURE_CLEAN_BATCH,
+ );
+ const iterator = [...Array(numberOfIterationsStates)];
+
+ const numberOfIterationsStatesAggregates = Math.ceil(
+ numberOfDeviceFeatureStateAggregateToDelete / this.STATES_TO_PURGE_PER_DEVICE_FEATURE_CLEAN_BATCH,
+ );
+ const iteratorAggregates = [...Array(numberOfIterationsStatesAggregates)];
+
+ const total = numberOfIterationsStates + numberOfIterationsStatesAggregates;
+ let currentBatch = 0;
+ let currentProgressPercent = 0;
+
+ // We only save progress to DB if it changed
+ // Because saving progress is expensive (DB write + Websocket call)
+ const updateProgressIfNeeded = async () => {
+ currentBatch += 1;
+ const newProgressPercent = Math.round((currentBatch * 100) / total);
+ if (currentProgressPercent !== newProgressPercent) {
+ currentProgressPercent = newProgressPercent;
+ await this.job.updateProgress(jobId, currentProgressPercent);
+ }
+ };
+
+ await Promise.each(iterator, async () => {
+ await db.sequelize.query(
+ `
+ DELETE FROM t_device_feature_state WHERE id IN (
+ SELECT id FROM t_device_feature_state
+ LIMIT :limit
+ );
+ `,
+ {
+ replacements: {
+ limit: this.STATES_TO_PURGE_PER_DEVICE_FEATURE_CLEAN_BATCH,
+ },
+ type: QueryTypes.SELECT,
+ },
+ );
+ await updateProgressIfNeeded();
+ await Promise.delay(this.WAIT_TIME_BETWEEN_DEVICE_FEATURE_CLEAN_BATCH);
+ });
+
+ await Promise.each(iteratorAggregates, async () => {
+ await db.sequelize.query(
+ `
+ DELETE FROM t_device_feature_state_aggregate WHERE id IN (
+ SELECT id FROM t_device_feature_state_aggregate
+ LIMIT :limit
+ );
+ `,
+ {
+ replacements: {
+ limit: this.STATES_TO_PURGE_PER_DEVICE_FEATURE_CLEAN_BATCH,
+ },
+ type: QueryTypes.SELECT,
+ },
+ );
+ await updateProgressIfNeeded();
+ await Promise.delay(this.WAIT_TIME_BETWEEN_DEVICE_FEATURE_CLEAN_BATCH);
+ });
+ this.purgeAllSQliteStatesInProgress = false;
+ return {
+ numberOfDeviceFeatureStateToDelete,
+ numberOfDeviceFeatureStateAggregateToDelete,
+ };
+ } catch (e) {
+ this.purgeAllSQliteStatesInProgress = false;
+ throw e;
+ }
+}
+
+module.exports = {
+ purgeAllSqliteStates,
+};
diff --git a/server/lib/device/device.purgeStates.js b/server/lib/device/device.purgeStates.js
index cd95f8e9e5..a0e8985d6f 100644
--- a/server/lib/device/device.purgeStates.js
+++ b/server/lib/device/device.purgeStates.js
@@ -1,4 +1,3 @@
-const { Op } = require('sequelize');
const db = require('../../models');
const logger = require('../../utils/logger');
const { SYSTEM_VARIABLE_NAMES } = require('../../utils/constants');
@@ -21,7 +20,6 @@ async function purgeStates() {
logger.debug('Not purging device feature states, deviceStateHistoryInDays = -1');
return Promise.resolve(false);
}
- const queryInterface = db.sequelize.getQueryInterface();
const now = new Date().getTime();
// all date before this timestamp will be removed
const timstampLimit = now - deviceStateHistoryInDaysInt * 24 * 60 * 60 * 1000;
@@ -29,11 +27,12 @@ async function purgeStates() {
logger.info(
`Purging device feature states of the last ${deviceStateHistoryInDaysInt} days. States older than ${dateLimit} will be purged.`,
);
- await queryInterface.bulkDelete('t_device_feature_state', {
- created_at: {
- [Op.lte]: dateLimit,
- },
- });
+ await db.duckDbWriteConnectionAllAsync(
+ `
+ DELETE FROM t_device_feature_state WHERE created_at < ?
+ `,
+ dateLimit,
+ );
return true;
}
diff --git a/server/lib/device/device.saveHistoricalState.js b/server/lib/device/device.saveHistoricalState.js
index 4e15c2484f..ab606a3311 100644
--- a/server/lib/device/device.saveHistoricalState.js
+++ b/server/lib/device/device.saveHistoricalState.js
@@ -2,6 +2,7 @@ const Joi = require('joi');
const { Op } = require('sequelize');
const db = require('../../models');
const logger = require('../../utils/logger');
+const { formatDateInUTC } = require('../../utils/date');
const { EVENTS, WEBSOCKET_MESSAGE_TYPES } = require('../../utils/constants');
const { BadParameters } = require('../../utils/coreErrors');
@@ -76,24 +77,24 @@ async function saveHistoricalState(deviceFeature, newValue, newValueCreatedAt) {
}
// if the deviceFeature should keep history, we save a new deviceFeatureState
if (deviceFeature.keep_history) {
- const valueAlreadyExistInDb = await db.DeviceFeatureState.findOne({
- where: {
- device_feature_id: deviceFeature.id,
- value: newValue,
- created_at: newValueCreatedAtDate,
- },
- });
+ const valueAlreadyExistInDb = await db.duckDbReadConnectionAllAsync(
+ `
+ SELECT *
+ FROM t_device_feature_state
+ WHERE device_feature_id = ?
+ AND value = ?
+ AND created_at = ?
+ `,
+ deviceFeature.id,
+ newValue,
+ formatDateInUTC(newValueCreatedAtDate),
+ );
// if the value already exist in the DB, don't re-create it
- if (valueAlreadyExistInDb !== null) {
+ if (valueAlreadyExistInDb.length > 0) {
logger.debug('device.saveHistoricalState: Not saving value in history, value already exists');
return;
}
- await db.DeviceFeatureState.create({
- device_feature_id: deviceFeature.id,
- value: newValue,
- created_at: newValueCreatedAtDate,
- updated_at: newValueCreatedAtDate,
- });
+ await db.duckDbInsertState(deviceFeature.id, newValue, newValueCreatedAtDate);
// We need to update last aggregate value
// So that aggregate is calculated again
const lastDayOfPreviousMonth = new Date(
diff --git a/server/lib/device/device.saveState.js b/server/lib/device/device.saveState.js
index 3bfa09b7b3..46e112e3cb 100644
--- a/server/lib/device/device.saveState.js
+++ b/server/lib/device/device.saveState.js
@@ -41,10 +41,7 @@ async function saveState(deviceFeature, newValue) {
);
// if the deviceFeature should keep history, we save a new deviceFeatureState
if (deviceFeature.keep_history) {
- await db.DeviceFeatureState.create({
- device_feature_id: deviceFeature.id,
- value: newValue,
- });
+ await db.duckDbInsertState(deviceFeature.id, newValue, now);
}
// send websocket event
diff --git a/server/lib/device/index.js b/server/lib/device/index.js
index 069948d22a..f69c3aca2f 100644
--- a/server/lib/device/index.js
+++ b/server/lib/device/index.js
@@ -12,17 +12,14 @@ const { add } = require('./device.add');
const { addFeature } = require('./device.addFeature');
const { addParam } = require('./device.addParam');
const { create } = require('./device.create');
-const { calculateAggregate } = require('./device.calculateAggregate');
const { destroy } = require('./device.destroy');
const { init } = require('./device.init');
const { get } = require('./device.get');
const { getBySelector } = require('./device.getBySelector');
const { getDeviceFeaturesAggregates } = require('./device.getDeviceFeaturesAggregates');
const { getDeviceFeaturesAggregatesMulti } = require('./device.getDeviceFeaturesAggregatesMulti');
-const { onHourlyDeviceAggregateEvent } = require('./device.onHourlyDeviceAggregateEvent');
const { onPurgeStatesEvent } = require('./device.onPurgeStatesEvent');
const { purgeStates } = require('./device.purgeStates');
-const { purgeAggregateStates } = require('./device.purgeAggregateStates');
const { purgeStatesByFeatureId } = require('./device.purgeStatesByFeatureId');
const { poll } = require('./device.poll');
const { pollAll } = require('./device.pollAll');
@@ -35,6 +32,9 @@ const { setupPoll } = require('./device.setupPoll');
const { newStateEvent } = require('./device.newStateEvent');
const { notify } = require('./device.notify');
const { checkBatteries } = require('./device.checkBatteries');
+const { migrateFromSQLiteToDuckDb } = require('./device.migrateFromSQLiteToDuckDb');
+const { getDuckDbMigrationState } = require('./device.getDuckDbMigrationState');
+const { purgeAllSqliteStates } = require('./device.purgeAllSqliteStates');
const DeviceManager = function DeviceManager(
eventManager,
@@ -72,39 +72,51 @@ const DeviceManager = function DeviceManager(
this.purgeStatesByFeatureId.bind(this),
);
+ this.purgeAllSqliteStates = this.job.wrapper(
+ JOB_TYPES.DEVICE_STATES_PURGE_ALL_SQLITE_STATES,
+ this.purgeAllSqliteStates.bind(this),
+ );
+
this.devicesByPollFrequency = {};
+
+ this.migrateFromSQLiteToDuckDb = this.job.wrapper(
+ JOB_TYPES.MIGRATE_SQLITE_TO_DUCKDB,
+ this.migrateFromSQLiteToDuckDb.bind(this),
+ );
+
// listen to events
this.eventManager.on(EVENTS.DEVICE.NEW_STATE, this.newStateEvent.bind(this));
this.eventManager.on(EVENTS.DEVICE.NEW, eventFunctionWrapper(this.create.bind(this)));
this.eventManager.on(EVENTS.DEVICE.ADD_FEATURE, eventFunctionWrapper(this.addFeature.bind(this)));
this.eventManager.on(EVENTS.DEVICE.ADD_PARAM, eventFunctionWrapper(this.addParam.bind(this)));
this.eventManager.on(EVENTS.DEVICE.PURGE_STATES, eventFunctionWrapper(this.onPurgeStatesEvent.bind(this)));
- this.eventManager.on(
- EVENTS.DEVICE.CALCULATE_HOURLY_AGGREGATE,
- eventFunctionWrapper(this.onHourlyDeviceAggregateEvent.bind(this)),
- );
this.eventManager.on(
EVENTS.DEVICE.PURGE_STATES_SINGLE_FEATURE,
eventFunctionWrapper(this.purgeStatesByFeatureId.bind(this)),
);
this.eventManager.on(EVENTS.DEVICE.CHECK_BATTERIES, eventFunctionWrapper(this.checkBatteries.bind(this)));
+ this.eventManager.on(
+ EVENTS.DEVICE.MIGRATE_FROM_SQLITE_TO_DUCKDB,
+ eventFunctionWrapper(this.migrateFromSQLiteToDuckDb.bind(this)),
+ );
+ this.eventManager.on(
+ EVENTS.DEVICE.PURGE_ALL_SQLITE_STATES,
+ eventFunctionWrapper(this.purgeAllSqliteStates.bind(this)),
+ );
};
DeviceManager.prototype.add = add;
DeviceManager.prototype.addFeature = addFeature;
DeviceManager.prototype.addParam = addParam;
DeviceManager.prototype.create = create;
-DeviceManager.prototype.calculateAggregate = calculateAggregate;
DeviceManager.prototype.destroy = destroy;
DeviceManager.prototype.init = init;
DeviceManager.prototype.get = get;
DeviceManager.prototype.getBySelector = getBySelector;
DeviceManager.prototype.getDeviceFeaturesAggregates = getDeviceFeaturesAggregates;
DeviceManager.prototype.getDeviceFeaturesAggregatesMulti = getDeviceFeaturesAggregatesMulti;
-DeviceManager.prototype.onHourlyDeviceAggregateEvent = onHourlyDeviceAggregateEvent;
DeviceManager.prototype.onPurgeStatesEvent = onPurgeStatesEvent;
DeviceManager.prototype.purgeStates = purgeStates;
-DeviceManager.prototype.purgeAggregateStates = purgeAggregateStates;
DeviceManager.prototype.purgeStatesByFeatureId = purgeStatesByFeatureId;
DeviceManager.prototype.poll = poll;
DeviceManager.prototype.pollAll = pollAll;
@@ -117,5 +129,8 @@ DeviceManager.prototype.setupPoll = setupPoll;
DeviceManager.prototype.setValue = setValue;
DeviceManager.prototype.notify = notify;
DeviceManager.prototype.checkBatteries = checkBatteries;
+DeviceManager.prototype.migrateFromSQLiteToDuckDb = migrateFromSQLiteToDuckDb;
+DeviceManager.prototype.getDuckDbMigrationState = getDuckDbMigrationState;
+DeviceManager.prototype.purgeAllSqliteStates = purgeAllSqliteStates;
module.exports = DeviceManager;
diff --git a/server/lib/gateway/gateway.backup.js b/server/lib/gateway/gateway.backup.js
index ae4e9fa27f..c149e88078 100644
--- a/server/lib/gateway/gateway.backup.js
+++ b/server/lib/gateway/gateway.backup.js
@@ -42,14 +42,17 @@ async function backup(jobId) {
const now = new Date();
const date = `${now.getFullYear()}-${now.getMonth() +
1}-${now.getDate()}-${now.getHours()}-${now.getMinutes()}-${now.getSeconds()}`;
- const backupFileName = `${BACKUP_NAME_BASE}-${date}.db`;
- const backupFilePath = path.join(this.config.backupsFolder, backupFileName);
- const compressedBackupFilePath = `${backupFilePath}.gz`;
+ const sqliteBackupFileName = `${BACKUP_NAME_BASE}-${date}.db`;
+ const sqliteBackupFilePath = path.join(this.config.backupsFolder, sqliteBackupFileName);
+ const duckDbBackupFolder = `${BACKUP_NAME_BASE}_${date}_parquet_folder`;
+ const duckDbBackupFolderPath = path.join(this.config.backupsFolder, duckDbBackupFolder);
+ const compressedBackupFileName = `${BACKUP_NAME_BASE}-${date}.tar.gz`;
+ const compressedBackupFilePath = path.join(this.config.backupsFolder, compressedBackupFileName);
const encryptedBackupFilePath = `${compressedBackupFilePath}.enc`;
// we ensure the backup folder exists
await fse.ensureDir(this.config.backupsFolder);
// we lock the database
- logger.info(`Gateway backup: Locking Database`);
+ logger.info(`Gateway backup: Locking SQLite Database`);
// It's possible to get "Cannot start a transaction within a transaction" errors
// So we might want to retry this part a few times
await retry(async (bail, attempt) => {
@@ -58,18 +61,28 @@ async function backup(jobId) {
// we delete old backups
await fse.emptyDir(this.config.backupsFolder);
// We backup database
- logger.info(`Starting Gateway backup in folder ${backupFilePath}`);
- await exec(`sqlite3 ${this.config.storage} ".backup '${backupFilePath}'"`);
+ logger.info(`Starting Gateway backup in folder ${sqliteBackupFilePath}`);
+ await exec(`sqlite3 ${this.config.storage} ".backup '${sqliteBackupFilePath}'"`);
logger.info(`Gateway backup: Unlocking Database`);
});
}, SQLITE_BACKUP_RETRY_OPTIONS);
await this.job.updateProgress(jobId, 10);
- const fileInfos = await fsPromise.stat(backupFilePath);
+ const fileInfos = await fsPromise.stat(sqliteBackupFilePath);
const fileSizeMB = Math.round(fileInfos.size / 1024 / 1024);
- logger.info(`Gateway backup : Success! File size is ${fileSizeMB}mb.`);
+ logger.info(`Gateway backup : SQLite file size is ${fileSizeMB}mb.`);
+ logger.info(`Gateway backup : Backing up DuckDB into a Parquet folder ${duckDbBackupFolderPath}`);
+ // DuckDB backup to parquet file
+ await db.duckDbWriteConnectionAllAsync(
+ ` EXPORT DATABASE '${duckDbBackupFolderPath}' (
+ FORMAT PARQUET,
+ COMPRESSION GZIP
+ )`,
+ );
// compress backup
- logger.info(`Gateway backup: Gzipping backup`);
- await exec(`gzip ${backupFilePath}`);
+ logger.info(`Gateway backup: Compressing backup`);
+ await exec(
+ `cd ${this.config.backupsFolder} && tar -czvf ${compressedBackupFileName} ${sqliteBackupFileName} ${duckDbBackupFolder}`,
+ );
await this.job.updateProgress(jobId, 20);
// encrypt backup
logger.info(`Gateway backup: Encrypting backup`);
@@ -82,7 +95,7 @@ async function backup(jobId) {
logger.info(
`Gateway backup: Uploading backup, size of encrypted backup = ${Math.round(
encryptedFileInfos.size / 1024 / 1024,
- )}mb.`,
+ )}mb. Path = ${encryptedBackupFilePath}`,
);
const initializeBackupResponse = await this.gladysGatewayClient.initializeMultiPartBackup({
file_size: encryptedFileInfos.size,
diff --git a/server/lib/gateway/gateway.downloadBackup.js b/server/lib/gateway/gateway.downloadBackup.js
index 91e41ae28e..99b2cb0f74 100644
--- a/server/lib/gateway/gateway.downloadBackup.js
+++ b/server/lib/gateway/gateway.downloadBackup.js
@@ -20,17 +20,21 @@ async function downloadBackup(fileUrl) {
if (encryptKey === null) {
throw new NotFoundError('GLADYS_GATEWAY_BACKUP_KEY_NOT_FOUND');
}
- // extract file name
+ // Extract file name
const fileWithoutSignedParams = fileUrl.split('?')[0];
- const encryptedBackupName = path.basename(fileWithoutSignedParams, '.enc');
const restoreFolderPath = path.join(this.config.backupsFolder, RESTORE_FOLDER);
- const encryptedBackupFilePath = path.join(restoreFolderPath, `${encryptedBackupName}.enc`);
- const compressedBackupFilePath = path.join(restoreFolderPath, `${encryptedBackupName}.db.gz`);
- const backupFilePath = path.join(restoreFolderPath, `${encryptedBackupName}.db`);
// we ensure the restore backup folder exists
await fse.ensureDir(restoreFolderPath);
// we empty the restore backup folder
await fse.emptyDir(restoreFolderPath);
+
+ const encryptedBackupName = path.basename(fileWithoutSignedParams, '.enc');
+ const encryptedBackupFilePath = path.join(restoreFolderPath, `${encryptedBackupName}.enc`);
+ const compressedBackupFilePath = path.join(restoreFolderPath, `${encryptedBackupName}.gz`);
+
+ let duckDbBackupFolderPath = null;
+ let sqliteBackupFilePath = null;
+
// we create a stream
const writeStream = fs.createWriteStream(encryptedBackupFilePath);
// and download the backup file
@@ -41,19 +45,39 @@ async function downloadBackup(fileUrl) {
await exec(
`openssl enc -aes-256-cbc -pass pass:${encryptKey} -d -in ${encryptedBackupFilePath} -out ${compressedBackupFilePath}`,
);
- // decompress backup
- await exec(`gzip -d ${compressedBackupFilePath}`);
+
+ try {
+ logger.info(`Trying to restore the backup new style (DuckDB)`);
+ await exec(`tar -xzvf ${compressedBackupFilePath} -C ${restoreFolderPath}`);
+ logger.info("Extracting worked. It's a DuckDB export.");
+ const itemsInFolder = await fse.readdir(restoreFolderPath);
+ sqliteBackupFilePath = path.join(
+ restoreFolderPath,
+ itemsInFolder.find((i) => i.endsWith('.db')),
+ );
+ duckDbBackupFolderPath = path.join(
+ restoreFolderPath,
+ itemsInFolder.find((i) => i.endsWith('_parquet_folder')),
+ );
+ } catch (e) {
+ logger.info(`Extracting failed using new strategy (Error: ${e})`);
+ logger.info(`Restoring using old backup strategy (SQLite only)`);
+ sqliteBackupFilePath = path.join(restoreFolderPath, `${encryptedBackupName}.db`);
+ await exec(`gzip -dc ${compressedBackupFilePath} > ${sqliteBackupFilePath}`);
+ }
// done!
logger.info(`Gladys backup downloaded with success.`);
// send websocket event to indicate that
this.event.emit(EVENTS.WEBSOCKET.SEND_ALL, {
type: WEBSOCKET_MESSAGE_TYPES.BACKUP.DOWNLOADED,
payload: {
- backupFilePath,
+ sqliteBackupFilePath,
+ duckDbBackupFolderPath,
},
});
return {
- backupFilePath,
+ sqliteBackupFilePath,
+ duckDbBackupFolderPath,
};
}
diff --git a/server/lib/gateway/gateway.getLatestGladysVersion.js b/server/lib/gateway/gateway.getLatestGladysVersion.js
index 411f682aeb..91e5df7747 100644
--- a/server/lib/gateway/gateway.getLatestGladysVersion.js
+++ b/server/lib/gateway/gateway.getLatestGladysVersion.js
@@ -9,14 +9,16 @@ const db = require('../../models');
async function getLatestGladysVersion() {
const systemInfos = await this.system.getInfos();
const clientId = await this.variable.getValue('GLADYS_INSTANCE_CLIENT_ID');
- const deviceStateCount = await db.DeviceFeatureState.count();
+ const [{ count: deviceStateCount }] = await db.duckDbReadConnectionAllAsync(`
+ SELECT COUNT(value) as count FROM t_device_feature_state;
+ `);
const serviceUsage = await this.serviceManager.getUsage();
const params = {
system: systemInfos.platform,
node_version: systemInfos.nodejs_version,
is_docker: systemInfos.is_docker,
client_id: clientId,
- device_state_count: deviceStateCount,
+ device_state_count: Number(deviceStateCount),
integrations: serviceUsage,
};
const latestGladysVersion = await this.gladysGatewayClient.getLatestGladysVersion(systemInfos.gladys_version, params);
diff --git a/server/lib/gateway/gateway.restoreBackup.js b/server/lib/gateway/gateway.restoreBackup.js
index fd7b25af1a..5a89e7d74b 100644
--- a/server/lib/gateway/gateway.restoreBackup.js
+++ b/server/lib/gateway/gateway.restoreBackup.js
@@ -1,26 +1,30 @@
const fse = require('fs-extra');
const sqlite3 = require('sqlite3');
+const duckdb = require('duckdb');
const { promisify } = require('util');
+
+const db = require('../../models');
const logger = require('../../utils/logger');
const { exec } = require('../../utils/childProcess');
const { NotFoundError } = require('../../utils/coreErrors');
/**
* @description Replace the local sqlite database with a backup.
- * @param {string} backupFilePath - The path of the backup.
+ * @param {string} sqliteBackupFilePath - The path of the sqlite backup.
+ * @param {string} [duckDbBackupFolderPath] - The path of the DuckDB backup folder.
* @example
* restoreBackup('/backup.db');
*/
-async function restoreBackup(backupFilePath) {
- logger.info(`Restoring back up ${backupFilePath}`);
+async function restoreBackup(sqliteBackupFilePath, duckDbBackupFolderPath) {
+ logger.info(`Restoring back up ${sqliteBackupFilePath} / ${duckDbBackupFolderPath}`);
// ensure that the file exists
- const exists = await fse.pathExists(backupFilePath);
- if (!exists) {
+ const sqliteBackupExists = await fse.pathExists(sqliteBackupFilePath);
+ if (!sqliteBackupExists) {
throw new NotFoundError('BACKUP_NOT_FOUND');
}
logger.info('Testing if backup is a valid Gladys SQLite database.');
// Testing if the backup is a valid backup
- const potentialNewDb = new sqlite3.Database(backupFilePath);
+ const potentialNewDb = new sqlite3.Database(sqliteBackupFilePath);
const getAsync = promisify(potentialNewDb.get.bind(potentialNewDb));
const closeAsync = promisify(potentialNewDb.close.bind(potentialNewDb));
// Getting the new user
@@ -34,9 +38,27 @@ async function restoreBackup(backupFilePath) {
// shutting down the current DB
await this.sequelize.close();
// copy the backupFile to the new DB
- await exec(`sqlite3 ${this.config.storage} ".restore '${backupFilePath}'"`);
+ await exec(`sqlite3 ${this.config.storage} ".restore '${sqliteBackupFilePath}'"`);
// done!
- logger.info(`Backup restored. Need reboot now.`);
+ logger.info(`SQLite backup restored`);
+ if (duckDbBackupFolderPath) {
+ // Closing DuckDB current database
+ logger.info(`Restoring DuckDB folder ${duckDbBackupFolderPath}`);
+ await new Promise((resolve) => {
+ db.duckDb.close(() => resolve());
+ });
+ // Delete current DuckDB file
+ const duckDbFilePath = `${this.config.storage.replace('.db', '')}.duckdb`;
+ await fse.remove(duckDbFilePath);
+ const duckDb = new duckdb.Database(duckDbFilePath);
+ const duckDbWriteConnection = duckDb.connect();
+ const duckDbWriteConnectionAllAsync = promisify(duckDbWriteConnection.all).bind(duckDbWriteConnection);
+ await duckDbWriteConnectionAllAsync(`IMPORT DATABASE '${duckDbBackupFolderPath}'`);
+ logger.info(`DuckDB restored with success`);
+ await new Promise((resolve) => {
+ duckDb.close(() => resolve());
+ });
+ }
}
module.exports = {
diff --git a/server/lib/gateway/gateway.restoreBackupEvent.js b/server/lib/gateway/gateway.restoreBackupEvent.js
index 29c08e5b43..5925a13305 100644
--- a/server/lib/gateway/gateway.restoreBackupEvent.js
+++ b/server/lib/gateway/gateway.restoreBackupEvent.js
@@ -13,8 +13,8 @@ async function restoreBackupEvent(event) {
this.restoreErrored = false;
this.restoreInProgress = true;
logger.info(`Receiving restore backup event. File url = ${event.file_url}`);
- const { backupFilePath } = await this.downloadBackup(event.file_url);
- await this.restoreBackup(backupFilePath);
+ const { sqliteBackupFilePath, duckDbBackupFolderPath } = await this.downloadBackup(event.file_url);
+ await this.restoreBackup(sqliteBackupFilePath, duckDbBackupFolderPath);
await this.system.shutdown();
} catch (e) {
logger.warn(e);
diff --git a/server/lib/index.js b/server/lib/index.js
index 1222ba3527..7e36c18c4b 100644
--- a/server/lib/index.js
+++ b/server/lib/index.js
@@ -40,7 +40,7 @@ const { EVENTS } = require('../utils/constants');
* @param {boolean} [params.disableSchedulerLoading] - If true, disable the loading of the scheduler.
* @param {boolean} [params.disableAreaLoading] - If true, disable the loading of the areas.
* @param {boolean} [params.disableJobInit] - If true, disable the pruning of background jobs.
- * @param {boolean} [params.disableDeviceStateAggregation] - If true, disable the aggregation of device states.
+ * @param {boolean} [params.disableDuckDbMigration] - If true, disable the DuckDB migration.
* @returns {object} Return gladys object.
* @example
* const gladys = Gladys();
@@ -131,6 +131,9 @@ function Gladys(params = {}) {
// Execute DB migrations
await db.umzug.up();
+ // Execute DuckDB DB migration
+ await db.duckDbCreateTableIfNotExist();
+
await system.init();
// this should be before device.init
@@ -150,7 +153,7 @@ function Gladys(params = {}) {
await scene.init();
}
if (!params.disableDeviceLoading) {
- await device.init(!params.disableDeviceStateAggregation);
+ await device.init(!params.disableDuckDbMigration);
}
if (!params.disableUserLoading) {
await user.init();
diff --git a/server/lib/job/job.get.js b/server/lib/job/job.get.js
index 942416cd14..0f4ea8a596 100644
--- a/server/lib/job/job.get.js
+++ b/server/lib/job/job.get.js
@@ -15,6 +15,7 @@ const DEFAULT_OPTIONS = {
* @param {number} [options.skip] - Number of elements to skip.
* @param {string} [options.order_by] - Order by.
* @param {string} [options.order_dir] - Order dir (asc/desc).
+ * @param {string} [options.type] - Job type to return.
* @returns {Promise} Resolve with array of jobs.
* @example
* const jobs = await gladys.jobs.get();
@@ -26,10 +27,14 @@ async function get(options) {
include: [],
offset: optionsWithDefault.skip,
order: [[optionsWithDefault.order_by, optionsWithDefault.order_dir]],
+ where: {},
};
if (optionsWithDefault.take !== undefined) {
queryParams.limit = optionsWithDefault.take;
}
+ if (optionsWithDefault.type !== undefined) {
+ queryParams.where.type = optionsWithDefault.type;
+ }
const jobs = await db.Job.findAll(queryParams);
jobs.forEach((job) => {
job.data = JSON.parse(job.data);
diff --git a/server/models/index.js b/server/models/index.js
index 080b7957bf..2573f26e01 100644
--- a/server/models/index.js
+++ b/server/models/index.js
@@ -1,8 +1,13 @@
const Sequelize = require('sequelize');
+const duckdb = require('duckdb');
const Umzug = require('umzug');
+const Promise = require('bluebird');
+const chunk = require('lodash.chunk');
const path = require('path');
+const util = require('util');
const getConfig = require('../utils/getConfig');
const logger = require('../utils/logger');
+const { formatDateInUTC } = require('../utils/date');
const config = getConfig();
@@ -79,10 +84,64 @@ Object.values(models)
.filter((model) => typeof model.associate === 'function')
.forEach((model) => model.associate(models));
+// DuckDB
+const duckDbFilePath = `${config.storage.replace('.db', '')}.duckdb`;
+const duckDb = new duckdb.Database(duckDbFilePath);
+const duckDbWriteConnection = duckDb.connect();
+const duckDbReadConnection = duckDb.connect();
+const duckDbWriteConnectionAllAsync = util.promisify(duckDbWriteConnection.all).bind(duckDbWriteConnection);
+const duckDbReadConnectionAllAsync = util.promisify(duckDbReadConnection.all).bind(duckDbReadConnection);
+
+const duckDbCreateTableIfNotExist = async () => {
+ logger.info(`DuckDB - Creating database table if not exist`);
+ await duckDbWriteConnectionAllAsync(`
+ CREATE TABLE IF NOT EXISTS t_device_feature_state (
+ device_feature_id UUID,
+ value DOUBLE,
+ created_at TIMESTAMPTZ
+ );
+ `);
+};
+
+const duckDbInsertState = async (deviceFeatureId, value, createdAt) => {
+ const createdAtInString = formatDateInUTC(createdAt);
+ await duckDbWriteConnectionAllAsync(
+ 'INSERT INTO t_device_feature_state VALUES (?, ?, ?)',
+ deviceFeatureId,
+ value,
+ createdAtInString,
+ );
+};
+
+const duckDbBatchInsertState = async (deviceFeatureId, states) => {
+ const chunks = chunk(states, 10000);
+ await Promise.each(chunks, async (oneStatesChunk, chunkIndex) => {
+ let queryString = `INSERT INTO t_device_feature_state (device_feature_id, value, created_at) VALUES `;
+ const queryParams = [];
+ oneStatesChunk.forEach((state, index) => {
+ if (index > 0) {
+ queryString += `,`;
+ }
+ queryString += '(?, ?, ?)';
+ queryParams.push(deviceFeatureId);
+ queryParams.push(state.value);
+ queryParams.push(formatDateInUTC(state.created_at));
+ });
+ logger.info(`DuckDB : Inserting chunk ${chunkIndex} for deviceFeature = ${deviceFeatureId}.`);
+ await duckDbWriteConnectionAllAsync(queryString, ...queryParams);
+ });
+};
+
const db = {
...models,
sequelize,
umzug,
+ duckDb,
+ duckDbWriteConnectionAllAsync,
+ duckDbReadConnectionAllAsync,
+ duckDbCreateTableIfNotExist,
+ duckDbInsertState,
+ duckDbBatchInsertState,
};
module.exports = db;
diff --git a/server/package-lock.json b/server/package-lock.json
index db08269606..5f6bb4afa6 100644
--- a/server/package-lock.json
+++ b/server/package-lock.json
@@ -21,7 +21,7 @@
"cross-env": "^7.0.3",
"dayjs": "^1.11.6",
"dockerode": "^3.3.4",
- "downsample": "^1.4.0",
+ "duckdb": "^1.0.0",
"express": "^4.18.2",
"express-rate-limit": "^6.7.0",
"form-data": "^2.3.3",
@@ -30,6 +30,7 @@
"handlebars": "^4.7.7",
"joi": "^17.7.0",
"jsonwebtoken": "^8.5.1",
+ "lodash.chunk": "^4.2.0",
"lodash.clonedeep": "^4.5.0",
"lodash.intersection": "^4.4.0",
"mathjs": "^11.8.0",
@@ -701,8 +702,7 @@
"node_modules/@gar/promisify": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.3.tgz",
- "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==",
- "optional": true
+ "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw=="
},
"node_modules/@gladysassistant/gladys-gateway-js": {
"version": "4.14.0",
@@ -2180,7 +2180,6 @@
"version": "4.2.1",
"resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.2.1.tgz",
"integrity": "sha512-Zn4cw2NEqd+9fiSVWMscnjyQ1a8Yfoc5oBajLeo5w+YBHgDUcEBY2hS4YpTz6iN5f/2zQiktcuM6tS8x1p9dpA==",
- "optional": true,
"dependencies": {
"debug": "^4.1.0",
"depd": "^1.1.2",
@@ -2194,7 +2193,6 @@
"version": "4.3.4",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
"integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
- "optional": true,
"dependencies": {
"ms": "2.1.2"
},
@@ -2211,7 +2209,6 @@
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
"integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==",
- "optional": true,
"engines": {
"node": ">= 0.6"
}
@@ -2219,14 +2216,12 @@
"node_modules/agentkeepalive/node_modules/ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
- "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
- "optional": true
+ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"node_modules/aggregate-error": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz",
"integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==",
- "devOptional": true,
"dependencies": {
"clean-stack": "^2.0.0",
"indent-string": "^4.0.0"
@@ -3102,7 +3097,6 @@
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz",
"integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==",
- "devOptional": true,
"engines": {
"node": ">=6"
}
@@ -3805,10 +3799,353 @@
"resolved": "https://registry.npmjs.org/doublearray/-/doublearray-0.0.2.tgz",
"integrity": "sha512-aw55FtZzT6AmiamEj2kvmR6BuFqvYgKZUkfQ7teqVRNqD5UE0rw8IeW/3gieHNKQ5sPuDKlljWEn4bzv5+1bHw=="
},
- "node_modules/downsample": {
- "version": "1.4.0",
- "resolved": "https://registry.npmjs.org/downsample/-/downsample-1.4.0.tgz",
- "integrity": "sha512-teYPhUPxqwtyICt47t1mP/LjhbRV/ghuKb/LmFDbcZ0CjqFD31tn6rVLZoeCEa1xr8+f2skW8UjRiLiGIKQE4w=="
+ "node_modules/duckdb": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/duckdb/-/duckdb-1.0.0.tgz",
+ "integrity": "sha512-QwpcIeN42A2lL19S70mUFibZgRcEcZpCkKHdzDgecHaYZhXj3+1i2cxSDyAk/RVg5CYnqj1Dp4jAuN4cc80udA==",
+ "hasInstallScript": true,
+ "dependencies": {
+ "@mapbox/node-pre-gyp": "^1.0.0",
+ "node-addon-api": "^7.0.0",
+ "node-gyp": "^9.3.0"
+ }
+ },
+ "node_modules/duckdb/node_modules/@npmcli/fs": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-2.1.2.tgz",
+ "integrity": "sha512-yOJKRvohFOaLqipNtwYB9WugyZKhC/DZC4VYPmpaCzDBrA8YpK3qHZ8/HGscMnE4GqbkLNuVcCnxkeQEdGt6LQ==",
+ "dependencies": {
+ "@gar/promisify": "^1.1.3",
+ "semver": "^7.3.5"
+ },
+ "engines": {
+ "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
+ }
+ },
+ "node_modules/duckdb/node_modules/@npmcli/move-file": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/@npmcli/move-file/-/move-file-2.0.1.tgz",
+ "integrity": "sha512-mJd2Z5TjYWq/ttPLLGqArdtnC74J6bOzg4rMDnN+p1xTacZ2yPRCk2y0oSWQtygLR9YVQXgOcONrwtnk3JupxQ==",
+ "deprecated": "This functionality has been moved to @npmcli/fs",
+ "dependencies": {
+ "mkdirp": "^1.0.4",
+ "rimraf": "^3.0.2"
+ },
+ "engines": {
+ "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
+ }
+ },
+ "node_modules/duckdb/node_modules/are-we-there-yet": {
+ "version": "3.0.1",
+ "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-3.0.1.tgz",
+ "integrity": "sha512-QZW4EDmGwlYur0Yyf/b2uGucHQMa8aFUP7eu9ddR73vvhFyt4V0Vl3QHPcTNJ8l6qYOBdxgXdnBXQrHilfRQBg==",
+ "deprecated": "This package is no longer supported.",
+ "dependencies": {
+ "delegates": "^1.0.0",
+ "readable-stream": "^3.6.0"
+ },
+ "engines": {
+ "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
+ }
+ },
+ "node_modules/duckdb/node_modules/brace-expansion": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
+ "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+ "dependencies": {
+ "balanced-match": "^1.0.0"
+ }
+ },
+ "node_modules/duckdb/node_modules/cacache": {
+ "version": "16.1.3",
+ "resolved": "https://registry.npmjs.org/cacache/-/cacache-16.1.3.tgz",
+ "integrity": "sha512-/+Emcj9DAXxX4cwlLmRI9c166RuL3w30zp4R7Joiv2cQTtTtA+jeuCAjH3ZlGnYS3tKENSrKhAzVVP9GVyzeYQ==",
+ "dependencies": {
+ "@npmcli/fs": "^2.1.0",
+ "@npmcli/move-file": "^2.0.0",
+ "chownr": "^2.0.0",
+ "fs-minipass": "^2.1.0",
+ "glob": "^8.0.1",
+ "infer-owner": "^1.0.4",
+ "lru-cache": "^7.7.1",
+ "minipass": "^3.1.6",
+ "minipass-collect": "^1.0.2",
+ "minipass-flush": "^1.0.5",
+ "minipass-pipeline": "^1.2.4",
+ "mkdirp": "^1.0.4",
+ "p-map": "^4.0.0",
+ "promise-inflight": "^1.0.1",
+ "rimraf": "^3.0.2",
+ "ssri": "^9.0.0",
+ "tar": "^6.1.11",
+ "unique-filename": "^2.0.0"
+ },
+ "engines": {
+ "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
+ }
+ },
+ "node_modules/duckdb/node_modules/cacache/node_modules/glob": {
+ "version": "8.1.0",
+ "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz",
+ "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==",
+ "deprecated": "Glob versions prior to v9 are no longer supported",
+ "dependencies": {
+ "fs.realpath": "^1.0.0",
+ "inflight": "^1.0.4",
+ "inherits": "2",
+ "minimatch": "^5.0.1",
+ "once": "^1.3.0"
+ },
+ "engines": {
+ "node": ">=12"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/isaacs"
+ }
+ },
+ "node_modules/duckdb/node_modules/debug": {
+ "version": "4.3.5",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz",
+ "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==",
+ "dependencies": {
+ "ms": "2.1.2"
+ },
+ "engines": {
+ "node": ">=6.0"
+ },
+ "peerDependenciesMeta": {
+ "supports-color": {
+ "optional": true
+ }
+ }
+ },
+ "node_modules/duckdb/node_modules/gauge": {
+ "version": "4.0.4",
+ "resolved": "https://registry.npmjs.org/gauge/-/gauge-4.0.4.tgz",
+ "integrity": "sha512-f9m+BEN5jkg6a0fZjleidjN51VE1X+mPFQ2DJ0uv1V39oCLCbsGe6yjbBnp7eK7z/+GAon99a3nHuqbuuthyPg==",
+ "deprecated": "This package is no longer supported.",
+ "dependencies": {
+ "aproba": "^1.0.3 || ^2.0.0",
+ "color-support": "^1.1.3",
+ "console-control-strings": "^1.1.0",
+ "has-unicode": "^2.0.1",
+ "signal-exit": "^3.0.7",
+ "string-width": "^4.2.3",
+ "strip-ansi": "^6.0.1",
+ "wide-align": "^1.1.5"
+ },
+ "engines": {
+ "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
+ }
+ },
+ "node_modules/duckdb/node_modules/lru-cache": {
+ "version": "7.18.3",
+ "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz",
+ "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==",
+ "engines": {
+ "node": ">=12"
+ }
+ },
+ "node_modules/duckdb/node_modules/make-fetch-happen": {
+ "version": "10.2.1",
+ "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-10.2.1.tgz",
+ "integrity": "sha512-NgOPbRiaQM10DYXvN3/hhGVI2M5MtITFryzBGxHM5p4wnFxsVCbxkrBrDsk+EZ5OB4jEOT7AjDxtdF+KVEFT7w==",
+ "dependencies": {
+ "agentkeepalive": "^4.2.1",
+ "cacache": "^16.1.0",
+ "http-cache-semantics": "^4.1.0",
+ "http-proxy-agent": "^5.0.0",
+ "https-proxy-agent": "^5.0.0",
+ "is-lambda": "^1.0.1",
+ "lru-cache": "^7.7.1",
+ "minipass": "^3.1.6",
+ "minipass-collect": "^1.0.2",
+ "minipass-fetch": "^2.0.3",
+ "minipass-flush": "^1.0.5",
+ "minipass-pipeline": "^1.2.4",
+ "negotiator": "^0.6.3",
+ "promise-retry": "^2.0.1",
+ "socks-proxy-agent": "^7.0.0",
+ "ssri": "^9.0.0"
+ },
+ "engines": {
+ "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
+ }
+ },
+ "node_modules/duckdb/node_modules/minimatch": {
+ "version": "5.1.6",
+ "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz",
+ "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==",
+ "dependencies": {
+ "brace-expansion": "^2.0.1"
+ },
+ "engines": {
+ "node": ">=10"
+ }
+ },
+ "node_modules/duckdb/node_modules/minipass-fetch": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-2.1.2.tgz",
+ "integrity": "sha512-LT49Zi2/WMROHYoqGgdlQIZh8mLPZmOrN2NdJjMXxYe4nkN6FUyuPuOAOedNJDrx0IRGg9+4guZewtp8hE6TxA==",
+ "dependencies": {
+ "minipass": "^3.1.6",
+ "minipass-sized": "^1.0.3",
+ "minizlib": "^2.1.2"
+ },
+ "engines": {
+ "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
+ },
+ "optionalDependencies": {
+ "encoding": "^0.1.13"
+ }
+ },
+ "node_modules/duckdb/node_modules/ms": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
+ },
+ "node_modules/duckdb/node_modules/node-addon-api": {
+ "version": "7.1.0",
+ "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.0.tgz",
+ "integrity": "sha512-mNcltoe1R8o7STTegSOHdnJNN7s5EUvhoS7ShnTHDyOSd+8H+UdWODq6qSv67PjC8Zc5JRT8+oLAMCr0SIXw7g==",
+ "engines": {
+ "node": "^16 || ^18 || >= 20"
+ }
+ },
+ "node_modules/duckdb/node_modules/node-gyp": {
+ "version": "9.4.1",
+ "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-9.4.1.tgz",
+ "integrity": "sha512-OQkWKbjQKbGkMf/xqI1jjy3oCTgMKJac58G2+bjZb3fza6gW2YrCSdMQYaoTb70crvE//Gngr4f0AgVHmqHvBQ==",
+ "dependencies": {
+ "env-paths": "^2.2.0",
+ "exponential-backoff": "^3.1.1",
+ "glob": "^7.1.4",
+ "graceful-fs": "^4.2.6",
+ "make-fetch-happen": "^10.0.3",
+ "nopt": "^6.0.0",
+ "npmlog": "^6.0.0",
+ "rimraf": "^3.0.2",
+ "semver": "^7.3.5",
+ "tar": "^6.1.2",
+ "which": "^2.0.2"
+ },
+ "bin": {
+ "node-gyp": "bin/node-gyp.js"
+ },
+ "engines": {
+ "node": "^12.13 || ^14.13 || >=16"
+ }
+ },
+ "node_modules/duckdb/node_modules/nopt": {
+ "version": "6.0.0",
+ "resolved": "https://registry.npmjs.org/nopt/-/nopt-6.0.0.tgz",
+ "integrity": "sha512-ZwLpbTgdhuZUnZzjd7nb1ZV+4DoiC6/sfiVKok72ym/4Tlf+DFdlHYmT2JPmcNNWV6Pi3SDf1kT+A4r9RTuT9g==",
+ "dependencies": {
+ "abbrev": "^1.0.0"
+ },
+ "bin": {
+ "nopt": "bin/nopt.js"
+ },
+ "engines": {
+ "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
+ }
+ },
+ "node_modules/duckdb/node_modules/npmlog": {
+ "version": "6.0.2",
+ "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-6.0.2.tgz",
+ "integrity": "sha512-/vBvz5Jfr9dT/aFWd0FIRf+T/Q2WBsLENygUaFUqstqsycmZAP/t5BvFJTK0viFmSUxiUKTUplWy5vt+rvKIxg==",
+ "deprecated": "This package is no longer supported.",
+ "dependencies": {
+ "are-we-there-yet": "^3.0.0",
+ "console-control-strings": "^1.1.0",
+ "gauge": "^4.0.3",
+ "set-blocking": "^2.0.0"
+ },
+ "engines": {
+ "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
+ }
+ },
+ "node_modules/duckdb/node_modules/p-map": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz",
+ "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==",
+ "dependencies": {
+ "aggregate-error": "^3.0.0"
+ },
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
+ "node_modules/duckdb/node_modules/socks-proxy-agent": {
+ "version": "7.0.0",
+ "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-7.0.0.tgz",
+ "integrity": "sha512-Fgl0YPZ902wEsAyiQ+idGd1A7rSFx/ayC1CQVMw5P+EQx2V0SgpGtf6OKFhVjPflPUl9YMmEOnmfjCdMUsygww==",
+ "dependencies": {
+ "agent-base": "^6.0.2",
+ "debug": "^4.3.3",
+ "socks": "^2.6.2"
+ },
+ "engines": {
+ "node": ">= 10"
+ }
+ },
+ "node_modules/duckdb/node_modules/ssri": {
+ "version": "9.0.1",
+ "resolved": "https://registry.npmjs.org/ssri/-/ssri-9.0.1.tgz",
+ "integrity": "sha512-o57Wcn66jMQvfHG1FlYbWeZWW/dHZhJXjpIcTfXldXEk5nz5lStPo3mK0OJQfGR3RbZUlbISexbljkJzuEj/8Q==",
+ "dependencies": {
+ "minipass": "^3.1.1"
+ },
+ "engines": {
+ "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
+ }
+ },
+ "node_modules/duckdb/node_modules/unique-filename": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-2.0.1.tgz",
+ "integrity": "sha512-ODWHtkkdx3IAR+veKxFV+VBkUMcN+FaqzUUd7IZzt+0zhDZFPFxhlqwPF3YQvMHx1TD0tdgYl+kuPnJ8E6ql7A==",
+ "dependencies": {
+ "unique-slug": "^3.0.0"
+ },
+ "engines": {
+ "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
+ }
+ },
+ "node_modules/duckdb/node_modules/unique-slug": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/unique-slug/-/unique-slug-3.0.0.tgz",
+ "integrity": "sha512-8EyMynh679x/0gqE9fT9oilG+qEt+ibFyqjuVTsZn1+CMxH+XLlpvr2UZx4nVcCwTpx81nICr2JQFkM+HPLq4w==",
+ "dependencies": {
+ "imurmurhash": "^0.1.4"
+ },
+ "engines": {
+ "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
+ }
+ },
+ "node_modules/duckdb/node_modules/which": {
+ "version": "2.0.2",
+ "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
+ "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
+ "dependencies": {
+ "isexe": "^2.0.0"
+ },
+ "bin": {
+ "node-which": "bin/node-which"
+ },
+ "engines": {
+ "node": ">= 8"
+ }
+ },
+ "node_modules/duckdb/node_modules/wide-align": {
+ "version": "1.1.5",
+ "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz",
+ "integrity": "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==",
+ "dependencies": {
+ "string-width": "^1.0.2 || 2 || 3 || 4"
+ }
},
"node_modules/ecdsa-sig-formatter": {
"version": "1.0.11",
@@ -3995,7 +4332,6 @@
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz",
"integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==",
- "optional": true,
"engines": {
"node": ">=6"
}
@@ -4015,8 +4351,7 @@
"node_modules/err-code": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz",
- "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==",
- "optional": true
+ "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA=="
},
"node_modules/error-ex": {
"version": "1.3.2",
@@ -4980,6 +5315,11 @@
"node": ">=0.8.x"
}
},
+ "node_modules/exponential-backoff": {
+ "version": "3.1.1",
+ "resolved": "https://registry.npmjs.org/exponential-backoff/-/exponential-backoff-3.1.1.tgz",
+ "integrity": "sha512-dX7e/LHVJ6W3DE1MHWi9S1EYzDESENfLrYohG2G++ovZrYOkm4Knwa0mc1cn84xJOR4KEU0WSchhLbd0UklbHw=="
+ },
"node_modules/expose-loader": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/expose-loader/-/expose-loader-4.1.0.tgz",
@@ -5813,8 +6153,7 @@
"node_modules/http-cache-semantics": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz",
- "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==",
- "optional": true
+ "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ=="
},
"node_modules/http-errors": {
"version": "2.0.0",
@@ -5902,7 +6241,6 @@
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
"integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==",
- "optional": true,
"dependencies": {
"ms": "^2.0.0"
}
@@ -5992,7 +6330,6 @@
"version": "0.1.4",
"resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
"integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==",
- "devOptional": true,
"engines": {
"node": ">=0.8.19"
}
@@ -6001,7 +6338,6 @@
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz",
"integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==",
- "devOptional": true,
"engines": {
"node": ">=8"
}
@@ -6009,8 +6345,7 @@
"node_modules/infer-owner": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/infer-owner/-/infer-owner-1.0.4.tgz",
- "integrity": "sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A==",
- "optional": true
+ "integrity": "sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A=="
},
"node_modules/inflection": {
"version": "1.13.4",
@@ -6065,8 +6400,7 @@
"node_modules/ip": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz",
- "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==",
- "optional": true
+ "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ=="
},
"node_modules/ipaddr.js": {
"version": "1.9.1",
@@ -6192,8 +6526,7 @@
"node_modules/is-lambda": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/is-lambda/-/is-lambda-1.0.1.tgz",
- "integrity": "sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ==",
- "optional": true
+ "integrity": "sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ=="
},
"node_modules/is-nan": {
"version": "1.3.2",
@@ -7123,6 +7456,11 @@
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
"integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
},
+ "node_modules/lodash.chunk": {
+ "version": "4.2.0",
+ "resolved": "https://registry.npmjs.org/lodash.chunk/-/lodash.chunk-4.2.0.tgz",
+ "integrity": "sha512-ZzydJKfUHJwHa+hF5X66zLFCBrWn5GeF28OHEr4WVWtNDXlQ/IjWKPBiikqKo2ne0+v6JgCgJ0GzJp8k8bHC7w=="
+ },
"node_modules/lodash.clonedeep": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz",
@@ -7607,7 +7945,6 @@
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/minipass-collect/-/minipass-collect-1.0.2.tgz",
"integrity": "sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA==",
- "optional": true,
"dependencies": {
"minipass": "^3.0.0"
},
@@ -7636,7 +7973,6 @@
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/minipass-flush/-/minipass-flush-1.0.5.tgz",
"integrity": "sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw==",
- "optional": true,
"dependencies": {
"minipass": "^3.0.0"
},
@@ -7648,7 +7984,6 @@
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz",
"integrity": "sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==",
- "optional": true,
"dependencies": {
"minipass": "^3.0.0"
},
@@ -7660,7 +7995,6 @@
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/minipass-sized/-/minipass-sized-1.0.3.tgz",
"integrity": "sha512-MbkQQ2CTiBMlA2Dm/5cY+9SWFEN8pzzOXi6rlM5Xxq0Yqbda5ZQy9sU75a673FE9ZK0Zsbr6Y5iP6u9nktfg2g==",
- "optional": true,
"dependencies": {
"minipass": "^3.0.0"
},
@@ -9117,14 +9451,12 @@
"node_modules/promise-inflight": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz",
- "integrity": "sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==",
- "optional": true
+ "integrity": "sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g=="
},
"node_modules/promise-retry": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz",
"integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==",
- "optional": true,
"dependencies": {
"err-code": "^2.0.2",
"retry": "^0.12.0"
@@ -9137,7 +9469,6 @@
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz",
"integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==",
- "optional": true,
"engines": {
"node": ">= 4"
}
@@ -10036,7 +10367,6 @@
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz",
"integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==",
- "optional": true,
"engines": {
"node": ">= 6.0.0",
"npm": ">= 3.0.0"
@@ -10114,7 +10444,6 @@
"version": "2.7.1",
"resolved": "https://registry.npmjs.org/socks/-/socks-2.7.1.tgz",
"integrity": "sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==",
- "optional": true,
"dependencies": {
"ip": "^2.0.0",
"smart-buffer": "^4.2.0"
@@ -12244,8 +12573,7 @@
"@gar/promisify": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.3.tgz",
- "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==",
- "optional": true
+ "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw=="
},
"@gladysassistant/gladys-gateway-js": {
"version": "4.14.0",
@@ -13589,7 +13917,6 @@
"version": "4.2.1",
"resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.2.1.tgz",
"integrity": "sha512-Zn4cw2NEqd+9fiSVWMscnjyQ1a8Yfoc5oBajLeo5w+YBHgDUcEBY2hS4YpTz6iN5f/2zQiktcuM6tS8x1p9dpA==",
- "optional": true,
"requires": {
"debug": "^4.1.0",
"depd": "^1.1.2",
@@ -13600,7 +13927,6 @@
"version": "4.3.4",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
"integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
- "optional": true,
"requires": {
"ms": "2.1.2"
}
@@ -13608,14 +13934,12 @@
"depd": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
- "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==",
- "optional": true
+ "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ=="
},
"ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
- "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
- "optional": true
+ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
}
}
},
@@ -13623,7 +13947,6 @@
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz",
"integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==",
- "devOptional": true,
"requires": {
"clean-stack": "^2.0.0",
"indent-string": "^4.0.0"
@@ -14276,8 +14599,7 @@
"clean-stack": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz",
- "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==",
- "devOptional": true
+ "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A=="
},
"cli-color": {
"version": "2.0.3",
@@ -14816,10 +15138,266 @@
"resolved": "https://registry.npmjs.org/doublearray/-/doublearray-0.0.2.tgz",
"integrity": "sha512-aw55FtZzT6AmiamEj2kvmR6BuFqvYgKZUkfQ7teqVRNqD5UE0rw8IeW/3gieHNKQ5sPuDKlljWEn4bzv5+1bHw=="
},
- "downsample": {
- "version": "1.4.0",
- "resolved": "https://registry.npmjs.org/downsample/-/downsample-1.4.0.tgz",
- "integrity": "sha512-teYPhUPxqwtyICt47t1mP/LjhbRV/ghuKb/LmFDbcZ0CjqFD31tn6rVLZoeCEa1xr8+f2skW8UjRiLiGIKQE4w=="
+ "duckdb": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/duckdb/-/duckdb-1.0.0.tgz",
+ "integrity": "sha512-QwpcIeN42A2lL19S70mUFibZgRcEcZpCkKHdzDgecHaYZhXj3+1i2cxSDyAk/RVg5CYnqj1Dp4jAuN4cc80udA==",
+ "requires": {
+ "@mapbox/node-pre-gyp": "^1.0.0",
+ "node-addon-api": "^7.0.0",
+ "node-gyp": "^9.3.0"
+ },
+ "dependencies": {
+ "@npmcli/fs": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-2.1.2.tgz",
+ "integrity": "sha512-yOJKRvohFOaLqipNtwYB9WugyZKhC/DZC4VYPmpaCzDBrA8YpK3qHZ8/HGscMnE4GqbkLNuVcCnxkeQEdGt6LQ==",
+ "requires": {
+ "@gar/promisify": "^1.1.3",
+ "semver": "^7.3.5"
+ }
+ },
+ "@npmcli/move-file": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/@npmcli/move-file/-/move-file-2.0.1.tgz",
+ "integrity": "sha512-mJd2Z5TjYWq/ttPLLGqArdtnC74J6bOzg4rMDnN+p1xTacZ2yPRCk2y0oSWQtygLR9YVQXgOcONrwtnk3JupxQ==",
+ "requires": {
+ "mkdirp": "^1.0.4",
+ "rimraf": "^3.0.2"
+ }
+ },
+ "are-we-there-yet": {
+ "version": "3.0.1",
+ "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-3.0.1.tgz",
+ "integrity": "sha512-QZW4EDmGwlYur0Yyf/b2uGucHQMa8aFUP7eu9ddR73vvhFyt4V0Vl3QHPcTNJ8l6qYOBdxgXdnBXQrHilfRQBg==",
+ "requires": {
+ "delegates": "^1.0.0",
+ "readable-stream": "^3.6.0"
+ }
+ },
+ "brace-expansion": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
+ "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+ "requires": {
+ "balanced-match": "^1.0.0"
+ }
+ },
+ "cacache": {
+ "version": "16.1.3",
+ "resolved": "https://registry.npmjs.org/cacache/-/cacache-16.1.3.tgz",
+ "integrity": "sha512-/+Emcj9DAXxX4cwlLmRI9c166RuL3w30zp4R7Joiv2cQTtTtA+jeuCAjH3ZlGnYS3tKENSrKhAzVVP9GVyzeYQ==",
+ "requires": {
+ "@npmcli/fs": "^2.1.0",
+ "@npmcli/move-file": "^2.0.0",
+ "chownr": "^2.0.0",
+ "fs-minipass": "^2.1.0",
+ "glob": "^8.0.1",
+ "infer-owner": "^1.0.4",
+ "lru-cache": "^7.7.1",
+ "minipass": "^3.1.6",
+ "minipass-collect": "^1.0.2",
+ "minipass-flush": "^1.0.5",
+ "minipass-pipeline": "^1.2.4",
+ "mkdirp": "^1.0.4",
+ "p-map": "^4.0.0",
+ "promise-inflight": "^1.0.1",
+ "rimraf": "^3.0.2",
+ "ssri": "^9.0.0",
+ "tar": "^6.1.11",
+ "unique-filename": "^2.0.0"
+ },
+ "dependencies": {
+ "glob": {
+ "version": "8.1.0",
+ "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz",
+ "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==",
+ "requires": {
+ "fs.realpath": "^1.0.0",
+ "inflight": "^1.0.4",
+ "inherits": "2",
+ "minimatch": "^5.0.1",
+ "once": "^1.3.0"
+ }
+ }
+ }
+ },
+ "debug": {
+ "version": "4.3.5",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz",
+ "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==",
+ "requires": {
+ "ms": "2.1.2"
+ }
+ },
+ "gauge": {
+ "version": "4.0.4",
+ "resolved": "https://registry.npmjs.org/gauge/-/gauge-4.0.4.tgz",
+ "integrity": "sha512-f9m+BEN5jkg6a0fZjleidjN51VE1X+mPFQ2DJ0uv1V39oCLCbsGe6yjbBnp7eK7z/+GAon99a3nHuqbuuthyPg==",
+ "requires": {
+ "aproba": "^1.0.3 || ^2.0.0",
+ "color-support": "^1.1.3",
+ "console-control-strings": "^1.1.0",
+ "has-unicode": "^2.0.1",
+ "signal-exit": "^3.0.7",
+ "string-width": "^4.2.3",
+ "strip-ansi": "^6.0.1",
+ "wide-align": "^1.1.5"
+ }
+ },
+ "lru-cache": {
+ "version": "7.18.3",
+ "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz",
+ "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA=="
+ },
+ "make-fetch-happen": {
+ "version": "10.2.1",
+ "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-10.2.1.tgz",
+ "integrity": "sha512-NgOPbRiaQM10DYXvN3/hhGVI2M5MtITFryzBGxHM5p4wnFxsVCbxkrBrDsk+EZ5OB4jEOT7AjDxtdF+KVEFT7w==",
+ "requires": {
+ "agentkeepalive": "^4.2.1",
+ "cacache": "^16.1.0",
+ "http-cache-semantics": "^4.1.0",
+ "http-proxy-agent": "^5.0.0",
+ "https-proxy-agent": "^5.0.0",
+ "is-lambda": "^1.0.1",
+ "lru-cache": "^7.7.1",
+ "minipass": "^3.1.6",
+ "minipass-collect": "^1.0.2",
+ "minipass-fetch": "^2.0.3",
+ "minipass-flush": "^1.0.5",
+ "minipass-pipeline": "^1.2.4",
+ "negotiator": "^0.6.3",
+ "promise-retry": "^2.0.1",
+ "socks-proxy-agent": "^7.0.0",
+ "ssri": "^9.0.0"
+ }
+ },
+ "minimatch": {
+ "version": "5.1.6",
+ "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz",
+ "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==",
+ "requires": {
+ "brace-expansion": "^2.0.1"
+ }
+ },
+ "minipass-fetch": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-2.1.2.tgz",
+ "integrity": "sha512-LT49Zi2/WMROHYoqGgdlQIZh8mLPZmOrN2NdJjMXxYe4nkN6FUyuPuOAOedNJDrx0IRGg9+4guZewtp8hE6TxA==",
+ "requires": {
+ "encoding": "^0.1.13",
+ "minipass": "^3.1.6",
+ "minipass-sized": "^1.0.3",
+ "minizlib": "^2.1.2"
+ }
+ },
+ "ms": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
+ },
+ "node-addon-api": {
+ "version": "7.1.0",
+ "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.0.tgz",
+ "integrity": "sha512-mNcltoe1R8o7STTegSOHdnJNN7s5EUvhoS7ShnTHDyOSd+8H+UdWODq6qSv67PjC8Zc5JRT8+oLAMCr0SIXw7g=="
+ },
+ "node-gyp": {
+ "version": "9.4.1",
+ "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-9.4.1.tgz",
+ "integrity": "sha512-OQkWKbjQKbGkMf/xqI1jjy3oCTgMKJac58G2+bjZb3fza6gW2YrCSdMQYaoTb70crvE//Gngr4f0AgVHmqHvBQ==",
+ "requires": {
+ "env-paths": "^2.2.0",
+ "exponential-backoff": "^3.1.1",
+ "glob": "^7.1.4",
+ "graceful-fs": "^4.2.6",
+ "make-fetch-happen": "^10.0.3",
+ "nopt": "^6.0.0",
+ "npmlog": "^6.0.0",
+ "rimraf": "^3.0.2",
+ "semver": "^7.3.5",
+ "tar": "^6.1.2",
+ "which": "^2.0.2"
+ }
+ },
+ "nopt": {
+ "version": "6.0.0",
+ "resolved": "https://registry.npmjs.org/nopt/-/nopt-6.0.0.tgz",
+ "integrity": "sha512-ZwLpbTgdhuZUnZzjd7nb1ZV+4DoiC6/sfiVKok72ym/4Tlf+DFdlHYmT2JPmcNNWV6Pi3SDf1kT+A4r9RTuT9g==",
+ "requires": {
+ "abbrev": "^1.0.0"
+ }
+ },
+ "npmlog": {
+ "version": "6.0.2",
+ "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-6.0.2.tgz",
+ "integrity": "sha512-/vBvz5Jfr9dT/aFWd0FIRf+T/Q2WBsLENygUaFUqstqsycmZAP/t5BvFJTK0viFmSUxiUKTUplWy5vt+rvKIxg==",
+ "requires": {
+ "are-we-there-yet": "^3.0.0",
+ "console-control-strings": "^1.1.0",
+ "gauge": "^4.0.3",
+ "set-blocking": "^2.0.0"
+ }
+ },
+ "p-map": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz",
+ "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==",
+ "requires": {
+ "aggregate-error": "^3.0.0"
+ }
+ },
+ "socks-proxy-agent": {
+ "version": "7.0.0",
+ "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-7.0.0.tgz",
+ "integrity": "sha512-Fgl0YPZ902wEsAyiQ+idGd1A7rSFx/ayC1CQVMw5P+EQx2V0SgpGtf6OKFhVjPflPUl9YMmEOnmfjCdMUsygww==",
+ "requires": {
+ "agent-base": "^6.0.2",
+ "debug": "^4.3.3",
+ "socks": "^2.6.2"
+ }
+ },
+ "ssri": {
+ "version": "9.0.1",
+ "resolved": "https://registry.npmjs.org/ssri/-/ssri-9.0.1.tgz",
+ "integrity": "sha512-o57Wcn66jMQvfHG1FlYbWeZWW/dHZhJXjpIcTfXldXEk5nz5lStPo3mK0OJQfGR3RbZUlbISexbljkJzuEj/8Q==",
+ "requires": {
+ "minipass": "^3.1.1"
+ }
+ },
+ "unique-filename": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-2.0.1.tgz",
+ "integrity": "sha512-ODWHtkkdx3IAR+veKxFV+VBkUMcN+FaqzUUd7IZzt+0zhDZFPFxhlqwPF3YQvMHx1TD0tdgYl+kuPnJ8E6ql7A==",
+ "requires": {
+ "unique-slug": "^3.0.0"
+ }
+ },
+ "unique-slug": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/unique-slug/-/unique-slug-3.0.0.tgz",
+ "integrity": "sha512-8EyMynh679x/0gqE9fT9oilG+qEt+ibFyqjuVTsZn1+CMxH+XLlpvr2UZx4nVcCwTpx81nICr2JQFkM+HPLq4w==",
+ "requires": {
+ "imurmurhash": "^0.1.4"
+ }
+ },
+ "which": {
+ "version": "2.0.2",
+ "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
+ "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
+ "requires": {
+ "isexe": "^2.0.0"
+ }
+ },
+ "wide-align": {
+ "version": "1.1.5",
+ "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz",
+ "integrity": "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==",
+ "requires": {
+ "string-width": "^1.0.2 || 2 || 3 || 4"
+ }
+ }
+ }
},
"ecdsa-sig-formatter": {
"version": "1.0.11",
@@ -14966,8 +15544,7 @@
"env-paths": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz",
- "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==",
- "optional": true
+ "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A=="
},
"envinfo": {
"version": "7.8.1",
@@ -14978,8 +15555,7 @@
"err-code": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz",
- "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==",
- "optional": true
+ "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA=="
},
"error-ex": {
"version": "1.3.2",
@@ -15701,6 +16277,11 @@
"integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
"dev": true
},
+ "exponential-backoff": {
+ "version": "3.1.1",
+ "resolved": "https://registry.npmjs.org/exponential-backoff/-/exponential-backoff-3.1.1.tgz",
+ "integrity": "sha512-dX7e/LHVJ6W3DE1MHWi9S1EYzDESENfLrYohG2G++ovZrYOkm4Knwa0mc1cn84xJOR4KEU0WSchhLbd0UklbHw=="
+ },
"expose-loader": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/expose-loader/-/expose-loader-4.1.0.tgz",
@@ -16295,8 +16876,7 @@
"http-cache-semantics": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz",
- "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==",
- "optional": true
+ "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ=="
},
"http-errors": {
"version": "2.0.0",
@@ -16363,7 +16943,6 @@
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
"integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==",
- "optional": true,
"requires": {
"ms": "^2.0.0"
}
@@ -16417,20 +16996,17 @@
"imurmurhash": {
"version": "0.1.4",
"resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
- "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==",
- "devOptional": true
+ "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA=="
},
"indent-string": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz",
- "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==",
- "devOptional": true
+ "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg=="
},
"infer-owner": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/infer-owner/-/infer-owner-1.0.4.tgz",
- "integrity": "sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A==",
- "optional": true
+ "integrity": "sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A=="
},
"inflection": {
"version": "1.13.4",
@@ -16476,8 +17052,7 @@
"ip": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz",
- "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==",
- "optional": true
+ "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ=="
},
"ipaddr.js": {
"version": "1.9.1",
@@ -16564,8 +17139,7 @@
"is-lambda": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/is-lambda/-/is-lambda-1.0.1.tgz",
- "integrity": "sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ==",
- "optional": true
+ "integrity": "sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ=="
},
"is-nan": {
"version": "1.3.2",
@@ -17281,6 +17855,11 @@
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
"integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
},
+ "lodash.chunk": {
+ "version": "4.2.0",
+ "resolved": "https://registry.npmjs.org/lodash.chunk/-/lodash.chunk-4.2.0.tgz",
+ "integrity": "sha512-ZzydJKfUHJwHa+hF5X66zLFCBrWn5GeF28OHEr4WVWtNDXlQ/IjWKPBiikqKo2ne0+v6JgCgJ0GzJp8k8bHC7w=="
+ },
"lodash.clonedeep": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz",
@@ -17692,7 +18271,6 @@
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/minipass-collect/-/minipass-collect-1.0.2.tgz",
"integrity": "sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA==",
- "optional": true,
"requires": {
"minipass": "^3.0.0"
}
@@ -17713,7 +18291,6 @@
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/minipass-flush/-/minipass-flush-1.0.5.tgz",
"integrity": "sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw==",
- "optional": true,
"requires": {
"minipass": "^3.0.0"
}
@@ -17722,7 +18299,6 @@
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz",
"integrity": "sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==",
- "optional": true,
"requires": {
"minipass": "^3.0.0"
}
@@ -17731,7 +18307,6 @@
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/minipass-sized/-/minipass-sized-1.0.3.tgz",
"integrity": "sha512-MbkQQ2CTiBMlA2Dm/5cY+9SWFEN8pzzOXi6rlM5Xxq0Yqbda5ZQy9sU75a673FE9ZK0Zsbr6Y5iP6u9nktfg2g==",
- "optional": true,
"requires": {
"minipass": "^3.0.0"
}
@@ -18843,14 +19418,12 @@
"promise-inflight": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz",
- "integrity": "sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==",
- "optional": true
+ "integrity": "sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g=="
},
"promise-retry": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz",
"integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==",
- "optional": true,
"requires": {
"err-code": "^2.0.2",
"retry": "^0.12.0"
@@ -18859,8 +19432,7 @@
"retry": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz",
- "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==",
- "optional": true
+ "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow=="
}
}
},
@@ -19526,8 +20098,7 @@
"smart-buffer": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz",
- "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==",
- "optional": true
+ "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg=="
},
"socket.io-client": {
"version": "4.5.4",
@@ -19583,7 +20154,6 @@
"version": "2.7.1",
"resolved": "https://registry.npmjs.org/socks/-/socks-2.7.1.tgz",
"integrity": "sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==",
- "optional": true,
"requires": {
"ip": "^2.0.0",
"smart-buffer": "^4.2.0"
diff --git a/server/package.json b/server/package.json
index 6a003616c7..c7cca8f9ff 100644
--- a/server/package.json
+++ b/server/package.json
@@ -86,7 +86,7 @@
"cross-env": "^7.0.3",
"dayjs": "^1.11.6",
"dockerode": "^3.3.4",
- "downsample": "^1.4.0",
+ "duckdb": "^1.0.0",
"express": "^4.18.2",
"express-rate-limit": "^6.7.0",
"form-data": "^2.3.3",
@@ -95,6 +95,7 @@
"handlebars": "^4.7.7",
"joi": "^17.7.0",
"jsonwebtoken": "^8.5.1",
+ "lodash.chunk": "^4.2.0",
"lodash.clonedeep": "^4.5.0",
"lodash.intersection": "^4.4.0",
"mathjs": "^11.8.0",
diff --git a/server/test/bootstrap.test.js b/server/test/bootstrap.test.js
index b610ca29f8..e12f7f5e27 100644
--- a/server/test/bootstrap.test.js
+++ b/server/test/bootstrap.test.js
@@ -20,7 +20,7 @@ before(async function before() {
disableService: true,
disableBrainLoading: true,
disableSchedulerLoading: true,
- disableDeviceStateAggregation: true,
+ disableDuckDbMigration: true,
jwtSecret: 'secret',
};
const gladys = Gladys(config);
diff --git a/server/test/controllers/device/device.controller.test.js b/server/test/controllers/device/device.controller.test.js
index f58facf988..f59e82caec 100644
--- a/server/test/controllers/device/device.controller.test.js
+++ b/server/test/controllers/device/device.controller.test.js
@@ -1,15 +1,9 @@
const { expect } = require('chai');
-const uuid = require('uuid');
-const EventEmitter = require('events');
-const { fake } = require('sinon');
const db = require('../../../models');
-const Device = require('../../../lib/device');
-const Job = require('../../../lib/job');
const { authenticatedRequest } = require('../request.test');
const insertStates = async (intervalInMinutes) => {
- const queryInterface = db.sequelize.getQueryInterface();
const deviceFeatureStateToInsert = [];
const now = new Date();
const statesToInsert = 2000;
@@ -17,14 +11,11 @@ const insertStates = async (intervalInMinutes) => {
const startAt = new Date(now.getTime() - intervalInMinutes * 60 * 1000);
const date = new Date(startAt.getTime() + ((intervalInMinutes * 60 * 1000) / statesToInsert) * i);
deviceFeatureStateToInsert.push({
- id: uuid.v4(),
- device_feature_id: 'ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4',
value: i,
created_at: date,
- updated_at: date,
});
}
- await queryInterface.bulkInsert('t_device_feature_state', deviceFeatureStateToInsert);
+ await db.duckDbBatchInsertState('ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4', deviceFeatureStateToInsert);
};
describe('POST /api/v1/device', () => {
@@ -64,15 +55,6 @@ describe('GET /api/v1/device_feature/aggregated_states', () => {
beforeEach(async function BeforeEach() {
this.timeout(10000);
await insertStates(365 * 24 * 60);
- const variable = {
- getValue: fake.resolves(null),
- };
- const event = new EventEmitter();
- const job = new Job(event);
- const device = new Device(event, {}, {}, {}, {}, variable, job);
- await device.calculateAggregate('hourly');
- await device.calculateAggregate('daily');
- await device.calculateAggregate('monthly');
});
it('should get device aggregated state by selector', async () => {
await authenticatedRequest
@@ -137,6 +119,49 @@ describe('GET /api/v1/device', () => {
});
});
+describe('GET /api/v1/device/duckdb_migration_state', () => {
+ it('should get duck db migration state', async () => {
+ await authenticatedRequest
+ .get('/api/v1/device/duckdb_migration_state')
+ .expect('Content-Type', /json/)
+ .expect(200)
+ .then((res) => {
+ expect(res.body).to.deep.equal({
+ duck_db_device_count: 0,
+ is_duck_db_migrated: false,
+ sqlite_db_device_state_count: 0,
+ });
+ });
+ });
+});
+
+describe('POST /api/v1/device/purge_all_sqlite_state', () => {
+ it('should delete all sqlite states', async () => {
+ await authenticatedRequest
+ .post('/api/v1/device/purge_all_sqlite_state')
+ .expect('Content-Type', /json/)
+ .expect(200)
+ .then((res) => {
+ expect(res.body).to.deep.equal({
+ success: true,
+ });
+ });
+ });
+});
+describe('POST /api/v1/device/migrate_from_sqlite_to_duckdb', () => {
+ it('should migrate to duckdb', async () => {
+ await authenticatedRequest
+ .post('/api/v1/device/migrate_from_sqlite_to_duckdb')
+ .expect('Content-Type', /json/)
+ .expect(200)
+ .then((res) => {
+ expect(res.body).to.deep.equal({
+ success: true,
+ });
+ });
+ });
+});
+
describe('GET /api/v1/service/:service_name/device', () => {
it('should return devices in service test-service', async () => {
await authenticatedRequest
diff --git a/server/test/helpers/db.test.js b/server/test/helpers/db.test.js
index af7da8a243..65f2ee68db 100644
--- a/server/test/helpers/db.test.js
+++ b/server/test/helpers/db.test.js
@@ -17,10 +17,13 @@ const seedDb = async () => {
};
const cleanDb = async () => {
+ // Clean SQLite database
const queryInterface = db.sequelize.getQueryInterface();
await Promise.each(reversedSeed, async (seed) => {
await seed.down(queryInterface);
});
+ // Clean DuckDB database
+ await db.duckDbWriteConnectionAllAsync('DELETE FROM t_device_feature_state');
};
module.exports = {
diff --git a/server/test/lib/device/device.calculateAggregate.test.js b/server/test/lib/device/device.calculateAggregate.test.js
deleted file mode 100644
index dfc843a3a9..0000000000
--- a/server/test/lib/device/device.calculateAggregate.test.js
+++ /dev/null
@@ -1,165 +0,0 @@
-const { expect } = require('chai');
-const uuid = require('uuid');
-const EventEmitter = require('events');
-const sinon = require('sinon');
-
-const { fake } = sinon;
-const db = require('../../../models');
-const Device = require('../../../lib/device');
-const Job = require('../../../lib/job');
-
-const event = new EventEmitter();
-
-const insertStates = async (fixedHour = true) => {
- const queryInterface = db.sequelize.getQueryInterface();
- const deviceFeatureStateToInsert = [];
- const now = new Date();
- for (let i = 1; i <= 30; i += 1) {
- for (let j = 0; j < 120; j += 1) {
- const date = new Date(now.getFullYear() - 1, 10, i, fixedHour ? 12 : Math.round((j + 1) / 5), 0);
- deviceFeatureStateToInsert.push({
- id: uuid.v4(),
- device_feature_id: 'ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4',
- value: i,
- created_at: date,
- updated_at: date,
- });
- }
- }
- await queryInterface.bulkInsert('t_device_feature_state', deviceFeatureStateToInsert);
-};
-
-describe('Device.calculateAggregate', function Before() {
- this.timeout(60000);
- let clock;
- beforeEach(async () => {
- const queryInterface = db.sequelize.getQueryInterface();
- await queryInterface.bulkDelete('t_device_feature_state');
- await queryInterface.bulkDelete('t_device_feature_state_aggregate');
- await db.DeviceFeature.update(
- {
- last_monthly_aggregate: null,
- last_daily_aggregate: null,
- last_hourly_aggregate: null,
- },
- { where: {} },
- );
- clock = sinon.useFakeTimers({
- now: 1635131280000,
- });
- });
- afterEach(() => {
- clock.restore();
- });
- it('should calculate hourly aggregate', async () => {
- await insertStates(false);
- const variable = {
- // we modify the retention policy to take the last 1000 days (it'll cover this last year)
- getValue: fake.resolves('1000'),
- };
- const job = new Job(event);
- const device = new Device(event, {}, {}, {}, {}, variable, job);
- await device.calculateAggregate('hourly');
- const deviceFeatureStates = await db.DeviceFeatureStateAggregate.findAll({
- raw: true,
- attributes: ['value', 'created_at'],
- order: [['created_at', 'ASC']],
- });
- // Max number of events
- expect(deviceFeatureStates.length).to.equal(3600);
- });
- it('should calculate hourly aggregate with retention policy instead of last aggregate', async () => {
- await insertStates(false);
- const deviceFeaturesInDb = await db.DeviceFeature.findAll();
- deviceFeaturesInDb[0].update({
- // old date
- last_hourly_aggregate: new Date(2010, 10, 10),
- });
- const variable = {
- // we modify the retention policy to take the last 5 days only (for testing)
- getValue: fake.resolves('5'),
- };
- const job = new Job(event);
- const device = new Device(event, {}, {}, {}, {}, variable, job);
- await device.calculateAggregate('hourly');
- });
- it('should calculate hourly aggregate with last aggregate from device', async () => {
- await insertStates(false);
- const deviceFeaturesInDb = await db.DeviceFeature.findAll();
- deviceFeaturesInDb[0].update({
- last_hourly_aggregate: new Date(),
- });
- const variable = {
- // we modify the retention policy to take the last 1000 days (for testing)
- getValue: fake.resolves('1000'),
- };
- const job = new Job(event);
- const device = new Device(event, {}, {}, {}, {}, variable, job);
- await device.calculateAggregate('hourly');
- });
- it('should calculate daily aggregate', async () => {
- await insertStates(true);
- const variable = {
- getValue: fake.resolves(null),
- };
- const job = new Job(event);
- const device = new Device(event, {}, {}, {}, {}, variable, job);
- await device.calculateAggregate('daily');
- const deviceFeatureStates = await db.DeviceFeatureStateAggregate.findAll({
- raw: true,
- attributes: ['value', 'created_at'],
- order: [['created_at', 'ASC']],
- });
- // 30 days * 100 = 3000
- expect(deviceFeatureStates.length).to.equal(3000);
- });
- it('should calculate monthly aggregate', async () => {
- await insertStates(true);
- const variable = {
- getValue: fake.resolves(null),
- };
- const job = new Job(event);
- const device = new Device(event, {}, {}, {}, {}, variable, job);
- await device.calculateAggregate('monthly');
- const deviceFeatureStates = await db.DeviceFeatureStateAggregate.findAll({
- raw: true,
- attributes: ['value', 'created_at'],
- order: [['created_at', 'ASC']],
- });
- // 1 month * 100
- expect(deviceFeatureStates.length).to.equal(100);
- });
- it('should run the hourly aggregate task', async () => {
- await insertStates(true);
- const variable = {
- // we modify the retention policy to take the last 1000 days (it'll cover this last year)
- getValue: fake.resolves('1000'),
- };
- const job = new Job(event);
- const device = new Device(event, {}, {}, {}, {}, variable, job);
- await device.onHourlyDeviceAggregateEvent();
- const deviceFeatureStates = await db.DeviceFeatureStateAggregate.findAll({
- raw: true,
- attributes: ['value', 'created_at'],
- order: [['created_at', 'ASC']],
- });
- // daily + hourly + monthly
- expect(deviceFeatureStates.length).to.equal(3000 + 3000 + 100);
- });
- it('should run the hourly aggregate task with failed job but not crash', async () => {
- await insertStates(true);
- const variable = {
- // we modify the retention policy to take the last 1000 days (it'll cover this last year)
- getValue: fake.resolves('1000'),
- };
- const job = new Job(event);
- job.wrapper = () => {
- return async () => {
- throw new Error('something failed');
- };
- };
- const device = new Device(event, {}, {}, {}, {}, variable, job);
- await device.onHourlyDeviceAggregateEvent();
- // if it doesn't crash, it worked
- });
-});
diff --git a/server/test/lib/device/device.destroy.test.js b/server/test/lib/device/device.destroy.test.js
index 70b4a91f9e..903ed7c1b4 100644
--- a/server/test/lib/device/device.destroy.test.js
+++ b/server/test/lib/device/device.destroy.test.js
@@ -1,5 +1,5 @@
const EventEmitter = require('events');
-const { assert } = require('chai');
+const { assert, expect } = require('chai');
const { fake, assert: sinonAssert } = require('sinon');
const Device = require('../../../lib/device');
const StateManager = require('../../../lib/state');
@@ -12,16 +12,22 @@ const event = new EventEmitter();
const job = new Job(event);
describe('Device.destroy', () => {
- it('should destroy device', async () => {
+ it('should destroy device and states', async () => {
const stateManager = new StateManager(event);
const serviceManager = new ServiceManager({}, stateManager);
const device = new Device(event, {}, stateManager, serviceManager, {}, {}, job);
+ await db.duckDbInsertState('ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4', 1);
device.devicesByPollFrequency[60000] = [
{
selector: 'test-device',
},
];
await device.destroy('test-device');
+ const res = await db.duckDbReadConnectionAllAsync(
+ 'SELECT * FROM t_device_feature_state WHERE device_feature_id = ?',
+ 'ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4',
+ );
+ expect(res).to.have.lengthOf(0);
});
it('should destroy device that has too much states', async () => {
const eventFake = {
diff --git a/server/test/lib/device/device.getDeviceFeaturesAggregates.test.js b/server/test/lib/device/device.getDeviceFeaturesAggregates.test.js
index b2daef1e8e..5802351d96 100644
--- a/server/test/lib/device/device.getDeviceFeaturesAggregates.test.js
+++ b/server/test/lib/device/device.getDeviceFeaturesAggregates.test.js
@@ -1,7 +1,6 @@
const EventEmitter = require('events');
const { expect, assert } = require('chai');
const sinon = require('sinon');
-const uuid = require('uuid');
const { fake } = require('sinon');
const db = require('../../../models');
const Device = require('../../../lib/device');
@@ -11,7 +10,6 @@ const event = new EventEmitter();
const job = new Job(event);
const insertStates = async (intervalInMinutes) => {
- const queryInterface = db.sequelize.getQueryInterface();
const deviceFeatureStateToInsert = [];
const now = new Date();
const statesToInsert = 2000;
@@ -19,14 +17,11 @@ const insertStates = async (intervalInMinutes) => {
const startAt = new Date(now.getTime() - intervalInMinutes * 60 * 1000);
const date = new Date(startAt.getTime() + ((intervalInMinutes * 60 * 1000) / statesToInsert) * i);
deviceFeatureStateToInsert.push({
- id: uuid.v4(),
- device_feature_id: 'ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4',
value: i,
created_at: date,
- updated_at: date,
});
}
- await queryInterface.bulkInsert('t_device_feature_state', deviceFeatureStateToInsert);
+ await db.duckDbBatchInsertState('ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4', deviceFeatureStateToInsert);
};
describe('Device.getDeviceFeaturesAggregates', function Describe() {
@@ -34,17 +29,7 @@ describe('Device.getDeviceFeaturesAggregates', function Describe() {
let clock;
beforeEach(async () => {
- const queryInterface = db.sequelize.getQueryInterface();
- await queryInterface.bulkDelete('t_device_feature_state');
- await queryInterface.bulkDelete('t_device_feature_state_aggregate');
- await db.DeviceFeature.update(
- {
- last_monthly_aggregate: null,
- last_daily_aggregate: null,
- last_hourly_aggregate: null,
- },
- { where: {} },
- );
+ await db.duckDbWriteConnectionAllAsync('DELETE FROM t_device_feature_state');
clock = sinon.useFakeTimers({
now: 1635131280000,
@@ -65,7 +50,6 @@ describe('Device.getDeviceFeaturesAggregates', function Describe() {
}),
};
const deviceInstance = new Device(event, {}, stateManager, {}, {}, variable, job);
- await deviceInstance.calculateAggregate('hourly');
const { values, device, deviceFeature } = await deviceInstance.getDeviceFeaturesAggregates(
'test-device-feature',
60,
@@ -87,7 +71,6 @@ describe('Device.getDeviceFeaturesAggregates', function Describe() {
}),
};
const device = new Device(event, {}, stateManager, {}, {}, variable, job);
- await device.calculateAggregate('hourly');
const { values } = await device.getDeviceFeaturesAggregates('test-device-feature', 24 * 60, 100);
expect(values).to.have.lengthOf(100);
});
@@ -103,9 +86,8 @@ describe('Device.getDeviceFeaturesAggregates', function Describe() {
}),
};
const device = new Device(event, {}, stateManager, {}, {}, variable, job);
- await device.calculateAggregate('hourly');
const { values } = await device.getDeviceFeaturesAggregates('test-device-feature', 3 * 24 * 60, 100);
- expect(values).to.have.lengthOf(72);
+ expect(values).to.have.lengthOf(100);
});
it('should return last month states', async () => {
await insertStates(2 * 30 * 24 * 60);
@@ -119,10 +101,8 @@ describe('Device.getDeviceFeaturesAggregates', function Describe() {
}),
};
const device = new Device(event, {}, stateManager, {}, {}, variable, job);
- await device.calculateAggregate('hourly');
- await device.calculateAggregate('daily');
const { values } = await device.getDeviceFeaturesAggregates('test-device-feature', 30 * 24 * 60, 100);
- expect(values).to.have.lengthOf(30);
+ expect(values).to.have.lengthOf(100);
});
it('should return last year states', async () => {
await insertStates(2 * 365 * 24 * 60);
@@ -136,9 +116,6 @@ describe('Device.getDeviceFeaturesAggregates', function Describe() {
}),
};
const device = new Device(event, {}, stateManager, {}, {}, variable, job);
- await device.calculateAggregate('hourly');
- await device.calculateAggregate('daily');
- await device.calculateAggregate('monthly');
const { values } = await device.getDeviceFeaturesAggregates('test-device-feature', 365 * 24 * 60, 100);
expect(values).to.have.lengthOf(100);
});
diff --git a/server/test/lib/device/device.getDeviceFeaturesAggregatesMulti.test.js b/server/test/lib/device/device.getDeviceFeaturesAggregatesMulti.test.js
index 34227090f2..27788e2b39 100644
--- a/server/test/lib/device/device.getDeviceFeaturesAggregatesMulti.test.js
+++ b/server/test/lib/device/device.getDeviceFeaturesAggregatesMulti.test.js
@@ -1,6 +1,5 @@
const EventEmitter = require('events');
const { expect } = require('chai');
-const uuid = require('uuid');
const { fake } = require('sinon');
const db = require('../../../models');
const Device = require('../../../lib/device');
@@ -10,7 +9,6 @@ const event = new EventEmitter();
const job = new Job(event);
const insertStates = async (intervalInMinutes) => {
- const queryInterface = db.sequelize.getQueryInterface();
const deviceFeatureStateToInsert = [];
const now = new Date();
const statesToInsert = 2000;
@@ -18,14 +16,11 @@ const insertStates = async (intervalInMinutes) => {
const startAt = new Date(now.getTime() - intervalInMinutes * 60 * 1000);
const date = new Date(startAt.getTime() + ((intervalInMinutes * 60 * 1000) / statesToInsert) * i);
deviceFeatureStateToInsert.push({
- id: uuid.v4(),
- device_feature_id: 'ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4',
value: i,
created_at: date,
- updated_at: date,
});
}
- await queryInterface.bulkInsert('t_device_feature_state', deviceFeatureStateToInsert);
+ await db.duckDbBatchInsertState('ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4', deviceFeatureStateToInsert);
};
describe('Device.getDeviceFeaturesAggregatesMulti', function Describe() {
@@ -55,7 +50,6 @@ describe('Device.getDeviceFeaturesAggregatesMulti', function Describe() {
}),
};
const device = new Device(event, {}, stateManager, {}, {}, variable, job);
- await device.calculateAggregate('hourly');
const response = await device.getDeviceFeaturesAggregatesMulti(['test-device-feature'], 60, 100);
expect(response).to.be.instanceOf(Array);
const { values } = response[0];
diff --git a/server/test/lib/device/device.getDuckDbMigrationState.test.js b/server/test/lib/device/device.getDuckDbMigrationState.test.js
new file mode 100644
index 0000000000..a8a5eea79b
--- /dev/null
+++ b/server/test/lib/device/device.getDuckDbMigrationState.test.js
@@ -0,0 +1,38 @@
+const EventEmitter = require('events');
+const { expect } = require('chai');
+const { fake } = require('sinon');
+const Device = require('../../../lib/device');
+const StateManager = require('../../../lib/state');
+const Job = require('../../../lib/job');
+
+const event = new EventEmitter();
+const job = new Job(event);
+
+describe('Device.getDuckDbMigrationState', () => {
+ it('should return migration not done', async () => {
+ const stateManager = new StateManager(event);
+ const variable = {
+ getValue: fake.resolves(null),
+ };
+ const device = new Device(event, {}, stateManager, {}, {}, variable, job);
+ const migrationState = await device.getDuckDbMigrationState();
+ expect(migrationState).to.deep.equal({
+ is_duck_db_migrated: false,
+ duck_db_device_count: 0,
+ sqlite_db_device_state_count: 0,
+ });
+ });
+ it('should return migration done', async () => {
+ const stateManager = new StateManager(event);
+ const variable = {
+ getValue: fake.resolves('true'),
+ };
+ const device = new Device(event, {}, stateManager, {}, {}, variable, job);
+ const migrationState = await device.getDuckDbMigrationState();
+ expect(migrationState).to.deep.equal({
+ is_duck_db_migrated: true,
+ duck_db_device_count: 0,
+ sqlite_db_device_state_count: 0,
+ });
+ });
+});
diff --git a/server/test/lib/device/device.init.test.js b/server/test/lib/device/device.init.test.js
index c06c7b8ff6..5d577196b1 100644
--- a/server/test/lib/device/device.init.test.js
+++ b/server/test/lib/device/device.init.test.js
@@ -1,7 +1,6 @@
const { assert, fake } = require('sinon');
const Device = require('../../../lib/device');
const StateManager = require('../../../lib/state');
-const { EVENTS } = require('../../../utils/constants');
const Job = require('../../../lib/job');
const event = {
@@ -21,8 +20,9 @@ describe('Device.init', () => {
const stateManager = new StateManager(event);
const job = new Job(event);
const device = new Device(event, {}, stateManager, service, {}, {}, job, brain);
+ device.migrateFromSQLiteToDuckDb = fake.returns(null);
- await device.init(true);
- assert.calledWith(event.emit, EVENTS.DEVICE.CALCULATE_HOURLY_AGGREGATE);
+ await device.init();
+ assert.called(device.migrateFromSQLiteToDuckDb);
});
});
diff --git a/server/test/lib/device/device.migrateFromSQLiteToDuckDb.test.js b/server/test/lib/device/device.migrateFromSQLiteToDuckDb.test.js
new file mode 100644
index 0000000000..ca1d1b3366
--- /dev/null
+++ b/server/test/lib/device/device.migrateFromSQLiteToDuckDb.test.js
@@ -0,0 +1,78 @@
+const { fake } = require('sinon');
+const { expect } = require('chai');
+const uuid = require('uuid');
+
+const db = require('../../../models');
+const Device = require('../../../lib/device');
+const StateManager = require('../../../lib/state');
+const Variable = require('../../../lib/variable');
+const { SYSTEM_VARIABLE_NAMES } = require('../../../utils/constants');
+
+const event = {
+ emit: fake.returns(null),
+ on: fake.returns(null),
+};
+
+const brain = {
+ addNamedEntity: fake.returns(null),
+};
+const service = {
+ getService: () => {},
+};
+
+const insertStates = async () => {
+ const queryInterface = db.sequelize.getQueryInterface();
+ const deviceFeatureStateToInsert = [];
+ const now = new Date();
+ for (let i = 1; i <= 1000; i += 1) {
+ const date = new Date(now.getFullYear() - 1, 10, i).toISOString();
+ deviceFeatureStateToInsert.push({
+ id: uuid.v4(),
+ device_feature_id: 'ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4',
+ value: i,
+ created_at: date,
+ updated_at: date,
+ });
+ }
+ await queryInterface.bulkInsert('t_device_feature_state', deviceFeatureStateToInsert);
+};
+
+describe('Device.migrateFromSQLiteToDuckDb', () => {
+ const variable = new Variable(event);
+ beforeEach(async () => {
+ await insertStates();
+ await variable.destroy(SYSTEM_VARIABLE_NAMES.DUCKDB_MIGRATED);
+ });
+ it('should migrate with success', async () => {
+ const stateManager = new StateManager(event);
+ const job = {
+ updateProgress: fake.resolves(null),
+ wrapper: (jobType, func) => func,
+ };
+ const device = new Device(event, {}, stateManager, service, {}, variable, job, brain);
+ await device.migrateFromSQLiteToDuckDb('2997ec9f-7a3e-4083-a183-f8b9b15d5bec', 500);
+ const res = await db.duckDbReadConnectionAllAsync(
+ 'SELECT COUNT(*) as nb_states FROM t_device_feature_state WHERE device_feature_id = $1;',
+ ['ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4'],
+ );
+ expect(res).to.deep.equal([{ nb_states: 1000n }]);
+ // Second call will not migrate (already migrated)
+ await device.migrateFromSQLiteToDuckDb('2997ec9f-7a3e-4083-a183-f8b9b15d5bec', 500);
+ });
+ it('should migrate 2 times if the first time was interrupted', async () => {
+ const stateManager = new StateManager(event);
+ const job = {
+ updateProgress: fake.resolves(null),
+ wrapper: (jobType, func) => func,
+ };
+ const device = new Device(event, {}, stateManager, service, {}, variable, job, brain);
+ await device.migrateFromSQLiteToDuckDb('2997ec9f-7a3e-4083-a183-f8b9b15d5bec', 500);
+ await variable.destroy(SYSTEM_VARIABLE_NAMES.DUCKDB_MIGRATED);
+ await device.migrateFromSQLiteToDuckDb('2997ec9f-7a3e-4083-a183-f8b9b15d5bec', 500);
+ const res = await db.duckDbReadConnectionAllAsync(
+ 'SELECT COUNT(*) as nb_states FROM t_device_feature_state WHERE device_feature_id = $1;',
+ ['ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4'],
+ );
+ expect(res).to.deep.equal([{ nb_states: 1000n }]);
+ });
+});
diff --git a/server/test/lib/device/device.purgeAggregateStates.test.js b/server/test/lib/device/device.purgeAggregateStates.test.js
deleted file mode 100644
index 4fcb2a4d06..0000000000
--- a/server/test/lib/device/device.purgeAggregateStates.test.js
+++ /dev/null
@@ -1,54 +0,0 @@
-const { expect } = require('chai');
-const EventEmitter = require('events');
-const { fake } = require('sinon');
-
-const Device = require('../../../lib/device');
-
-const StateManager = require('../../../lib/state');
-const Job = require('../../../lib/job');
-
-const event = new EventEmitter();
-const job = new Job(event);
-
-describe('Device.purgeAggregateStates', () => {
- it('should purgeAggregateStates', async () => {
- const variable = {
- getValue: fake.resolves(30),
- };
- const stateManager = new StateManager(event);
- const service = {};
- const device = new Device(event, {}, stateManager, service, {}, variable, job);
- const devicePurged = await device.purgeAggregateStates();
- expect(devicePurged).to.equal(true);
- });
- it('should not purgeAggregateStates, invalid date', async () => {
- const variable = {
- getValue: fake.resolves('NOT A DATE'),
- };
- const stateManager = new StateManager(event);
- const service = {};
- const device = new Device(event, {}, stateManager, service, {}, variable, job);
- const devicePurged = await device.purgeAggregateStates();
- expect(devicePurged).to.equal(false);
- });
- it('should not purgeAggregateStates, null', async () => {
- const variable = {
- getValue: fake.resolves(null),
- };
- const stateManager = new StateManager(event);
- const service = {};
- const device = new Device(event, {}, stateManager, service, {}, variable, job);
- const devicePurged = await device.purgeAggregateStates();
- expect(devicePurged).to.equal(false);
- });
- it('should not purgeAggregateStates, date = -1', async () => {
- const variable = {
- getValue: fake.resolves('-1'),
- };
- const stateManager = new StateManager(event);
- const service = {};
- const device = new Device(event, {}, stateManager, service, {}, variable, job);
- const devicePurged = await device.purgeAggregateStates();
- expect(devicePurged).to.equal(false);
- });
-});
diff --git a/server/test/lib/device/device.purgeAllSqliteStates.test.js b/server/test/lib/device/device.purgeAllSqliteStates.test.js
new file mode 100644
index 0000000000..d9cac1ce47
--- /dev/null
+++ b/server/test/lib/device/device.purgeAllSqliteStates.test.js
@@ -0,0 +1,65 @@
+const { expect } = require('chai');
+const EventEmitter = require('events');
+const { fake } = require('sinon');
+const uuid = require('uuid');
+
+const db = require('../../../models');
+const Device = require('../../../lib/device');
+
+const StateManager = require('../../../lib/state');
+
+const event = new EventEmitter();
+
+describe('Device', () => {
+ beforeEach(async () => {
+ const queryInterface = db.sequelize.getQueryInterface();
+ const deviceFeatureStateToInsert = [];
+ for (let i = 1; i <= 110; i += 1) {
+ const date = new Date();
+ deviceFeatureStateToInsert.push({
+ id: uuid.v4(),
+ device_feature_id: 'ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4',
+ value: i,
+ created_at: date,
+ updated_at: date,
+ });
+ }
+ await queryInterface.bulkInsert('t_device_feature_state', deviceFeatureStateToInsert);
+ await db.DeviceFeatureStateAggregate.create({
+ type: 'daily',
+ device_feature_id: 'ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4',
+ value: 12,
+ });
+ await db.DeviceFeatureStateAggregate.create({
+ type: 'hourly',
+ device_feature_id: 'ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4',
+ value: 12,
+ });
+ await db.DeviceFeatureStateAggregate.create({
+ type: 'monthly',
+ device_feature_id: 'ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4',
+ value: 12,
+ });
+ });
+ it('should purge all sqlite states', async () => {
+ const variable = {
+ getValue: fake.resolves(30),
+ };
+ const stateManager = new StateManager(event);
+ const service = {};
+ const job = {
+ updateProgress: fake.resolves(null),
+ wrapper: (type, func) => func,
+ };
+ const device = new Device(event, {}, stateManager, service, {}, variable, job);
+ const devicePurgedPromise = device.purgeAllSqliteStates('632c6d92-a79a-4a38-bf5b-a2024721c101');
+ const emptyRes = await device.purgeAllSqliteStates('632c6d92-a79a-4a38-bf5b-a2024721c101');
+ const devicePurged = await devicePurgedPromise;
+ expect(devicePurged).to.deep.equal({
+ numberOfDeviceFeatureStateAggregateToDelete: 3,
+ numberOfDeviceFeatureStateToDelete: 110,
+ });
+ // should not start a new purge when a purge is running
+ expect(emptyRes).to.equal(null);
+ });
+});
diff --git a/server/test/lib/device/device.purgeStates.test.js b/server/test/lib/device/device.purgeStates.test.js
index 3f300c7f92..e97c512d83 100644
--- a/server/test/lib/device/device.purgeStates.test.js
+++ b/server/test/lib/device/device.purgeStates.test.js
@@ -1,6 +1,7 @@
const { expect } = require('chai');
const EventEmitter = require('events');
const { fake } = require('sinon');
+const db = require('../../../models');
const Device = require('../../../lib/device');
@@ -15,11 +16,29 @@ describe('Device', () => {
const variable = {
getValue: fake.resolves(30),
};
+ const currentDate = new Date();
+ const fortyDaysAgo = new Date(currentDate);
+ fortyDaysAgo.setDate(currentDate.getDate() - 40);
+ await db.duckDbBatchInsertState('ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4', [
+ {
+ value: 1,
+ created_at: fortyDaysAgo,
+ },
+ {
+ value: 10,
+ created_at: currentDate,
+ },
+ ]);
const stateManager = new StateManager(event);
const service = {};
const device = new Device(event, {}, stateManager, service, {}, variable, job);
const devicePurged = await device.purgeStates();
expect(devicePurged).to.equal(true);
+ const res = await db.duckDbReadConnectionAllAsync(
+ 'SELECT value FROM t_device_feature_state WHERE device_feature_id = ?',
+ 'ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4',
+ );
+ expect(res).to.deep.equal([{ value: 10 }]);
});
it('should not purgeStates, invalid date', async () => {
const variable = {
diff --git a/server/test/lib/device/device.saveHistoricalState.test.js b/server/test/lib/device/device.saveHistoricalState.test.js
index 9507beed19..7c89a393f3 100644
--- a/server/test/lib/device/device.saveHistoricalState.test.js
+++ b/server/test/lib/device/device.saveHistoricalState.test.js
@@ -67,12 +67,12 @@ describe('Device.saveHistoricalState', () => {
// Should not save 2 times the same event
await device.saveHistoricalState(deviceFeature, 12, newDate);
await device.saveHistoricalState(deviceFeature, 12, newDate);
- const deviceStates = await db.DeviceFeatureState.findAll({
- where: {
- device_feature_id: 'ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4',
- },
- raw: true,
- });
+ const deviceStates = await db.duckDbReadConnectionAllAsync(
+ `
+ SELECT * FROM t_device_feature_state WHERE device_feature_id = ?
+ `,
+ 'ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4',
+ );
expect(deviceStates).to.have.lengthOf(1);
});
it('should save old state and keep history, and update aggregate', async () => {
diff --git a/server/test/lib/gateway/encoded-gladys-db-and-duckdb-backup.tar.gz.enc b/server/test/lib/gateway/encoded-gladys-db-and-duckdb-backup.tar.gz.enc
new file mode 100644
index 0000000000..ee1931130a
Binary files /dev/null and b/server/test/lib/gateway/encoded-gladys-db-and-duckdb-backup.tar.gz.enc differ
diff --git a/server/test/lib/gateway/encoded-gladys-db-backup.db.gz.enc b/server/test/lib/gateway/encoded-old-gladys-db-backup.db.gz.enc
similarity index 100%
rename from server/test/lib/gateway/encoded-gladys-db-backup.db.gz.enc
rename to server/test/lib/gateway/encoded-old-gladys-db-backup.db.gz.enc
diff --git a/server/test/lib/gateway/gateway.backup.test.js b/server/test/lib/gateway/gateway.backup.test.js
index a3f2aeb767..257efe66e7 100644
--- a/server/test/lib/gateway/gateway.backup.test.js
+++ b/server/test/lib/gateway/gateway.backup.test.js
@@ -33,7 +33,7 @@ describe('gateway.backup', async function describe() {
updateProgress: fake.resolves({}),
};
- variable.getValue = fake.resolves('variable');
+ variable.getValue = fake.resolves('key');
variable.setValue = fake.resolves(null);
event.on = fake.returns(null);
@@ -125,6 +125,7 @@ describe('gateway.backup', async function describe() {
value: 1,
}),
);
+ promisesDevices.push(db.duckDbInsertState('ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4', 1));
}
// start backup
promises.push(gateway.backup());
@@ -136,6 +137,7 @@ describe('gateway.backup', async function describe() {
value: 1,
}),
);
+ promisesDevices.push(db.duckDbInsertState('ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4', 1));
}
await Promise.all(promisesDevices);
await Promise.all(promises);
diff --git a/server/test/lib/gateway/gateway.downloadBackup.test.js b/server/test/lib/gateway/gateway.downloadBackup.test.js
index 0aff79f922..23901d3838 100644
--- a/server/test/lib/gateway/gateway.downloadBackup.test.js
+++ b/server/test/lib/gateway/gateway.downloadBackup.test.js
@@ -71,13 +71,26 @@ describe('gateway.downloadBackup', () => {
assert.notCalled(event.emit);
});
- it('should download a backup', async () => {
- const encryptedBackupFilePath = path.join(__dirname, 'encoded-gladys-db-backup.db.gz.enc');
- const { backupFilePath } = await gateway.downloadBackup(encryptedBackupFilePath);
+ it('should download a backup (new style, sqlite + parquet)', async () => {
+ const encryptedBackupFilePath = path.join(__dirname, 'encoded-gladys-db-and-duckdb-backup.tar.gz.enc');
+ await gateway.downloadBackup(encryptedBackupFilePath);
assert.calledOnceWithExactly(event.emit, EVENTS.WEBSOCKET.SEND_ALL, {
type: WEBSOCKET_MESSAGE_TYPES.BACKUP.DOWNLOADED,
payload: {
- backupFilePath,
+ duckDbBackupFolderPath: 'gladys-backups/restore/gladys-db-backup_2024-6-29-13-47-50_parquet_folder',
+ sqliteBackupFilePath: 'gladys-backups/restore/gladys-db-backup-2024-6-29-13-47-50.db',
+ },
+ });
+ });
+
+ it('should download a backup (old style, sqlite)', async () => {
+ const encryptedBackupFilePath = path.join(__dirname, 'encoded-old-gladys-db-backup.db.gz.enc');
+ await gateway.downloadBackup(encryptedBackupFilePath);
+ assert.calledOnceWithExactly(event.emit, EVENTS.WEBSOCKET.SEND_ALL, {
+ type: WEBSOCKET_MESSAGE_TYPES.BACKUP.DOWNLOADED,
+ payload: {
+ duckDbBackupFolderPath: null,
+ sqliteBackupFilePath: 'gladys-backups/restore/encoded-old-gladys-db-backup.db.gz.db',
},
});
});
diff --git a/server/test/lib/gateway/gateway.restoreBackup.test.js b/server/test/lib/gateway/gateway.restoreBackup.test.js
index fe20caf765..1b7e4fdb9c 100644
--- a/server/test/lib/gateway/gateway.restoreBackup.test.js
+++ b/server/test/lib/gateway/gateway.restoreBackup.test.js
@@ -57,7 +57,14 @@ describe('gateway.restoreBackup', () => {
sinon.reset();
});
- it('should restore a backup', async () => {
+ it('should restore a new style backup (sqlite + duckDB)', async () => {
+ const backupFilePath = path.join(__dirname, 'real-gladys-db-backup.db.gz.dbfile');
+ const parquetFolderPath = path.join(__dirname, 'gladys_backup_parquet_folder');
+ await gateway.restoreBackup(backupFilePath, parquetFolderPath);
+ assert.calledOnceWithExactly(sequelize.close);
+ });
+
+ it('should restore a old style backup (just sqlite)', async () => {
const backupFilePath = path.join(__dirname, 'real-gladys-db-backup.db.gz.dbfile');
await gateway.restoreBackup(backupFilePath);
assert.calledOnceWithExactly(sequelize.close);
diff --git a/server/test/lib/gateway/gateway.restoreBackupEvent.test.js b/server/test/lib/gateway/gateway.restoreBackupEvent.test.js
index 293ac22fdd..bc36b1eb12 100644
--- a/server/test/lib/gateway/gateway.restoreBackupEvent.test.js
+++ b/server/test/lib/gateway/gateway.restoreBackupEvent.test.js
@@ -4,7 +4,8 @@ const proxyquire = require('proxyquire').noCallThru();
const path = require('path');
const GladysGatewayClientMock = require('./GladysGatewayClientMock.test');
-
+const db = require('../../../models');
+const { cleanDb } = require('../../helpers/db.test');
const getConfig = require('../../../utils/getConfig');
const { fake, assert } = sinon;
@@ -55,12 +56,28 @@ describe('gateway.restoreBackupEvent', () => {
gateway = new Gateway(variable, event, system, sequelize, config, {}, {}, {}, job, scheduler);
});
- afterEach(() => {
+ afterEach(async () => {
sinon.reset();
+ await db.umzug.up();
+ await cleanDb();
+ });
+
+ it('should download and restore new backup (sqlite + parquet), then shutdown', async () => {
+ const encryptedBackupFilePath = path.join(__dirname, 'encoded-gladys-db-and-duckdb-backup.tar.gz.enc');
+ const restoreBackupEvent = {
+ file_url: encryptedBackupFilePath,
+ };
+
+ await gateway.restoreBackupEvent(restoreBackupEvent);
+
+ expect(gateway.restoreErrored).equals(false);
+ expect(gateway.restoreInProgress).equals(true);
+
+ assert.calledOnceWithExactly(system.shutdown);
});
- it('should download and restore backup, then shutdown', async () => {
- const encryptedBackupFilePath = path.join(__dirname, 'encoded-gladys-db-backup.db.gz.enc');
+ it('should download and restore old sqlite backup, then shutdown', async () => {
+ const encryptedBackupFilePath = path.join(__dirname, 'encoded-old-gladys-db-backup.db.gz.enc');
const restoreBackupEvent = {
file_url: encryptedBackupFilePath,
};
diff --git a/server/test/lib/gateway/gladys_backup_parquet_folder/load.sql b/server/test/lib/gateway/gladys_backup_parquet_folder/load.sql
new file mode 100644
index 0000000000..d4883c0840
--- /dev/null
+++ b/server/test/lib/gateway/gladys_backup_parquet_folder/load.sql
@@ -0,0 +1 @@
+COPY t_device_feature_state FROM 'gladys-backups/gladys-db-backup_2024-6-28-14-57-29_parquet_folder/t_device_feature_state.parquet' (FORMAT 'parquet', COMPRESSION 'GZIP');
diff --git a/server/test/lib/gateway/gladys_backup_parquet_folder/schema.sql b/server/test/lib/gateway/gladys_backup_parquet_folder/schema.sql
new file mode 100644
index 0000000000..dd5284ec2e
--- /dev/null
+++ b/server/test/lib/gateway/gladys_backup_parquet_folder/schema.sql
@@ -0,0 +1,8 @@
+
+
+
+CREATE TABLE t_device_feature_state(device_feature_id UUID, "value" DOUBLE, created_at TIMESTAMP WITH TIME ZONE);
+
+
+
+
diff --git a/server/test/lib/gateway/gladys_backup_parquet_folder/t_device_feature_state.parquet b/server/test/lib/gateway/gladys_backup_parquet_folder/t_device_feature_state.parquet
new file mode 100644
index 0000000000..83eb743619
Binary files /dev/null and b/server/test/lib/gateway/gladys_backup_parquet_folder/t_device_feature_state.parquet differ
diff --git a/server/test/lib/job/job.test.js b/server/test/lib/job/job.test.js
index 73e0b78d60..80645938f4 100644
--- a/server/test/lib/job/job.test.js
+++ b/server/test/lib/job/job.test.js
@@ -18,8 +18,8 @@ describe('Job', () => {
describe('job.create', () => {
const job = new Job(event);
it('should create a job', async () => {
- const newJob = await job.start(JOB_TYPES.DAILY_DEVICE_STATE_AGGREGATE);
- expect(newJob).to.have.property('type', JOB_TYPES.DAILY_DEVICE_STATE_AGGREGATE);
+ const newJob = await job.start(JOB_TYPES.GLADYS_GATEWAY_BACKUP);
+ expect(newJob).to.have.property('type', JOB_TYPES.GLADYS_GATEWAY_BACKUP);
expect(newJob).to.have.property('status', JOB_STATUS.IN_PROGRESS);
assert.calledWith(event.emit, EVENTS.WEBSOCKET.SEND_ALL, {
type: WEBSOCKET_MESSAGE_TYPES.JOB.NEW,
@@ -31,14 +31,14 @@ describe('Job', () => {
return chaiAssert.isRejected(promise, 'Validation error: Validation isIn on type failed');
});
it('should not create a job, invalid data', async () => {
- const promise = job.start(JOB_TYPES.DAILY_DEVICE_STATE_AGGREGATE, []);
+ const promise = job.start(JOB_TYPES.GLADYS_GATEWAY_BACKUP, []);
return chaiAssert.isRejected(promise, 'Validation error: "value" must be of type object');
});
});
describe('job.finish', () => {
const job = new Job(event);
it('should finish a job', async () => {
- const newJob = await job.start(JOB_TYPES.DAILY_DEVICE_STATE_AGGREGATE);
+ const newJob = await job.start(JOB_TYPES.GLADYS_GATEWAY_BACKUP);
const updatedJob = await job.finish(newJob.id, JOB_STATUS.SUCCESS, {});
assert.calledWith(event.emit, EVENTS.WEBSOCKET.SEND_ALL, {
type: WEBSOCKET_MESSAGE_TYPES.JOB.UPDATED,
@@ -53,7 +53,7 @@ describe('Job', () => {
describe('job.updateProgress', () => {
const job = new Job(event);
it('should update the progress a job', async () => {
- const newJob = await job.start(JOB_TYPES.DAILY_DEVICE_STATE_AGGREGATE);
+ const newJob = await job.start(JOB_TYPES.GLADYS_GATEWAY_BACKUP);
const updatedJob = await job.updateProgress(newJob.id, 50);
assert.calledWith(event.emit, EVENTS.WEBSOCKET.SEND_ALL, {
type: WEBSOCKET_MESSAGE_TYPES.JOB.UPDATED,
@@ -61,12 +61,12 @@ describe('Job', () => {
});
});
it('should not update the progress a job, invalid progress', async () => {
- const newJob = await job.start(JOB_TYPES.DAILY_DEVICE_STATE_AGGREGATE);
+ const newJob = await job.start(JOB_TYPES.GLADYS_GATEWAY_BACKUP);
const promise = job.updateProgress(newJob.id, 101);
return chaiAssert.isRejected(promise, 'Validation error: Validation max on progress failed');
});
it('should not update the progress a job, invalid progress', async () => {
- const newJob = await job.start(JOB_TYPES.DAILY_DEVICE_STATE_AGGREGATE);
+ const newJob = await job.start(JOB_TYPES.GLADYS_GATEWAY_BACKUP);
const promise = job.updateProgress(newJob.id, -1);
return chaiAssert.isRejected(promise, 'Validation error: Validation min on progress failed');
});
@@ -74,15 +74,24 @@ describe('Job', () => {
describe('job.get', () => {
const job = new Job(event);
it('should get all job', async () => {
- await job.start(JOB_TYPES.DAILY_DEVICE_STATE_AGGREGATE);
+ await job.start(JOB_TYPES.GLADYS_GATEWAY_BACKUP);
const jobs = await job.get();
expect(jobs).to.be.instanceOf(Array);
jobs.forEach((oneJob) => {
expect(oneJob).to.have.property('type');
});
});
+ it('should get gateway backup job only', async () => {
+ await job.start(JOB_TYPES.GLADYS_GATEWAY_BACKUP);
+ await job.start(JOB_TYPES.VACUUM);
+ const jobs = await job.get({ type: JOB_TYPES.GLADYS_GATEWAY_BACKUP });
+ expect(jobs).to.be.instanceOf(Array);
+ jobs.forEach((oneJob) => {
+ expect(oneJob).to.have.property('type', JOB_TYPES.GLADYS_GATEWAY_BACKUP);
+ });
+ });
it('should get 0 job', async () => {
- await job.start(JOB_TYPES.DAILY_DEVICE_STATE_AGGREGATE);
+ await job.start(JOB_TYPES.GLADYS_GATEWAY_BACKUP);
const jobs = await job.get({
take: 0,
});
@@ -93,7 +102,7 @@ describe('Job', () => {
describe('job.init', () => {
const job = new Job(event);
it('should init jobs and mark unfinished jobs as failed', async () => {
- const jobCreated = await job.start(JOB_TYPES.DAILY_DEVICE_STATE_AGGREGATE);
+ const jobCreated = await job.start(JOB_TYPES.GLADYS_GATEWAY_BACKUP);
await job.init();
const jobs = await job.get();
expect(jobs).to.be.instanceOf(Array);
@@ -102,7 +111,7 @@ describe('Job', () => {
.map((oneJob) => ({ type: oneJob.type, status: oneJob.status, data: oneJob.data }));
expect(jobsFiltered).to.deep.equal([
{
- type: 'daily-device-state-aggregate',
+ type: 'gladys-gateway-backup',
status: 'failed',
data: { error_type: 'purged-when-restarted' },
},
@@ -112,7 +121,7 @@ describe('Job', () => {
describe('job.purge', () => {
const job = new Job(event);
it('should purge old jobs', async () => {
- await job.start(JOB_TYPES.DAILY_DEVICE_STATE_AGGREGATE);
+ await job.start(JOB_TYPES.GLADYS_GATEWAY_BACKUP);
const dateInThePast = new Date(new Date().getTime() - 10 * 24 * 60 * 60 * 1000);
await db.Job.update({ created_at: dateInThePast }, { where: {} });
await job.purge();
@@ -123,7 +132,7 @@ describe('Job', () => {
describe('job.wrapper', () => {
const job = new Job(event);
it('should test wrapper', async () => {
- const wrapped = job.wrapper(JOB_TYPES.DAILY_DEVICE_STATE_AGGREGATE, () => {});
+ const wrapped = job.wrapper(JOB_TYPES.GLADYS_GATEWAY_BACKUP, () => {});
await wrapped();
const jobs = await job.get();
expect(jobs).to.be.instanceOf(Array);
@@ -131,7 +140,7 @@ describe('Job', () => {
expect(lastJob).to.have.property('status', JOB_STATUS.SUCCESS);
});
it('should test wrapper with failed job', async () => {
- const wrapped = job.wrapper(JOB_TYPES.DAILY_DEVICE_STATE_AGGREGATE, () => {
+ const wrapped = job.wrapper(JOB_TYPES.GLADYS_GATEWAY_BACKUP, () => {
throw new Error('failed');
});
try {
diff --git a/server/utils/constants.js b/server/utils/constants.js
index 7312489248..95eae8e42a 100644
--- a/server/utils/constants.js
+++ b/server/utils/constants.js
@@ -131,6 +131,7 @@ const SYSTEM_VARIABLE_NAMES = {
TIMEZONE: 'TIMEZONE',
DEVICE_BATTERY_LEVEL_WARNING_THRESHOLD: 'DEVICE_BATTERY_LEVEL_WARNING_THRESHOLD',
DEVICE_BATTERY_LEVEL_WARNING_ENABLED: 'DEVICE_BATTERY_LEVEL_WARNING_ENABLED',
+ DUCKDB_MIGRATED: 'DUCKDB_MIGRATED',
};
const EVENTS = {
@@ -158,6 +159,8 @@ const EVENTS = {
CALCULATE_HOURLY_AGGREGATE: 'device.calculate-hourly-aggregate',
PURGE_STATES_SINGLE_FEATURE: 'device.purge-states-single-feature',
CHECK_BATTERIES: 'device.check-batteries',
+ MIGRATE_FROM_SQLITE_TO_DUCKDB: 'device.migrate-from-sqlite-to-duckdb',
+ PURGE_ALL_SQLITE_STATES: 'device.purge-all-sqlite-states',
},
GATEWAY: {
CREATE_BACKUP: 'gateway.create-backup',
@@ -1064,9 +1067,11 @@ const JOB_TYPES = {
GLADYS_GATEWAY_BACKUP: 'gladys-gateway-backup',
DEVICE_STATES_PURGE_SINGLE_FEATURE: 'device-state-purge-single-feature',
DEVICE_STATES_PURGE: 'device-state-purge',
+ DEVICE_STATES_PURGE_ALL_SQLITE_STATES: 'device-state-purge-all-sqlite-states',
VACUUM: 'vacuum',
SERVICE_ZIGBEE2MQTT_BACKUP: 'service-zigbee2mqtt-backup',
SERVICE_NODE_RED_BACKUP: 'service-node-red-backup',
+ MIGRATE_SQLITE_TO_DUCKDB: 'migrate-sqlite-to-duckdb',
};
const JOB_STATUS = {
diff --git a/server/utils/date.js b/server/utils/date.js
new file mode 100644
index 0000000000..b87b6653e3
--- /dev/null
+++ b/server/utils/date.js
@@ -0,0 +1,28 @@
+const dayjs = require('dayjs');
+const utc = require('dayjs/plugin/utc');
+const timezone = require('dayjs/plugin/timezone');
+
+// Extend dayjs with the necessary plugins
+dayjs.extend(utc);
+dayjs.extend(timezone);
+
+/**
+ * @description Format a date in UTC string.
+ * @param {Date} date - Javascript date.
+ * @returns {string} - Return a formatted string, utc time.
+ * @example const dateStringUtc = formatDateInUTC(new Date());
+ */
+function formatDateInUTC(date) {
+ // Convert date to UTC
+ const dateInUTC = dayjs(date).utc();
+
+ // Format the date
+ const formattedDate = dateInUTC.format('YYYY-MM-DD HH:mm:ss.SSSSSS');
+
+ // Append the UTC offset
+ return `${formattedDate}+00`;
+}
+
+module.exports = {
+ formatDateInUTC,
+};