Purge states should purge duckDB states + remove purge state agregate…

…s + improve shutdown DB closing

front/src/actions/signup/signupSetPreferences.js

@@ -59,12 +59,6 @@ function createActions(store) {
         await state.httpClient.post(`/api/v1/variable/${SYSTEM_VARIABLE_NAMES.DEVICE_STATE_HISTORY_IN_DAYS}`, {
           value: state.signupSystemPreferences[SYSTEM_VARIABLE_NAMES.DEVICE_STATE_HISTORY_IN_DAYS]
         });
-        await state.httpClient.post(
-          `/api/v1/variable/${SYSTEM_VARIABLE_NAMES.DEVICE_AGGREGATE_STATE_HISTORY_IN_DAYS}`,
-          {
-            value: state.signupSystemPreferences[SYSTEM_VARIABLE_NAMES.DEVICE_STATE_HISTORY_IN_DAYS]
-          }
-        );
         store.setState({
           signupSaveSystemPreferences: RequestStatus.Success
         });

front/src/routes/settings/settings-system/SettingsSystemPage.jsx

@@ -5,7 +5,6 @@ import SettingsSystemContainers from './SettingsSystemContainers';
 import SettingsSystemOperations from './SettingsSystemOperations';
 import SettingsSystemTimezone from './SettingsSystemTimezone';
 import SettingsSystemKeepDeviceHistory from './SettingsSystemKeepDeviceHistory';
-import SettingsSystemKeepAggregatedStates from './SettingsSystemKeepAggregatedStates';
 import SettingsSystemTimeExpiryState from './SettingsSystemTimeExpiryState';
 import SettingsSystemDatabaseCleaning from './SettingsSystemDatabaseCleaning';
 import SettingsSystemDuckDbMigration from './SettingsSystemDuckDbMigration';
@@ -89,7 +88,6 @@ const SystemPage = ({ children, ...props }) => (
         <SettingsSystemOperations />
         <SettingsSystemDuckDbMigration />
         <SettingsSystemKeepDeviceHistory />
-        <SettingsSystemKeepAggregatedStates />
         <SettingsSystemTimeExpiryState />
       </div>
       <div class="col-lg-6">

server/index.js

@@ -21,26 +21,35 @@ process.on('uncaughtException', (error, promise) => {
   logger.error(error);
 });
 
-const shutdown = async (signal) => {
-  logger.info(`${signal} received.`);
-  // We give Gladys 10 seconds to properly shutdown, otherwise we do it
-  setTimeout(() => {
-    logger.info('Timeout to shutdown expired, forcing shut down.');
-    process.exit();
-  }, 10 * 1000);
-  logger.info('Closing database connections.');
+const closeSQLite = async () => {
   try {
     await db.sequelize.close();
+    logger.info('SQLite closed.');
   } catch (e) {
     logger.info('SQLite database is probably already closed');
     logger.warn(e);
   }
+};
+
+const closeDuckDB = async () => {
   try {
     await db.duckDb.close();
+    logger.info('DuckDB closed.');
   } catch (e) {
     logger.info('DuckDB database is probably already closed');
     logger.warn(e);
   }
+};
+
+const shutdown = async (signal) => {
+  logger.info(`${signal} received.`);
+  // We give Gladys 10 seconds to properly shutdown, otherwise we do it
+  setTimeout(() => {
+    logger.info('Timeout to shutdown expired, forcing shut down.');
+    process.exit();
+  }, 10 * 1000);
+  logger.info('Closing database connections.');
+  await Promise.all([closeSQLite(), closeDuckDB()]);
   process.exit();
 };
 

server/lib/device/device.onPurgeStatesEvent.js

@@ -8,7 +8,6 @@ const { JOB_TYPES } = require('../../utils/constants');
 async function onPurgeStatesEvent() {
   const purgeAllStates = this.job.wrapper(JOB_TYPES.DEVICE_STATES_PURGE, async () => {
     await this.purgeStates();
-    await this.purgeAggregateStates();
   });
   await purgeAllStates();
 }

server/lib/device/device.purgeStates.js

@@ -1,4 +1,3 @@
-const { Op } = require('sequelize');
 const db = require('../../models');
 const logger = require('../../utils/logger');
 const { SYSTEM_VARIABLE_NAMES } = require('../../utils/constants');
@@ -21,19 +20,19 @@ async function purgeStates() {
     logger.debug('Not purging device feature states, deviceStateHistoryInDays = -1');
     return Promise.resolve(false);
   }
-  const queryInterface = db.sequelize.getQueryInterface();
   const now = new Date().getTime();
   // all date before this timestamp will be removed
   const timstampLimit = now - deviceStateHistoryInDaysInt * 24 * 60 * 60 * 1000;
   const dateLimit = new Date(timstampLimit);
   logger.info(
     `Purging device feature states of the last ${deviceStateHistoryInDaysInt} days. States older than ${dateLimit} will be purged.`,
   );
-  await queryInterface.bulkDelete('t_device_feature_state', {
-    created_at: {
-      [Op.lte]: dateLimit,
-    },
-  });
+  await db.duckDbWriteConnectionAllAsync(
+    `
+    DELETE FROM t_device_feature_state WHERE created_at < ?
+  `,
+    dateLimit,
+  );
   return true;
 }
 

server/lib/device/index.js

@@ -20,7 +20,6 @@ const { getDeviceFeaturesAggregates } = require('./device.getDeviceFeaturesAggre
 const { getDeviceFeaturesAggregatesMulti } = require('./device.getDeviceFeaturesAggregatesMulti');
 const { onPurgeStatesEvent } = require('./device.onPurgeStatesEvent');
 const { purgeStates } = require('./device.purgeStates');
-const { purgeAggregateStates } = require('./device.purgeAggregateStates');
 const { purgeStatesByFeatureId } = require('./device.purgeStatesByFeatureId');
 const { poll } = require('./device.poll');
 const { pollAll } = require('./device.pollAll');
@@ -114,7 +113,6 @@ DeviceManager.prototype.getDeviceFeaturesAggregates = getDeviceFeaturesAggregate
 DeviceManager.prototype.getDeviceFeaturesAggregatesMulti = getDeviceFeaturesAggregatesMulti;
 DeviceManager.prototype.onPurgeStatesEvent = onPurgeStatesEvent;
 DeviceManager.prototype.purgeStates = purgeStates;
-DeviceManager.prototype.purgeAggregateStates = purgeAggregateStates;
 DeviceManager.prototype.purgeStatesByFeatureId = purgeStatesByFeatureId;
 DeviceManager.prototype.poll = poll;
 DeviceManager.prototype.pollAll = pollAll;

server/test/lib/device/device.purgeStates.test.js

@@ -1,6 +1,7 @@
 const { expect } = require('chai');
 const EventEmitter = require('events');
 const { fake } = require('sinon');
+const db = require('../../../models');
 
 const Device = require('../../../lib/device');
 
@@ -15,11 +16,29 @@ describe('Device', () => {
     const variable = {
       getValue: fake.resolves(30),
     };
+    const currentDate = new Date();
+    const fortyDaysAgo = new Date(currentDate);
+    fortyDaysAgo.setDate(currentDate.getDate() - 40);
+    await db.duckDbBatchInsertState('ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4', [
+      {
+        value: 1,
+        created_at: fortyDaysAgo,
+      },
+      {
+        value: 10,
+        created_at: currentDate,
+      },
+    ]);
     const stateManager = new StateManager(event);
     const service = {};
     const device = new Device(event, {}, stateManager, service, {}, variable, job);
     const devicePurged = await device.purgeStates();
     expect(devicePurged).to.equal(true);
+    const res = await db.duckDbReadConnectionAllAsync(
+      'SELECT value FROM t_device_feature_state WHERE device_feature_id = ?',
+      'ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4',
+    );
+    expect(res).to.deep.equal([{ value: 10 }]);
   });
   it('should not purgeStates, invalid date', async () => {
     const variable = {