diff --git a/x-pack/plugins/security_solution/server/config.mock.ts b/x-pack/plugins/security_solution/server/config.mock.ts index 1d0d31e9387e2..5fb3dc7b3b48d 100644 --- a/x-pack/plugins/security_solution/server/config.mock.ts +++ b/x-pack/plugins/security_solution/server/config.mock.ts @@ -10,6 +10,7 @@ import type { ExperimentalFeatures } from '../common/experimental_features'; import { parseExperimentalConfigValue } from '../common/experimental_features'; import { getDefaultConfigSettings } from '../common/config_settings'; import type { ConfigType } from './config'; +import { duration } from 'moment'; export const createMockConfig = (): ConfigType => { const enableExperimental: Array = ['responseActionUploadEnabled']; @@ -45,6 +46,8 @@ export const createMockConfig = (): ConfigType => { }, }, entityStore: { + frequency: duration('1m'), + syncDelay: duration('5m'), developer: { pipelineDebugMode: false, }, diff --git a/x-pack/plugins/security_solution/server/config.ts b/x-pack/plugins/security_solution/server/config.ts index 1265aa4c25749..240e452cd44bc 100644 --- a/x-pack/plugins/security_solution/server/config.ts +++ b/x-pack/plugins/security_solution/server/config.ts @@ -176,6 +176,8 @@ export const configSchema = schema.object({ }), }), entityStore: schema.object({ + syncDelay: schema.duration({ defaultValue: '60s' }), + frequency: schema.duration({ defaultValue: '60s' }), developer: schema.object({ pipelineDebugMode: schema.boolean({ defaultValue: false }), }), diff --git a/x-pack/plugins/security_solution/server/endpoint/mocks/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/mocks/mocks.ts index 91a2bc40454b9..03c2e7e857e10 100644 --- a/x-pack/plugins/security_solution/server/endpoint/mocks/mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/mocks/mocks.ts @@ -76,6 +76,7 @@ import type { EndpointAuthz } from '../../../common/endpoint/types/authz'; import { createLicenseServiceMock } from '../../../common/license/mocks'; import { createFeatureUsageServiceMock } from '../services/feature_usage/mocks'; import { createProductFeaturesServiceMock } from '../../lib/product_features_service/mocks'; +import type { ConfigType } from '../../config'; /** * Creates a mocked EndpointAppContext. @@ -163,11 +164,15 @@ export const createMockEndpointAppContextServiceSetupContract = }; }; +type CreateMockEndpointAppContextServiceStartContractType = Omit< + DeeplyMockedKeys, + 'config' +> & { config: ConfigType }; // DeeplyMockedKeys doesn't support moment.Duration /** * Creates a mocked input contract for the `EndpointAppContextService#start()` method */ export const createMockEndpointAppContextServiceStartContract = - (): DeeplyMockedKeys => { + (): CreateMockEndpointAppContextServiceStartContractType => { const config = createMockConfig(); const logger = loggingSystemMock.create().get('mock_endpoint_app_context'); @@ -189,7 +194,7 @@ export const createMockEndpointAppContextServiceStartContract = securityMock.createMockAuthenticatedUser({ roles: ['superuser'] }) ); - const startContract: DeeplyMockedKeys = { + const startContract: CreateMockEndpointAppContextServiceStartContractType = { security, config, productFeaturesService: createProductFeaturesServiceMock( diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/constants.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/constants.ts index 796932d79b364..8b2e802b17b6d 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/constants.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/constants.ts @@ -9,8 +9,6 @@ import type { EngineStatus } from '../../../../common/api/entity_analytics'; export const DEFAULT_LOOKBACK_PERIOD = '24h'; -export const DEFAULT_INTERVAL = '30s'; - export const ENGINE_STATUS: Record, EngineStatus> = { INSTALLING: 'installing', STARTED: 'started', diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts index 858047952801d..733e85fd6ed55 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts @@ -15,6 +15,7 @@ import type { SortOrder } from '@elastic/elasticsearch/lib/api/types'; import type { EntityType } from '../../../../common/api/entity_analytics/entity_store/common.gen'; import type { DataViewsService } from '@kbn/data-views-plugin/common'; import type { AppClient } from '../../..'; +import type { EntityStoreConfig } from './types'; describe('EntityStoreDataClient', () => { const mockSavedObjectClient = savedObjectsClientMock.create(); @@ -29,6 +30,7 @@ describe('EntityStoreDataClient', () => { kibanaVersion: '9.0.0', dataViewsService: {} as DataViewsService, appClient: {} as AppClient, + config: {} as EntityStoreConfig, }); const defaultSearchParams = { diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts index 6b1d52661fa2c..ca76c1bf1ceb1 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts @@ -55,11 +55,11 @@ import { isPromiseFulfilled, isPromiseRejected, } from './utils'; +import type { EntityRecord, EntityStoreConfig } from './types'; import { ENTITY_ENGINE_INITIALIZATION_EVENT, ENTITY_ENGINE_RESOURCE_INIT_FAILURE_EVENT, } from '../../telemetry/event_based/events'; -import type { EntityRecord } from './types'; import { CRITICALITY_VALUES } from '../asset_criticality/constants'; interface EntityStoreClientOpts { @@ -72,6 +72,7 @@ interface EntityStoreClientOpts { kibanaVersion: string; dataViewsService: DataViewsService; appClient: AppClient; + config: EntityStoreConfig; telemetry?: AnalyticsServiceSetup; } @@ -130,7 +131,7 @@ export class EntityStoreDataClient { throw new Error('Task Manager is not available'); } - const { logger } = this.options; + const { logger, config } = this.options; await this.riskScoreDataClient.createRiskScoreLatestIndex(); @@ -161,9 +162,10 @@ export class EntityStoreDataClient { this.options.taskManager, indexPattern, filter, + config, pipelineDebugMode ).catch((error) => { - logger.error(`There was an error during async setup of the Entity Store: ${error}`); + logger.error(`There was an error during async setup of the Entity Store: ${error.message}`); }); return descriptor; @@ -175,6 +177,7 @@ export class EntityStoreDataClient { taskManager: TaskManagerStartContract, indexPattern: string, filter: string, + config: EntityStoreConfig, pipelineDebugMode: boolean ) { const setupStartTime = moment().utc().toISOString(); @@ -186,6 +189,8 @@ export class EntityStoreDataClient { entityType, namespace, fieldHistoryLength, + syncDelay: `${config.syncDelay.asSeconds()}s`, + frequency: `${config.frequency.asSeconds()}s`, }); const { entityManagerDefinition } = unitedDefinition; @@ -348,16 +353,20 @@ export class EntityStoreDataClient { taskManager: TaskManagerStartContract, options = { deleteData: false, deleteEngine: true } ) { - const { namespace, logger, appClient, dataViewsService } = this.options; + const { namespace, logger, appClient, dataViewsService, config } = this.options; const { deleteData, deleteEngine } = options; const descriptor = await this.engineClient.maybeGet(entityType); const indexPatterns = await buildIndexPatterns(namespace, appClient, dataViewsService); + + // TODO delete unitedDefinition from this method. we only need the id for deletion const unitedDefinition = getUnitedEntityDefinition({ indexPatterns, entityType, namespace: this.options.namespace, fieldHistoryLength: descriptor?.fieldHistoryLength ?? 10, + syncDelay: `${config.syncDelay.asSeconds()}s`, + frequency: `${config.frequency.asSeconds()}s`, }); const { entityManagerDefinition } = unitedDefinition; logger.info(`In namespace ${namespace}: Deleting entity store for ${entityType}`); diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/stop.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/stop.ts index e1c28bc2cc073..3ec84e13aa1db 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/stop.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/stop.ts @@ -47,7 +47,7 @@ export const stopEntityEngineRoute = ( return response.ok({ body: { stopped: engine.status === ENGINE_STATUS.STOPPED } }); } catch (e) { - logger.error('Error in StopEntityEngine:', e); + logger.error(`Error in StopEntityEngine: ${e.message}`); const error = transformError(e); return siemResponse.error({ statusCode: error.statusCode, diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/types.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/types.ts index e5f1e6db36bca..b71380b2e0677 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/types.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/types.ts @@ -7,6 +7,7 @@ import type { HostEntity, UserEntity } from '../../../../common/api/entity_analytics'; import type { CriticalityValues } from '../asset_criticality/constants'; +import type { EntityAnalyticsConfig } from '../types'; export interface HostEntityRecord extends Omit { asset?: { @@ -24,3 +25,5 @@ export interface UserEntityRecord extends Omit { * It represents the data stored in the entity store index. */ export type EntityRecord = HostEntityRecord | UserEntityRecord; + +export type EntityStoreConfig = EntityAnalyticsConfig['entityStore']; diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.test.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.test.ts index d9c54e1fcd288..fa443ffa94047 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.test.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.test.ts @@ -15,6 +15,8 @@ describe('getUnitedEntityDefinition', () => { namespace: 'test', fieldHistoryLength: 10, indexPatterns, + syncDelay: '1m', + frequency: '1m', }); it('mapping', () => { @@ -172,6 +174,10 @@ describe('getUnitedEntityDefinition', () => { ], "latest": Object { "lookbackPeriod": "24h", + "settings": Object { + "frequency": "1m", + "syncDelay": "1m", + }, "timestampField": "@timestamp", }, "managed": true, @@ -312,6 +318,8 @@ describe('getUnitedEntityDefinition', () => { namespace: 'test', fieldHistoryLength: 10, indexPatterns, + syncDelay: '1m', + frequency: '1m', }); it('mapping', () => { @@ -445,6 +453,10 @@ describe('getUnitedEntityDefinition', () => { ], "latest": Object { "lookbackPeriod": "24h", + "settings": Object { + "frequency": "1m", + "syncDelay": "1m", + }, "timestampField": "@timestamp", }, "managed": true, diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts index 32cb52a61d469..ba4963d5fea0a 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts @@ -25,6 +25,8 @@ interface Options { namespace: string; fieldHistoryLength: number; indexPatterns: string[]; + syncDelay: string; + frequency: string; } export const getUnitedEntityDefinition = memoize( @@ -33,6 +35,8 @@ export const getUnitedEntityDefinition = memoize( namespace, fieldHistoryLength, indexPatterns, + syncDelay, + frequency, }: Options): UnitedEntityDefinition => { const unitedDefinition = unitedDefinitionBuilders[entityType](fieldHistoryLength); @@ -47,6 +51,8 @@ export const getUnitedEntityDefinition = memoize( ...unitedDefinition, namespace, indexPatterns, + syncDelay, + frequency, }); } ); diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/united_entity_definition.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/united_entity_definition.ts index c5315c5dca2b0..eced765c75193 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/united_entity_definition.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/united_entity_definition.ts @@ -7,7 +7,7 @@ import { entityDefinitionSchema, type EntityDefinition } from '@kbn/entities-schema'; import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; import type { EntityType } from '../../../../../common/api/entity_analytics/entity_store/common.gen'; -import { DEFAULT_INTERVAL, DEFAULT_LOOKBACK_PERIOD } from '../constants'; +import { DEFAULT_LOOKBACK_PERIOD } from '../constants'; import { buildEntityDefinitionId, getIdentityFieldForEntityType } from '../utils'; import type { FieldRetentionDefinition, @@ -25,6 +25,8 @@ export class UnitedEntityDefinition { entityManagerDefinition: EntityDefinition; fieldRetentionDefinition: FieldRetentionDefinition; indexMappings: MappingTypeMapping; + syncDelay: string; + frequency: string; constructor(opts: { version: string; @@ -32,11 +34,15 @@ export class UnitedEntityDefinition { indexPatterns: string[]; fields: UnitedDefinitionField[]; namespace: string; + syncDelay: string; + frequency: string; }) { this.version = opts.version; this.entityType = opts.entityType; this.indexPatterns = opts.indexPatterns; this.fields = opts.fields; + this.frequency = opts.frequency; + this.syncDelay = opts.syncDelay; this.namespace = opts.namespace; this.entityManagerDefinition = this.toEntityManagerDefinition(); this.fieldRetentionDefinition = this.toFieldRetentionDefinition(); @@ -44,7 +50,7 @@ export class UnitedEntityDefinition { } private toEntityManagerDefinition(): EntityDefinition { - const { entityType, namespace, indexPatterns } = this; + const { entityType, namespace, indexPatterns, syncDelay, frequency } = this; const identityField = getIdentityFieldForEntityType(this.entityType); const metadata = this.fields .filter((field) => field.definition) @@ -61,7 +67,10 @@ export class UnitedEntityDefinition { latest: { timestampField: '@timestamp', lookbackPeriod: DEFAULT_LOOKBACK_PERIOD, - interval: DEFAULT_INTERVAL, + settings: { + syncDelay, + frequency, + }, }, version: this.version, managed: true, diff --git a/x-pack/plugins/security_solution/server/request_context_factory.ts b/x-pack/plugins/security_solution/server/request_context_factory.ts index e57141b3a5ae7..bd5c29651e26e 100644 --- a/x-pack/plugins/security_solution/server/request_context_factory.ts +++ b/x-pack/plugins/security_solution/server/request_context_factory.ts @@ -225,6 +225,7 @@ export class RequestContextFactory implements IRequestContextFactory { taskManager: startPlugins.taskManager, auditLogger: getAuditLogger(), kibanaVersion: options.kibanaVersion, + config: config.entityAnalytics.entityStore, telemetry: core.analytics, }); }),