From d0e534e9d2262da59baac83f00607094f0c845c4 Mon Sep 17 00:00:00 2001 From: gabrielgheorghescu Date: Tue, 3 Dec 2024 11:40:49 +0200 Subject: [PATCH 1/6] Implement in-memory cache --- .../ResourceProviderServiceBase.cs | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/src/dotnet/Common/Services/ResourceProviders/ResourceProviderServiceBase.cs b/src/dotnet/Common/Services/ResourceProviders/ResourceProviderServiceBase.cs index fc0a1f5378..58f2716866 100644 --- a/src/dotnet/Common/Services/ResourceProviders/ResourceProviderServiceBase.cs +++ b/src/dotnet/Common/Services/ResourceProviders/ResourceProviderServiceBase.cs @@ -11,6 +11,7 @@ using FoundationaLLM.Common.Models.ResourceProviders; using FoundationaLLM.Common.Services.Events; using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using System.Collections.Immutable; @@ -37,6 +38,12 @@ public class ResourceProviderServiceBase : IResourceProvider private readonly bool _useInternalReferencesStore; private readonly SemaphoreSlim _lock = new(1, 1); + private readonly IMemoryCache? _resourceCache; + private readonly MemoryCacheEntryOptions _cacheEntryOptions = new MemoryCacheEntryOptions() + .SetAbsoluteExpiration(TimeSpan.FromMinutes(10)) // Cache entries are valid for 10 minutes. + .SetSlidingExpiration(TimeSpan.FromMinutes(5)) // Reset expiration time if accessed within 5 minutes. + .SetSize(1); // Each cache entry is a single resource. + /// /// The resource reference store used by the resource provider. /// @@ -122,6 +129,7 @@ public class ResourceProviderServiceBase : IResourceProvider /// The of the main dependency injection container. /// The list of Event Service event namespaces to subscribe to for local event processing. /// Indicates whether the resource provider should use the internal resource references store or provide one of its own. + /// Determines whether an in-memory cache is used for storing and retrieving resources. public ResourceProviderServiceBase( InstanceSettings instanceSettings, IAuthorizationService authorizationService, @@ -131,7 +139,8 @@ public ResourceProviderServiceBase( IServiceProvider serviceProvider, ILogger logger, List? eventNamespacesToSubscribe = default, - bool useInternalReferencesStore = false) + bool useInternalReferencesStore = false, + bool cacheResources = false) { _authorizationService = authorizationService; _storageService = storageService; @@ -143,6 +152,15 @@ public ResourceProviderServiceBase( _eventNamespacesToSubscribe = eventNamespacesToSubscribe; _useInternalReferencesStore = useInternalReferencesStore; + if (cacheResources) + { + _resourceCache = new MemoryCache(new MemoryCacheOptions + { + SizeLimit = 5000, // Limit cache size to 5000 resources. + ExpirationScanFrequency = TimeSpan.FromMinutes(1) // Scan for expired items every minute. + }); + } + _allowedResourceProviders = [_name]; _allowedResourceTypes = GetResourceTypes(); @@ -996,6 +1014,11 @@ protected async Task>> LoadResources( $"The resource reference {resourceReference.Name} is not of the expected type {typeof(T).Name}.", StatusCodes.Status400BadRequest); + if (_resourceCache != null && _resourceCache.TryGetValue(resourceReference.Name, out T? cachedResource)) + { + return cachedResource; + } + if (await _storageService.FileExistsAsync(_storageContainerName, resourceReference.Filename, default)) { var fileContent = @@ -1038,6 +1061,11 @@ protected async Task>> LoadResources( ?? throw new ResourceProviderException($"Could not locate the {resourceName} resource.", StatusCodes.Status404NotFound); + if (_resourceCache != null && _resourceCache.TryGetValue(resourceReference.Name, out T? cachedResource)) + { + return cachedResource; + } + if (await _storageService.FileExistsAsync(_storageContainerName, resourceReference.Filename, default)) { var fileContent = @@ -1086,6 +1114,9 @@ await _storageService.WriteFileAsync( default); await _resourceReferenceStore!.AddResourceReference(resourceReference); + + // Add resource to cache if caching is enabled. + _resourceCache?.Set(resourceReference.Name, resource, _cacheEntryOptions); } finally { @@ -1200,6 +1231,9 @@ await _storageService.WriteFileAsync( JsonSerializer.Serialize(resource, _serializerSettings), default, default); + + // Update resource to cache if caching is enabled. + _resourceCache?.Set(resourceReference.Name, resource, _cacheEntryOptions); } finally { From 6e993259104a948c6c24763417506292b516ced2 Mon Sep 17 00:00:00 2001 From: gabrielgheorghescu Date: Tue, 3 Dec 2024 11:52:25 +0200 Subject: [PATCH 2/6] Add to cache also when loading resources --- .../ResourceProviders/ResourceProviderServiceBase.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/dotnet/Common/Services/ResourceProviders/ResourceProviderServiceBase.cs b/src/dotnet/Common/Services/ResourceProviders/ResourceProviderServiceBase.cs index 58f2716866..a3faeb1747 100644 --- a/src/dotnet/Common/Services/ResourceProviders/ResourceProviderServiceBase.cs +++ b/src/dotnet/Common/Services/ResourceProviders/ResourceProviderServiceBase.cs @@ -14,6 +14,8 @@ using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using MimeDetective.Storage; +using OpenTelemetry.Resources; using System.Collections.Immutable; using System.Text; using System.Text.Json; @@ -1032,6 +1034,8 @@ protected async Task>> LoadResources( ?? throw new ResourceProviderException($"Failed to load the resource {resourceReference.Name}. Its content file might be corrupt.", StatusCodes.Status500InternalServerError); + _resourceCache?.Set(resourceReference.Name, resourceObject, _cacheEntryOptions); + return resourceObject; } catch (Exception ex) @@ -1076,6 +1080,8 @@ protected async Task>> LoadResources( ?? throw new ResourceProviderException($"Failed to load the resource {resourceReference.Name}. Its content file might be corrupt.", StatusCodes.Status400BadRequest); + _resourceCache?.Set(resourceReference.Name, resourceObject, _cacheEntryOptions); + return resourceObject; } From 8861e34739a5cead0230ce977a338177d1abbab4 Mon Sep 17 00:00:00 2001 From: gabrielgheorghescu Date: Wed, 11 Dec 2024 18:01:48 +0200 Subject: [PATCH 3/6] Configuration item to control ResourceProvider level-caching --- docs/release-notes/breaking-changes.md | 1 + .../Constants/Data/AppConfiguration.json | 8 ++ .../Instance/InstanceSettings.cs | 5 ++ .../ResourceProviderServiceBase.cs | 8 +- .../Common/Templates/AppConfigurationKeys.cs | 7 ++ .../Templates/AuthorizableActionNames.cs | 90 ------------------- .../Common/Templates/appconfig.template.json | 7 ++ 7 files changed, 30 insertions(+), 96 deletions(-) diff --git a/docs/release-notes/breaking-changes.md b/docs/release-notes/breaking-changes.md index 4919b8aef5..1a429ed535 100644 --- a/docs/release-notes/breaking-changes.md +++ b/docs/release-notes/breaking-changes.md @@ -17,6 +17,7 @@ The following new App Configuration settings are required: |`FoundationaLLM:APIEndpoints:LangChainAPI:Configuration:ExternalModules:Storage:AuthenticationType` | `-` | |`FoundationaLLM:APIEndpoints:LangChainAPI:Configuration:ExternalModules:RootStorageContainer` | `-` | |`FoundationaLLM:APIEndpoints:LangChainAPI:Configuration:ExternalModules:Modules` | `-` | +`FoundationaLLM:Instance:EnableResourceProvidersCache` | `Enables caching for resource providers` | #### Agent Tool configuration changes diff --git a/src/dotnet/Common/Constants/Data/AppConfiguration.json b/src/dotnet/Common/Constants/Data/AppConfiguration.json index 50d0cfbe65..b4ec4ed592 100644 --- a/src/dotnet/Common/Constants/Data/AppConfiguration.json +++ b/src/dotnet/Common/Constants/Data/AppConfiguration.json @@ -37,6 +37,14 @@ "value": "", "content_type": "", "first_version": "0.8.0" + }, + { + "name": "EnableResourceProvidersCache", + "description": "Enable caching for resource providers.", + "secret": "", + "value": "false", + "content_type": "", + "first_version": "0.9.1" } ] }, diff --git a/src/dotnet/Common/Models/Configuration/Instance/InstanceSettings.cs b/src/dotnet/Common/Models/Configuration/Instance/InstanceSettings.cs index 326abbffba..e13ea1d188 100644 --- a/src/dotnet/Common/Models/Configuration/Instance/InstanceSettings.cs +++ b/src/dotnet/Common/Models/Configuration/Instance/InstanceSettings.cs @@ -25,5 +25,10 @@ public class InstanceSettings /// The Regex pattern used to validate the values allowed as User Principal Name (UPN) substitutes in the X-USER-IDENTITY header. /// public string? IdentitySubstitutionUserPrincipalNamePattern { get; set; } + + /// + /// Enabling caching for resource providers. + /// + public bool EnableResourceProvidersCache { get; set; } = false; } } diff --git a/src/dotnet/Common/Services/ResourceProviders/ResourceProviderServiceBase.cs b/src/dotnet/Common/Services/ResourceProviders/ResourceProviderServiceBase.cs index a3faeb1747..9d4aab4475 100644 --- a/src/dotnet/Common/Services/ResourceProviders/ResourceProviderServiceBase.cs +++ b/src/dotnet/Common/Services/ResourceProviders/ResourceProviderServiceBase.cs @@ -14,8 +14,6 @@ using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; -using MimeDetective.Storage; -using OpenTelemetry.Resources; using System.Collections.Immutable; using System.Text; using System.Text.Json; @@ -131,7 +129,6 @@ public class ResourceProviderServiceBase : IResourceProvider /// The of the main dependency injection container. /// The list of Event Service event namespaces to subscribe to for local event processing. /// Indicates whether the resource provider should use the internal resource references store or provide one of its own. - /// Determines whether an in-memory cache is used for storing and retrieving resources. public ResourceProviderServiceBase( InstanceSettings instanceSettings, IAuthorizationService authorizationService, @@ -141,8 +138,7 @@ public ResourceProviderServiceBase( IServiceProvider serviceProvider, ILogger logger, List? eventNamespacesToSubscribe = default, - bool useInternalReferencesStore = false, - bool cacheResources = false) + bool useInternalReferencesStore = false) { _authorizationService = authorizationService; _storageService = storageService; @@ -154,7 +150,7 @@ public ResourceProviderServiceBase( _eventNamespacesToSubscribe = eventNamespacesToSubscribe; _useInternalReferencesStore = useInternalReferencesStore; - if (cacheResources) + if (_instanceSettings.EnableResourceProvidersCache) { _resourceCache = new MemoryCache(new MemoryCacheOptions { diff --git a/src/dotnet/Common/Templates/AppConfigurationKeys.cs b/src/dotnet/Common/Templates/AppConfigurationKeys.cs index 8487672a77..db860e4c83 100644 --- a/src/dotnet/Common/Templates/AppConfigurationKeys.cs +++ b/src/dotnet/Common/Templates/AppConfigurationKeys.cs @@ -41,6 +41,13 @@ public static class AppConfigurationKeys /// public const string FoundationaLLM_Instance_IdentitySubstitutionUserPrincipalNamePattern = "FoundationaLLM:Instance:IdentitySubstitutionUserPrincipalNamePattern"; + + /// + /// The app configuration key for the FoundationaLLM:Instance:EnableResourceProvidersCache setting. + /// Value description:
Enable caching for resource providers. + /// + public const string FoundationaLLM_Instance_EnableResourceProvidersCache = + "FoundationaLLM:Instance:EnableResourceProvidersCache"; #endregion diff --git a/src/dotnet/Common/Templates/AuthorizableActionNames.cs b/src/dotnet/Common/Templates/AuthorizableActionNames.cs index 3c56d43dd0..720c087a23 100644 --- a/src/dotnet/Common/Templates/AuthorizableActionNames.cs +++ b/src/dotnet/Common/Templates/AuthorizableActionNames.cs @@ -1,13 +1,3 @@ - - - - - - - - - - namespace FoundationaLLM.Common.Constants.Authorization { /// @@ -15,414 +5,334 @@ namespace FoundationaLLM.Common.Constants.Authorization /// public static class AuthorizableActionNames { - #region Authorization - /// /// Read role assignments. /// public const string FoundationaLLM_Authorization_RoleAssignments_Read = "FoundationaLLM.Authorization/roleAssignments/read"; - /// /// Create or update role assignments. /// public const string FoundationaLLM_Authorization_RoleAssignments_Write = "FoundationaLLM.Authorization/roleAssignments/write"; - /// /// Delete role assignments. /// public const string FoundationaLLM_Authorization_RoleAssignments_Delete = "FoundationaLLM.Authorization/roleAssignments/delete"; - /// /// Read role definitions. /// public const string FoundationaLLM_Authorization_RoleDefinitions_Read = "FoundationaLLM.Authorization/roleDefinitions/read"; - #endregion - #region Agent - /// /// Read agents. /// public const string FoundationaLLM_Agent_Agents_Read = "FoundationaLLM.Agent/agents/read"; - /// /// Create or update agents. /// public const string FoundationaLLM_Agent_Agents_Write = "FoundationaLLM.Agent/agents/write"; - /// /// Delete agents. /// public const string FoundationaLLM_Agent_Agents_Delete = "FoundationaLLM.Agent/agents/delete"; - /// /// Read workflows. /// public const string FoundationaLLM_Agent_Workflows_Read = "FoundationaLLM.Agent/workflows/read"; - /// /// Create or update workflows. /// public const string FoundationaLLM_Agent_Workflows_Write = "FoundationaLLM.Agent/workflows/write"; - /// /// Delete workflows. /// public const string FoundationaLLM_Agent_Workflows_Delete = "FoundationaLLM.Agent/workflows/delete"; - - #endregion - #region AzureOpenAI - /// /// Read Azure OpenAI conversation mappings. /// public const string FoundationaLLM_AzureOpenAI_ConversationMappings_Read = "FoundationaLLM.AzureOpenAI/conversationMappings/read"; - /// /// Create or update Azure OpenAI conversation mappings. /// public const string FoundationaLLM_AzureOpenAI_ConversationMappings_Write = "FoundationaLLM.AzureOpenAI/conversationMappings/write"; - /// /// Delete Azure OpenAI conversation mappings. /// public const string FoundationaLLM_AzureOpenAI_ConversationMappings_Delete = "FoundationaLLM.AzureOpenAI/conversationMappings/delete"; - /// /// Read Azure OpenAI file mappings. /// public const string FoundationaLLM_AzureOpenAI_FileMappings_Read = "FoundationaLLM.AzureOpenAI/fileMappings/read"; - /// /// Create or update Azure OpenAI file mappings. /// public const string FoundationaLLM_AzureOpenAI_FileMappings_Write = "FoundationaLLM.AzureOpenAI/fileMappings/write"; - /// /// Delete Azure OpenAI file mappings. /// public const string FoundationaLLM_AzureOpenAI_FileMappings_Delete = "FoundationaLLM.AzureOpenAI/fileMappings/delete"; - #endregion - #region Configuration - /// /// Read app configurations. /// public const string FoundationaLLM_Configuration_AppConfigurations_Read = "FoundationaLLM.Configuration/appConfigurations/read"; - /// /// Create or update app configurations. /// public const string FoundationaLLM_Configuration_AppConfigurations_Write = "FoundationaLLM.Configuration/appConfigurations/write"; - /// /// Delete app configurations. /// public const string FoundationaLLM_Configuration_AppConfigurations_Delete = "FoundationaLLM.Configuration/appConfigurations/delete"; - /// /// Read key vault secrets. /// public const string FoundationaLLM_Configuration_KeyVaultSecrets_Read = "FoundationaLLM.Configuration/keyVaultSecrets/read"; - /// /// Create or update key vault secrets. /// public const string FoundationaLLM_Configuration_KeyVaultSecrets_Write = "FoundationaLLM.Configuration/keyVaultSecrets/write"; - /// /// Delete key vault secrets. /// public const string FoundationaLLM_Configuration_KeyVaultSecrets_Delete = "FoundationaLLM.Configuration/keyVaultSecrets/delete"; - /// /// Read API endpoint configurations. /// public const string FoundationaLLM_Configuration_APIEndpointConfigurations_Read = "FoundationaLLM.Configuration/apiEndpointConfigurations/read"; - /// /// Create or update API endpoint configurations. /// public const string FoundationaLLM_Configuration_APIEndpointConfigurations_Write = "FoundationaLLM.Configuration/apiEndpointConfigurations/write"; - /// /// Delete API endpoint configurations. /// public const string FoundationaLLM_Configuration_APIEndpoinConfigurations_Delete = "FoundationaLLM.Configuration/apiEndpointConfigurations/delete"; - #endregion - #region DataSource - /// /// Read data sources. /// public const string FoundationaLLM_DataSource_DataSources_Read = "FoundationaLLM.DataSource/dataSources/read"; - /// /// Create or update data sources. /// public const string FoundationaLLM_DataSource_DataSources_Write = "FoundationaLLM.DataSource/dataSources/write"; - /// /// Delete data sources. /// public const string FoundationaLLM_DataSource_DataSources_Delete = "FoundationaLLM.DataSource/dataSources/delete"; - #endregion - #region Prompt - /// /// Read prompts. /// public const string FoundationaLLM_Prompt_Prompts_Read = "FoundationaLLM.Prompt/prompts/read"; - /// /// Create or update prompts. /// public const string FoundationaLLM_Prompt_Prompts_Write = "FoundationaLLM.Prompt/prompts/write"; - /// /// Delete prompts. /// public const string FoundationaLLM_Prompt_Prompts_Delete = "FoundationaLLM.Prompt/prompts/delete"; - #endregion - #region Vectorization - /// /// Read vectorization pipelines. /// public const string FoundationaLLM_Vectorization_VectorizationPipelines_Read = "FoundationaLLM.Vectorization/vectorizationPipelines/read"; - /// /// Create or update vectorization pipelines. /// public const string FoundationaLLM_Vectorization_VectorizationPipelines_Write = "FoundationaLLM.Vectorization/vectorizationPipelines/write"; - /// /// Delete vectorization pipelines. /// public const string FoundationaLLM_Vectorization_VectorizationPipelines_Delete = "FoundationaLLM.Vectorization/vectorizationPipelines/delete"; - /// /// Read vectorization requests. /// public const string FoundationaLLM_Vectorization_VectorizationRequests_Read = "FoundationaLLM.Vectorization/vectorizationRequests/read"; - /// /// Create or update vectorization requests. /// public const string FoundationaLLM_Vectorization_VectorizationRequests_Write = "FoundationaLLM.Vectorization/vectorizationRequests/write"; - /// /// Delete vectorization requests. /// public const string FoundationaLLM_Vectorization_VectorizationRequests_Delete = "FoundationaLLM.Vectorization/vectorizationRequests/delete"; - /// /// Read vectorization content source profiles. /// public const string FoundationaLLM_Vectorization_ContentSourceProfiles_Read = "FoundationaLLM.Vectorization/contentSourceProfiles/read"; - /// /// Create or update vectorization content source profiles. /// public const string FoundationaLLM_Vectorization_ContentSourceProfiles_Write = "FoundationaLLM.Vectorization/contentSourceProfiles/write"; - /// /// Delete vectorization content source profiles. /// public const string FoundationaLLM_Vectorization_ContentSourceProfiles_Delete = "FoundationaLLM.Vectorization/contentSourceProfiles/delete"; - /// /// Read vectorization text partitioning profiles. /// public const string FoundationaLLM_Vectorization_TextPartitioningProfiles_Read = "FoundationaLLM.Vectorization/textPartitioningProfiles/read"; - /// /// Create or update vectorization text partitioning profiles. /// public const string FoundationaLLM_Vectorization_TextPartitioningProfiles_Write = "FoundationaLLM.Vectorization/textPartitioningProfiles/write"; - /// /// Delete vectorization text partitioning profiles. /// public const string FoundationaLLM_Vectorization_TextPartitioningProfiles_Delete = "FoundationaLLM.Vectorization/textPartitioningProfiles/delete"; - /// /// Read vectorization text embedding profiles. /// public const string FoundationaLLM_Vectorization_TextEmbeddingProfiles_Read = "FoundationaLLM.Vectorization/textEmbeddingProfiles/read"; - /// /// Create or update vectorization text embedding profiles. /// public const string FoundationaLLM_Vectorization_TextEmbeddingProfiles_Write = "FoundationaLLM.Vectorization/textEmbeddingProfiles/write"; - /// /// Delete vectorization text embedding profiles. /// public const string FoundationaLLM_Vectorization_TextEmbeddingProfiles_Delete = "FoundationaLLM.Vectorization/textEmbeddingProfiles/delete"; - /// /// Read vectorization indexing profiles. /// public const string FoundationaLLM_Vectorization_IndexingProfiles_Read = "FoundationaLLM.Vectorization/indexingProfiles/read"; - /// /// Create or update vectorization indexing profiles. /// public const string FoundationaLLM_Vectorization_IndexingProfiles_Write = "FoundationaLLM.Vectorization/indexingProfiles/write"; - /// /// Delete vectorization indexing profiles. /// public const string FoundationaLLM_Vectorization_IndexingProfiles_Delete = "FoundationaLLM.Vectorization/indexingProfiles/delete"; - #endregion - #region Attachment - /// /// Read attachments. /// public const string FoundationaLLM_Attachment_Attachments_Read = "FoundationaLLM.Attachment/attachments/read"; - /// /// Create or update attachments. /// public const string FoundationaLLM_Attachment_Attachments_Write = "FoundationaLLM.Attachment/attachments/write"; - /// /// Delete attachments. /// public const string FoundationaLLM_Attachment_Attachments_Delete = "FoundationaLLM.Attachment/attachments/delete"; - #endregion - #region AIModel - /// /// Read AI models /// public const string FoundationaLLM_AIModel_AIModels_Read = "FoundationaLLM.AIModel/aiModels/read"; - /// /// Create or update AI models. /// public const string FoundationaLLM_AIModel_AIModels_Write = "FoundationaLLM.AIModel/aiModels/write"; - /// /// Delete AI models. /// public const string FoundationaLLM_AIModel_AIModels_Delete = "FoundationaLLM.AIModel/aiModels/delete"; - #endregion - #region Conversation - /// /// Read conversations /// public const string FoundationaLLM_Conversation_Conversations_Read = "FoundationaLLM.Conversation/conversations/read"; - /// /// Create or update conversations. /// public const string FoundationaLLM_Conversation_Conversations_Write = "FoundationaLLM.Conversation/conversations/write"; - /// /// Delete conversations. /// public const string FoundationaLLM_Conversation_Conversations_Delete = "FoundationaLLM.Conversation/conversations/delete"; - #endregion - } } diff --git a/src/dotnet/Common/Templates/appconfig.template.json b/src/dotnet/Common/Templates/appconfig.template.json index cef5a74dac..ab34bcc9fd 100644 --- a/src/dotnet/Common/Templates/appconfig.template.json +++ b/src/dotnet/Common/Templates/appconfig.template.json @@ -35,6 +35,13 @@ "content_type": "", "tags": {} }, + { + "key": "FoundationaLLM:Instance:EnableResourceProvidersCache", + "value": "false", + "label": null, + "content_type": "", + "tags": {} + }, { "key": "FoundationaLLM:Configuration:KeyVaultURI", "value": "${env:AZURE_KEY_VAULT_ENDPOINT}", From a4341a0bd4f8c2016f252e704202ec7027748c28 Mon Sep 17 00:00:00 2001 From: Ciprian Jichici Date: Wed, 11 Dec 2024 23:52:44 +0200 Subject: [PATCH 4/6] Define resource provider resource cache interface --- .../IResourceProviderResourceCacheService.cs | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 src/dotnet/Common/Interfaces/IResourceProviderResourceCacheService.cs diff --git a/src/dotnet/Common/Interfaces/IResourceProviderResourceCacheService.cs b/src/dotnet/Common/Interfaces/IResourceProviderResourceCacheService.cs new file mode 100644 index 0000000000..410ce664dd --- /dev/null +++ b/src/dotnet/Common/Interfaces/IResourceProviderResourceCacheService.cs @@ -0,0 +1,27 @@ +using FoundationaLLM.Common.Models.ResourceProviders; + +namespace FoundationaLLM.Common.Interfaces +{ + /// + /// Provides the resource caching services used by FoundationaLLM resource providers. + /// + public interface IResourceProviderResourceCacheService + { + /// + /// Tries to get a resource value identified by a resource reference from the cache. + /// + /// The type of resource value to be retrieved. + /// The used as a key in the cache. + /// The resource value to be retrieved. + /// is the resource value was found in the cache, otherwise. + bool TryGetValue(ResourceReference resourceReference, out T? resourceValue) where T: ResourceBase; + + /// + /// Sets a resource value identified by a resource reference in the cache. + /// + /// The type of resource value to be set. + /// The used as a key in the cache. + /// The resource value to be set. + void SetValue(ResourceReference resourceReference, T resourceValue) where T : ResourceBase; + } +} From 8040047f405a6a939dc1ddc0ff2a1e0ea73560d5 Mon Sep 17 00:00:00 2001 From: Ciprian Jichici Date: Thu, 12 Dec 2024 01:24:27 +0200 Subject: [PATCH 5/6] Final version of resource provider resource cache --- .../ResourceProviderResourceCacheService.cs | 72 +++++++++++++++++++ .../ResourceProviderServiceBase.cs | 35 ++++----- 2 files changed, 85 insertions(+), 22 deletions(-) create mode 100644 src/dotnet/Common/Services/ResourceProviders/ResourceProviderResourceCacheService.cs diff --git a/src/dotnet/Common/Services/ResourceProviders/ResourceProviderResourceCacheService.cs b/src/dotnet/Common/Services/ResourceProviders/ResourceProviderResourceCacheService.cs new file mode 100644 index 0000000000..b10cad677b --- /dev/null +++ b/src/dotnet/Common/Services/ResourceProviders/ResourceProviderResourceCacheService.cs @@ -0,0 +1,72 @@ +using FoundationaLLM.Common.Interfaces; +using FoundationaLLM.Common.Models.ResourceProviders; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; + +namespace FoundationaLLM.Common.Services.ResourceProviders +{ + /// + /// Provides the resource caching services used by FoundationaLLM resource providers. + /// + /// The used to log information. + public class ResourceProviderResourceCacheService( + ILogger logger) : IResourceProviderResourceCacheService + { + private readonly ILogger _logger = logger; + + private readonly IMemoryCache _cache = new MemoryCache(new MemoryCacheOptions + { + SizeLimit = 10000, // Limit cache size to 5000 resources. + ExpirationScanFrequency = TimeSpan.FromMinutes(5) // Scan for expired items every five minutes. + }); + private readonly MemoryCacheEntryOptions _cacheEntryOptions = new MemoryCacheEntryOptions() + .SetAbsoluteExpiration(TimeSpan.FromMinutes(60)) // Cache entries are valid for 60 minutes. + .SetSlidingExpiration(TimeSpan.FromMinutes(30)) // Reset expiration time if accessed within 5 minutes. + .SetSize(1); // Each cache entry is a single resource. + + /// + public void SetValue(ResourceReference resourceReference, T resourceValue) where T : ResourceBase + { + try + { + _cache.Set(resourceReference, resourceValue, _cacheEntryOptions); + _logger.LogInformation("The resource {ResourceName} of type {ResourceType} has been set in the cache.", + resourceReference.Name, + resourceReference.Type); + } + catch (Exception ex) + { + _logger.LogError(ex, "There was an error setting the resource {ResourceName} of type {ResourceType} in the cache.", + resourceReference.Name, + resourceReference.Type); + } + } + + /// + public bool TryGetValue(ResourceReference resourceReference, out T? resourceValue) where T : ResourceBase + { + resourceValue = default; + + try + { + if (_cache.TryGetValue(resourceReference, out T? cachedValue) + && cachedValue != null) + { + resourceValue = cachedValue; + _logger.LogInformation("The resource {ResourceName} of type {ResourceType} has been retrieved from the cache.", + resourceReference.Name, + resourceReference.Type); + return true; + } + } + catch (Exception ex) + { + _logger.LogError(ex, "There was an error getting the resource {ResourceName} of type {ResourceType} from the cache.", + resourceReference.Name, + resourceReference.Type); + } + + return false; + } + } +} diff --git a/src/dotnet/Common/Services/ResourceProviders/ResourceProviderServiceBase.cs b/src/dotnet/Common/Services/ResourceProviders/ResourceProviderServiceBase.cs index dd0af1a408..10030167dd 100644 --- a/src/dotnet/Common/Services/ResourceProviders/ResourceProviderServiceBase.cs +++ b/src/dotnet/Common/Services/ResourceProviders/ResourceProviderServiceBase.cs @@ -38,11 +38,7 @@ public class ResourceProviderServiceBase : IResourceProvider private readonly bool _useInternalReferencesStore; private readonly SemaphoreSlim _lock = new(1, 1); - private readonly IMemoryCache? _resourceCache; - private readonly MemoryCacheEntryOptions _cacheEntryOptions = new MemoryCacheEntryOptions() - .SetAbsoluteExpiration(TimeSpan.FromMinutes(10)) // Cache entries are valid for 10 minutes. - .SetSlidingExpiration(TimeSpan.FromMinutes(5)) // Reset expiration time if accessed within 5 minutes. - .SetSize(1); // Each cache entry is a single resource. + private readonly IResourceProviderResourceCacheService? _resourceCache; /// /// The resource reference store used by the resource provider. @@ -151,13 +147,7 @@ public ResourceProviderServiceBase( _useInternalReferencesStore = useInternalReferencesStore; if (_instanceSettings.EnableResourceProvidersCache) - { - _resourceCache = new MemoryCache(new MemoryCacheOptions - { - SizeLimit = 5000, // Limit cache size to 5000 resources. - ExpirationScanFrequency = TimeSpan.FromMinutes(1) // Scan for expired items every minute. - }); - } + _resourceCache = new ResourceProviderResourceCacheService(_logger); _allowedResourceProviders = [_name]; _allowedResourceTypes = GetResourceTypes(); @@ -1012,10 +1002,11 @@ protected async Task>> LoadResources( $"The resource reference {resourceReference.Name} is not of the expected type {typeof(T).Name}.", StatusCodes.Status400BadRequest); - if (_resourceCache != null && _resourceCache.TryGetValue(resourceReference.Name, out T? cachedResource)) - { + + if (_resourceCache != null + && _resourceCache.TryGetValue(resourceReference, out T? cachedResource) + && cachedResource != null) return cachedResource; - } if (await _storageService.FileExistsAsync(_storageContainerName, resourceReference.Filename, default)) { @@ -1030,7 +1021,7 @@ protected async Task>> LoadResources( ?? throw new ResourceProviderException($"Failed to load the resource {resourceReference.Name}. Its content file might be corrupt.", StatusCodes.Status500InternalServerError); - _resourceCache?.Set(resourceReference.Name, resourceObject, _cacheEntryOptions); + _resourceCache?.SetValue(resourceReference, resourceObject); return resourceObject; } @@ -1061,10 +1052,10 @@ protected async Task>> LoadResources( ?? throw new ResourceProviderException($"Could not locate the {resourceName} resource.", StatusCodes.Status404NotFound); - if (_resourceCache != null && _resourceCache.TryGetValue(resourceReference.Name, out T? cachedResource)) - { + if (_resourceCache != null + && _resourceCache.TryGetValue(resourceReference, out T? cachedResource) + && cachedResource != null) return cachedResource; - } if (await _storageService.FileExistsAsync(_storageContainerName, resourceReference.Filename, default)) { @@ -1076,7 +1067,7 @@ protected async Task>> LoadResources( ?? throw new ResourceProviderException($"Failed to load the resource {resourceReference.Name}. Its content file might be corrupt.", StatusCodes.Status400BadRequest); - _resourceCache?.Set(resourceReference.Name, resourceObject, _cacheEntryOptions); + _resourceCache?.SetValue(resourceReference, resourceObject); return resourceObject; } @@ -1118,7 +1109,7 @@ await _storageService.WriteFileAsync( await _resourceReferenceStore!.AddResourceReference(resourceReference); // Add resource to cache if caching is enabled. - _resourceCache?.Set(resourceReference.Name, resource, _cacheEntryOptions); + _resourceCache?.SetValue(resourceReference, resource); } finally { @@ -1235,7 +1226,7 @@ await _storageService.WriteFileAsync( default); // Update resource to cache if caching is enabled. - _resourceCache?.Set(resourceReference.Name, resource, _cacheEntryOptions); + _resourceCache?.SetValue(resourceReference, resource); } finally { From 6aafd5a4b827fb0baf6715a757fae31767cdb4a2 Mon Sep 17 00:00:00 2001 From: Ciprian Jichici Date: Thu, 12 Dec 2024 01:32:43 +0200 Subject: [PATCH 6/6] Improve cache key generation --- .../ResourceProviderResourceCacheService.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/dotnet/Common/Services/ResourceProviders/ResourceProviderResourceCacheService.cs b/src/dotnet/Common/Services/ResourceProviders/ResourceProviderResourceCacheService.cs index b10cad677b..9b69ae76b5 100644 --- a/src/dotnet/Common/Services/ResourceProviders/ResourceProviderResourceCacheService.cs +++ b/src/dotnet/Common/Services/ResourceProviders/ResourceProviderResourceCacheService.cs @@ -29,7 +29,7 @@ public void SetValue(ResourceReference resourceReference, T resourceValue) wh { try { - _cache.Set(resourceReference, resourceValue, _cacheEntryOptions); + _cache.Set(GetCacheKey(resourceReference), resourceValue, _cacheEntryOptions); _logger.LogInformation("The resource {ResourceName} of type {ResourceType} has been set in the cache.", resourceReference.Name, resourceReference.Type); @@ -49,7 +49,7 @@ public bool TryGetValue(ResourceReference resourceReference, out T? resourceV try { - if (_cache.TryGetValue(resourceReference, out T? cachedValue) + if (_cache.TryGetValue(GetCacheKey(resourceReference), out T? cachedValue) && cachedValue != null) { resourceValue = cachedValue; @@ -68,5 +68,8 @@ public bool TryGetValue(ResourceReference resourceReference, out T? resourceV return false; } + + private string GetCacheKey(ResourceReference resourceReference) => + $"{resourceReference.Type}|{resourceReference.Name}"; } }