feat: store aggregates and regular values in same collection to save …

…money on collections in Cosmos DB
alsami · Dec 7, 2020 · b7df57a · b7df57a
1 parent ad684ee
commit b7df57a
Show file tree

Hide file tree

Showing 22 changed files with 210 additions and 116 deletions.
diff --git a/src/Covid19Api.Mongo/CollectionNames.cs → src/Covid19Api.Domain/CollectionNames.cs b/src/Covid19Api.Mongo/CollectionNames.cs → src/Covid19Api.Domain/CollectionNames.cs
diff --git a/src/Covid19Api.Domain/CountryStatistic.cs b/src/Covid19Api.Domain/CountryStatistic.cs
@@ -1,6 +1,7 @@
 using System;
 using System.Security.Cryptography;
 using System.Text;
+using Covid19Api.Mongo;
 
 // ReSharper disable UnusedAutoPropertyAccessor.Local
 // ReSharper disable AutoPropertyCanBeMadeGetOnly.Local
@@ -30,6 +31,10 @@ public class CountryStatistic
 
         public DateTime FetchedAt { get; private set; }
 
+        // ReSharper disable once UnusedMember.Global
+        // This needs to be present for mongo-db!
+        public string Key { get; private set; } = CollectionNames.CountryStatistics;
+
         public CountryStatistic(string country, string? countryCode, int totalCases, int newCases, int totalDeaths,
             int newDeaths,
             int recoveredCases, int activeCases, DateTime fetchedAt)

diff --git a/src/Covid19Api.Domain/GlobalStatistics.cs b/src/Covid19Api.Domain/GlobalStatistics.cs
@@ -1,6 +1,7 @@
 using System;
 using System.Security.Cryptography;
 using System.Text;
+using Covid19Api.Mongo;
 
 // ReSharper disable AutoPropertyCanBeMadeGetOnly.Local
 // ReSharper disable UnusedAutoPropertyAccessor.Global
@@ -19,6 +20,10 @@ public class GlobalStatistics
 
         public DateTime FetchedAt { get; private set; }
 
+        // ReSharper disable once UnusedMember.Global
+        // This needs to be present for mongo-db!
+        public string Key { get; private set; } = CollectionNames.GlobalStatistics;
+
         public GlobalStatistics(int total, int recovered, int deaths, DateTime fetchedAt)
         {
             this.Total = total;

diff --git a/src/Covid19Api.IoC/Extensions/ContainerBuilderExtensions.cs b/src/Covid19Api.IoC/Extensions/ContainerBuilderExtensions.cs
@@ -21,16 +21,24 @@ public static class ContainerBuilderExtensions
         public static ContainerBuilder RegisterRepositories(this ContainerBuilder builder,
             IHostEnvironment hostEnvironment, IConfiguration configuration)
         {
-            builder.RegisterType<GlobalStatisticsRepository>()
-                .As<IGlobalStatisticsRepository>()
+            builder.RegisterType<GlobalStatisticsReadRepository>()
+                .As<IGlobalStatisticsReadRepository>()
+                .InstancePerLifetimeScope();
+
+            builder.RegisterType<GlobalStatisticsWriteRepository>()
+                .As<IGlobalStatisticsWriteRepository>()
                 .InstancePerLifetimeScope();
 
             builder.RegisterType<GlobalStatisticsAggregatesRepository>()
                 .As<IGlobalStatisticsAggregatesRepository>()
                 .InstancePerLifetimeScope();
 
-            builder.RegisterType<CountryStatisticsRepository>()
-                .As<ICountryStatisticsRepository>()
+            builder.RegisterType<CountryStatisticsReadRepository>()
+                .As<ICountryStatisticsReadRepository>()
+                .InstancePerLifetimeScope();
+
+            builder.RegisterType<CountryStatisticsWriteRepository>()
+                .As<ICountryStatisticsWriteRepository>()
                 .InstancePerLifetimeScope();
 
             builder.RegisterType<CountryStatisticsAggregatesRepository>()
@@ -76,10 +84,11 @@ public static ContainerBuilder RegisterServices(this ContainerBuilder builder)
             return builder;
         }
 
-        public static ContainerBuilder RegisterWorker(this ContainerBuilder builder, IHostEnvironment hostEnvironment, bool enableAggregates)
+        public static ContainerBuilder RegisterWorker(this ContainerBuilder builder, IHostEnvironment hostEnvironment,
+            bool enableAggregates)
         {
             if (hostEnvironment.IsContinuousIntegration()) return builder;
-            
+
             builder.RegisterType<DataRefreshWorker>()
                 .As<IHostedService>()
                 .InstancePerDependency();

diff --git a/...tractions/ICountryStatisticsRepository.cs → ...tions/ICountryStatisticsReadRepository.cs b/...tractions/ICountryStatisticsRepository.cs → ...tions/ICountryStatisticsReadRepository.cs
@@ -5,7 +5,7 @@
 
 namespace Covid19Api.Repositories.Abstractions
 {
-    public interface ICountryStatisticsRepository
+    public interface ICountryStatisticsReadRepository
     {
         Task<CountryStatistic> MostRecentAsync(string country);
         Task<IEnumerable<CountryStatistic>> HistoricalAsync(DateTime minFetchedAt);
@@ -14,7 +14,5 @@ public interface ICountryStatisticsRepository
 
         Task<CountryStatistic?> FindInRangeAsync(string country, DateTime inclusiveStart,
             DateTime exclusiveEnd);
-
-        Task StoreManyAsync(IEnumerable<CountryStatistic> countryStats);
     }
 }
diff --git a/src/Covid19Api.Repositories.Abstractions/ICountryStatisticsWriteRepository.cs b/src/Covid19Api.Repositories.Abstractions/ICountryStatisticsWriteRepository.cs
@@ -0,0 +1,11 @@
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Covid19Api.Domain;
+
+namespace Covid19Api.Repositories.Abstractions
+{
+    public interface ICountryStatisticsWriteRepository
+    {
+        Task StoreManyAsync(IEnumerable<CountryStatistic> countryStats);
+    }
+}
diff --git a/...stractions/IGlobalStatisticsRepository.cs → ...ctions/IGlobalStatisticsReadRepository.cs b/...stractions/IGlobalStatisticsRepository.cs → ...ctions/IGlobalStatisticsReadRepository.cs
@@ -5,9 +5,8 @@
 
 namespace Covid19Api.Repositories.Abstractions
 {
-    public interface IGlobalStatisticsRepository
+    public interface IGlobalStatisticsReadRepository
     {
-        Task StoreAsync(GlobalStatistics globalStatistics);
         Task<IEnumerable<GlobalStatistics>> HistoricalAsync(DateTime minFetchedAt);
         Task<IEnumerable<GlobalStatistics>> HistoricalForDayAsync(DateTime minFetchedAt);
         Task<GlobalStatistics?> FindInRangeAsync(DateTime inclusiveStart, DateTime inclusiveEnd);

diff --git a/src/Covid19Api.Repositories.Abstractions/IGlobalStatisticsWriteRepository.cs b/src/Covid19Api.Repositories.Abstractions/IGlobalStatisticsWriteRepository.cs
@@ -0,0 +1,10 @@
+using System.Threading.Tasks;
+using Covid19Api.Domain;
+
+namespace Covid19Api.Repositories.Abstractions
+{
+    public interface IGlobalStatisticsWriteRepository
+    {
+        Task StoreAsync(GlobalStatistics globalStatistics);
+    }
+}
diff --git a/...positories/CountryStatisticsRepository.cs → ...tories/CountryStatisticsReadRepository.cs b/...positories/CountryStatisticsRepository.cs → ...tories/CountryStatisticsReadRepository.cs
@@ -5,18 +5,17 @@
 using Covid19Api.Domain;
 using Covid19Api.Mongo;
 using Covid19Api.Repositories.Abstractions;
-using Covid19Api.Repositories.Extensions;
 using MongoDB.Driver;
 
 // ReSharper disable SpecifyStringComparison
 
 namespace Covid19Api.Repositories
 {
-    public class CountryStatisticsRepository : ICountryStatisticsRepository
+    public class CountryStatisticsReadRepository : ICountryStatisticsReadRepository
     {
         private readonly Covid19ApiDbContext context;
 
-        public CountryStatisticsRepository(Covid19ApiDbContext context)
+        public CountryStatisticsReadRepository(Covid19ApiDbContext context)
         {
             this.context = context;
         }
@@ -30,9 +29,10 @@ public async Task<CountryStatistic> MostRecentAsync(string country)
                 .Descending(nameof(CountryStatistic.FetchedAt));
 
             // ReSharper disable once SpecifyStringComparison
-            var cursor = await collection.FindAsync(existingCountryStats =>
-                    existingCountryStats.FetchedAt >= DateTime.UtcNow.Date.AddDays(-1) &&
-                    existingCountryStats.Country.ToLower() == country.ToLower(),
+            var cursor = await collection.FindAsync(statistics =>
+                    statistics.FetchedAt >= DateTime.UtcNow.Date.AddDays(-1) &&
+                    statistics.Country.ToLower() == country.ToLower() &&
+                    statistics.Key == CollectionNames.CountryStatistics,
                 new FindOptions<CountryStatistic>
                 {
                     Sort = sort,
@@ -51,7 +51,8 @@ public async Task<IEnumerable<CountryStatistic>> HistoricalAsync(DateTime minFet
                 .Ascending(nameof(CountryStatistic.Country));
 
             var cursor = await collection.FindAsync(
-                existingCountryStats => existingCountryStats.FetchedAt >= minFetchedAt,
+                statistics => statistics.FetchedAt >= minFetchedAt &&
+                              statistics.Key == CollectionNames.CountryStatistics,
                 new FindOptions<CountryStatistic>
                 {
                     Sort = sort,
@@ -77,8 +78,9 @@ public async Task<IEnumerable<CountryStatistic>> HistoricalAsync(DateTime minFet
                 .Ascending(nameof(CountryStatistic.Country));
 
             var cursor = await collection.FindAsync(
-                existingCountryStats => existingCountryStats.FetchedAt >= minFetchedAt &&
-                                        existingCountryStats.Country.ToLowerInvariant() == country.ToLowerInvariant(),
+                statistics => statistics.FetchedAt >= minFetchedAt &&
+                              statistics.Country.ToLowerInvariant() == country.ToLowerInvariant() &&
+                              statistics.Key == CollectionNames.CountryStatistics,
                 new FindOptions<CountryStatistic>
                 {
                     Sort = sort,
@@ -92,8 +94,9 @@ public async Task<IEnumerable<CountryStatistic>> HistoricalForDayAsync(DateTime
             var collection = this.GetCollection();
 
             var cursor = await collection.FindAsync(
-                existingCountryStats => existingCountryStats.FetchedAt >= minFetchedAt &&
-                                        existingCountryStats.Country.ToLowerInvariant() == country.ToLowerInvariant());
+                statistics => statistics.FetchedAt >= minFetchedAt &&
+                              statistics.Country.ToLowerInvariant() == country.ToLowerInvariant() &&
+                              statistics.Key == CollectionNames.CountryStatistics);
 
             return await cursor.ToListAsync();
         }
@@ -115,9 +118,13 @@ public async Task<IEnumerable<CountryStatistic>> HistoricalForDayAsync(DateTime
                 Builders<CountryStatistic>.Filter.Where(
                     statistics => statistics.FetchedAt <= exclusiveEnd);
 
+            var keyFilter =
+                Builders<CountryStatistic>.Filter.Where(statistics =>
+                    statistics.Key == CollectionNames.CountryStatistics);
+
             var sort = Builders<CountryStatistic>.Sort.Descending(statistics => statistics.FetchedAt);
 
-            var filter = countryFilter & startFilter & endFilter;
+            var filter = countryFilter & startFilter & endFilter & keyFilter;
 
             var cursor = await collection.FindAsync(filter, new FindOptions<CountryStatistic>
             {
@@ -127,39 +134,6 @@ public async Task<IEnumerable<CountryStatistic>> HistoricalForDayAsync(DateTime
             return await cursor.FirstOrDefaultAsync();
         }
 
-        public async Task StoreManyAsync(IEnumerable<CountryStatistic> countryStats)
-        {
-            var collection = this.GetCollection();
-
-            var updates = countryStats.Select(currentStats =>
-                {
-                    var filterDefinition =
-                        new FilterDefinitionBuilder<CountryStatistic>().Where(existingStats =>
-                            existingStats.Id == currentStats.Id);
-
-                    return new ReplaceOneModel<CountryStatistic>(filterDefinition, currentStats)
-                    {
-                        IsUpsert = true
-                    };
-                })
-                .ToList();
-
-            foreach (var chunk in updates.CreateChunks(50))
-            {
-                try
-                {
-                    await collection.BulkWriteAsync(chunk, new BulkWriteOptions
-                    {
-                        IsOrdered = false,
-                    });
-                }
-                catch (Exception exception) when (exception is MongoBulkWriteException)
-                {
-                    // Might happen when having duplicate ids!
-                }
-            }
-        }
-
         private IMongoCollection<CountryStatistic> GetCollection()
             => this.context.Database.GetCollection<CountryStatistic>(CollectionNames.CountryStatistics);
     }

diff --git a/src/Covid19Api.Repositories/CountryStatisticsWriteRepository.cs b/src/Covid19Api.Repositories/CountryStatisticsWriteRepository.cs
@@ -0,0 +1,58 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Covid19Api.Domain;
+using Covid19Api.Mongo;
+using Covid19Api.Repositories.Abstractions;
+using Covid19Api.Repositories.Extensions;
+using MongoDB.Driver;
+
+namespace Covid19Api.Repositories
+{
+    public class CountryStatisticsWriteRepository : ICountryStatisticsWriteRepository
+    {
+        private readonly Covid19ApiDbContext context;
+
+        public CountryStatisticsWriteRepository(Covid19ApiDbContext context)
+        {
+            this.context = context;
+        }
+
+        public async Task StoreManyAsync(IEnumerable<CountryStatistic> countryStats)
+        {
+            var collection = this.GetCollection();
+
+            var updates = countryStats.Select(currentStats =>
+                {
+                    var filterDefinition =
+                        new FilterDefinitionBuilder<CountryStatistic>().Where(existingStats =>
+                            existingStats.Id == currentStats.Id);
+
+                    return new ReplaceOneModel<CountryStatistic>(filterDefinition, currentStats)
+                    {
+                        IsUpsert = true
+                    };
+                })
+                .ToList();
+
+            foreach (var chunk in updates.CreateChunks(50))
+            {
+                try
+                {
+                    await collection.BulkWriteAsync(chunk, new BulkWriteOptions
+                    {
+                        IsOrdered = false,
+                    });
+                }
+                catch (Exception exception) when (exception is MongoBulkWriteException)
+                {
+                    // Might happen when having duplicate ids!
+                }
+            }
+        }
+
+        private IMongoCollection<CountryStatistic> GetCollection()
+            => this.context.Database.GetCollection<CountryStatistic>(CollectionNames.CountryStatistics);
+    }
+}
diff --git a/...epositories/GlobalStatisticsRepository.cs → ...itories/GlobalStatisticsReadRepository.cs b/...epositories/GlobalStatisticsRepository.cs → ...itories/GlobalStatisticsReadRepository.cs
@@ -9,26 +9,15 @@
 
 namespace Covid19Api.Repositories
 {
-    public class GlobalStatisticsRepository : IGlobalStatisticsRepository
+    public class GlobalStatisticsReadRepository : IGlobalStatisticsReadRepository
     {
         private readonly Covid19ApiDbContext context;
 
-        public GlobalStatisticsRepository(Covid19ApiDbContext context)
+        public GlobalStatisticsReadRepository(Covid19ApiDbContext context)
         {
             this.context = context;
         }
 
-        public async Task StoreAsync(GlobalStatistics globalStatistics)
-        {
-            var collection = this.GetCollection();
-
-            await collection.ReplaceOneAsync(stats => stats.Id == globalStatistics.Id, globalStatistics,
-                new ReplaceOptions
-                {
-                    IsUpsert = true
-                });
-        }
-
         public async Task<IEnumerable<GlobalStatistics>> HistoricalAsync(DateTime minFetchedAt)
         {
             var collection = this.GetCollection();
@@ -38,7 +27,8 @@ public async Task<IEnumerable<GlobalStatistics>> HistoricalAsync(DateTime minFet
                 .Descending(nameof(GlobalStatistics.FetchedAt));
 
             var cursor = await collection.FindAsync(
-                existingClosedCaseStats => existingClosedCaseStats.FetchedAt >= minFetchedAt,
+                globalStatistics => globalStatistics.FetchedAt >= minFetchedAt &&
+                                    globalStatistics.Key == CollectionNames.GlobalStatistics,
                 new FindOptions<GlobalStatistics>
                 {
                     Sort = sort
@@ -52,7 +42,8 @@ public async Task<IEnumerable<GlobalStatistics>> HistoricalForDayAsync(DateTime
             var collection = this.GetCollection();
 
             var cursor = await collection.FindAsync(
-                existingCountryStats => existingCountryStats.FetchedAt >= minFetchedAt);
+                globalStatistics => globalStatistics.FetchedAt >= minFetchedAt &&
+                                    globalStatistics.Key == CollectionNames.GlobalStatistics);
 
             var all = await cursor.ToListAsync();
 
@@ -65,7 +56,8 @@ public async Task<IEnumerable<GlobalStatistics>> HistoricalForDayAsync(DateTime
 
             var leftFilter = Builders<GlobalStatistics>.Filter.Where(global => global.FetchedAt >= inclusiveStart);
             var rightFilter = Builders<GlobalStatistics>.Filter.Where(global => global.FetchedAt <= inclusiveEnd);
-            var combinedFilter = leftFilter & rightFilter;
+            var keyFilter = Builders<GlobalStatistics>.Filter.Where(global => global.Key == CollectionNames.GlobalStatistics);
+            var combinedFilter = leftFilter & rightFilter & keyFilter;
             var sort = Builders<GlobalStatistics>.Sort.Descending(global => global.FetchedAt);
 
             var cursor = await collection.FindAsync(combinedFilter, new FindOptions<GlobalStatistics>