Skip to content

Commit

Permalink
RDBCL-3407 : server wide backups that are defined using a backup-scri…
Browse files Browse the repository at this point in the history 
…pt should use database name (instead of shard names) when generating backup folder name
  • Loading branch information
aviv committed Apr 14, 2024
1 parent d7a96a4 commit eff52b7
Show file tree
Hide file tree
Showing 3 changed files with 257 additions and 2 deletions.
3 changes: 2 additions & 1 deletion src/Raven.Server/ServerWide/Commands/PutServerWideBackupConfigurationCommand.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using Raven.Client.ServerWide.Operations.Configuration;
using Raven.Server.Rachis;
using Raven.Server.ServerWide.Context;
using Raven.Server.Utils;
using Sparrow.Json;
using Sparrow.Json.Parsing;

Expand Down Expand Up @@ -212,7 +213,7 @@ private static string GetUpdatedPath(string str, string databaseName, char separ
if (str.EndsWith(separator) == false)
str += separator;

return str + databaseName;
return str + ShardHelper.ToDatabaseName(databaseName);
}
}
}
131 changes: 131 additions & 0 deletions test/SlowTests/Sharding/Backup/ShardedBackupTests .cs
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Raven.Client.Documents;
using Raven.Client.Documents.Commands;
using Raven.Client.Documents.Operations.Backups;
Expand All @@ -17,13 +19,16 @@
using Raven.Server.Documents.Sharding;
using Raven.Server.ServerWide;
using Raven.Server.ServerWide.Context;
using Raven.Server.Utils;
using Raven.Tests.Core.Utils.Entities;
using Sparrow.Json;
using Sparrow.Platform;
using Tests.Infrastructure;
using Tests.Infrastructure.Entities;
using Xunit;
using Xunit.Abstractions;
using BackupTask = Raven.Server.Documents.PeriodicBackup.BackupTask;
using Directory = System.IO.Directory;

namespace SlowTests.Sharding.Backup
{
Expand Down Expand Up @@ -278,6 +283,108 @@ await store1.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOp
}
}

[RavenTheory(RavenTestCategory.BackupExportImport | RavenTestCategory.Sharding)]
[RavenData(DatabaseMode = RavenDatabaseMode.All)]
public async Task CanBackupShardedServerWide_UsingScript(Options options)
{
DoNotReuseServer();

const string usersPrefix = "Users";
const string ordersPrefix = "Orders";

var backupPath = NewDataPath(suffix: "_BackupFolder");

using (var store1 = Sharding.GetDocumentStore())
using (var store2 = GetDocumentStore())
{
// generate data on store1 and store2
using (var session1 = store1.OpenAsyncSession())
using (var session2 = store2.OpenAsyncSession())
{
for (int i = 0; i < 100; i++)
{
await session1.StoreAsync(new User(), $"{usersPrefix}/{i}");
await session2.StoreAsync(new Order(), $"{ordersPrefix}/{i}");
}

await session1.SaveChangesAsync();
await session2.SaveChangesAsync();
}

// use backup configuration script for local settings
var scriptPath = GenerateConfigurationScript(backupPath, out var command);
var config = new ServerWideBackupConfiguration
{
FullBackupFrequency = "0 0 1 1 *",
Disabled = false,
LocalSettings = new LocalSettings
{
FolderPath = backupPath,
GetBackupConfigurationScript = new GetBackupConfigurationScript
{
Exec = command,
Arguments = scriptPath
}
}
};

// define server wide backup
await store1.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(config));

// wait for backups to complete
var backupsDone = await Sharding.Backup.WaitForBackupToComplete(store1);
var backupsDone2 = await Backup.WaitForBackupToComplete(store2);

foreach (var store in new[] { store1, store2 })
{
var databaseRecord = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
Assert.Equal(1, databaseRecord.PeriodicBackups.Count);

var taskId = databaseRecord.PeriodicBackups[0].TaskId;
if (databaseRecord.IsSharded)
await Sharding.Backup.RunBackupAsync(store.Database, taskId, isFullBackup: true);
else
await Backup.RunBackupAsync(Server, taskId, store, isFullBackup: true);
}

Assert.True(WaitHandle.WaitAll(backupsDone, TimeSpan.FromMinutes(1)));
Assert.True(WaitHandle.WaitAll(backupsDone2, TimeSpan.FromMinutes(1)));

// one backup folder per database
var dirs = Directory.GetDirectories(backupPath);
Assert.Equal(2, dirs.Length);

// should have one root folder for all shards
Assert.Contains(store1.Database, dirs[0]);
Assert.DoesNotContain('$', dirs[0]);

var store1Backups = Directory.GetDirectories(Path.Combine(backupPath, store1.Database));
var store2Backup = Directory.GetDirectories(Path.Combine(backupPath, store2.Database));

Assert.Equal(3, store1Backups.Length); // one per shard
Assert.Single(store2Backup);

Assert.Contains(ShardHelper.ToShardName(store1.Database, 0), store1Backups[0]);
Assert.Contains(ShardHelper.ToShardName(store1.Database, 1), store1Backups[1]);
Assert.Contains(ShardHelper.ToShardName(store1.Database, 2), store1Backups[2]);

// import data to new stores and assert
using (var store3 = GetDocumentStore(options))
using (var store4 = GetDocumentStore(options))
{
foreach (var dir in store1Backups)
{
await store3.Smuggler.ImportIncrementalAsync(new DatabaseSmugglerImportOptions(), dir);
}

await store4.Smuggler.ImportIncrementalAsync(new DatabaseSmugglerImportOptions(), store2Backup[0]);

await AssertDocs(store3, idPrefix: usersPrefix, dbMode: options.DatabaseMode);
await AssertDocs(store4, idPrefix: ordersPrefix, dbMode: options.DatabaseMode);
}
}
}

[RavenTheory(RavenTestCategory.BackupExportImport | RavenTestCategory.Sharding)]
[RavenData(DatabaseMode = RavenDatabaseMode.All)]
public async Task OneTimeBackupSharded(Options options)
Expand Down Expand Up @@ -720,5 +827,29 @@ private async Task AssertDocsInShardedDb(IDocumentStore store, Dictionary<int, L
}
}

private static string GenerateConfigurationScript(string path, out string command)
{
var scriptPath = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".ps1"));
var localSetting = new LocalSettings { FolderPath = path };
var localSettingsString = JsonConvert.SerializeObject(localSetting);

string script;
if (PlatformDetails.RunningOnPosix)
{
command = "bash";
script = $"#!/bin/bash\r\necho '{localSettingsString}'";
File.WriteAllText(scriptPath, script);
Process.Start("chmod", $"700 {scriptPath}");
}
else
{
command = "powershell";
script = $"echo '{localSettingsString}'";
File.WriteAllText(scriptPath, script);
}

return scriptPath;
}

}
}
125 changes: 124 additions & 1 deletion test/SlowTests/Sharding/Backup/ShardedRestoreBackupTests.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net.Http;
Expand All @@ -15,6 +16,7 @@
using Raven.Client.Exceptions;
using Raven.Client.ServerWide;
using Raven.Client.ServerWide.Operations;
using Raven.Client.ServerWide.Operations.Configuration;
using Raven.Client.ServerWide.Sharding;
using Raven.Server.Config;
using Raven.Server.Documents.PeriodicBackup.Aws;
Expand All @@ -27,6 +29,7 @@
using Raven.Server.ServerWide.Context;
using Raven.Server.Utils;
using Raven.Tests.Core.Utils.Entities;
using Sparrow.Platform;
using Tests.Infrastructure;
using Tests.Infrastructure.Entities;
using Xunit;
Expand All @@ -40,7 +43,7 @@ public class ShardedRestoreBackupTests : ClusterTestBase
private readonly string _restoreFromS3TestsPrefix = $"sharding/tests/backup-restore/{nameof(ShardedRestoreBackupTests)}-{Guid.NewGuid()}";
private readonly string _restoreFromAzureTestsPrefix = $"sharding/tests/backup-restore/{nameof(ShardedRestoreBackupTests)}-{Guid.NewGuid()}";
private readonly string _restoreFromGoogleCloudTestsPrefix = $"sharding/tests/backup-restore/{nameof(ShardedRestoreBackupTests)}-{Guid.NewGuid()}";
private static readonly BackupConfiguration DefaultBackupConfiguration;
internal static readonly BackupConfiguration DefaultBackupConfiguration;

static ShardedRestoreBackupTests()
{
Expand Down Expand Up @@ -1684,6 +1687,102 @@ public async Task CanRestoreShardedDatabase_UsingRestorePoint_FromGoogleCloudBac
await DeleteObjects(googleCloudSettings);
}
}

[AmazonS3RetryFact]
public async Task CanRestoreShardedDatabase_FromServerWideBackup()
{
var s3Settings = GetS3Settings();

try
{
DoNotReuseServer();

using (var store = Sharding.GetDocumentStore())
{
await Sharding.Backup.InsertData(store);

// use backup configuration script for S3 settings
var scriptPath = GenerateConfigurationScriptForS3(s3Settings, out var command);
var serverWideConfig = new ServerWideBackupConfiguration
{
FullBackupFrequency = "0 0 1 1 *",
Disabled = false,
S3Settings = new S3Settings
{
GetBackupConfigurationScript = new GetBackupConfigurationScript
{
Exec = command,
Arguments = scriptPath
}
}
};

// define server wide backup
await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(serverWideConfig));

// wait for backup to complete
var backupsDone = await Sharding.Backup.WaitForBackupToComplete(store);

var databaseRecord = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
Assert.Equal(1, databaseRecord.PeriodicBackups.Count);

var taskId = databaseRecord.PeriodicBackups[0].TaskId;
await Sharding.Backup.RunBackupAsync(store.Database, taskId, isFullBackup: true);

Assert.True(WaitHandle.WaitAll(backupsDone, TimeSpan.FromMinutes(1)));

var sharding = await Sharding.GetShardingConfigurationAsync(store);

ShardedRestoreSettings settings;
using (var s3Client = new RavenAwsS3Client(s3Settings, ShardedRestoreBackupTests.DefaultBackupConfiguration))
{
var prefix = $"{s3Settings.RemoteFolderName}/";
var cloudObjects = await s3Client.ListObjectsAsync(prefix, "/", listFolders: true);

// should have one root folder for all shards
Assert.Equal(1, cloudObjects.FileInfoDetails.Count);

var rootFolderName = cloudObjects.FileInfoDetails[0].FullPath;

Assert.EndsWith($"{store.Database}/", rootFolderName);
Assert.DoesNotContain('$', rootFolderName);

var shardsBackupFolders = await s3Client.ListObjectsAsync(rootFolderName, "/", listFolders: true);

// one backup folder per shard
Assert.Equal(3, shardsBackupFolders.FileInfoDetails.Count);

var backupPaths = shardsBackupFolders.FileInfoDetails.Select(x => x.FullPath).ToList();

Assert.Contains(ShardHelper.ToShardName(store.Database, 0), backupPaths[0]);
Assert.Contains(ShardHelper.ToShardName(store.Database, 1), backupPaths[1]);
Assert.Contains(ShardHelper.ToShardName(store.Database, 2), backupPaths[2]);

settings = Sharding.Backup.GenerateShardRestoreSettings(backupPaths, sharding);
}

var restoredDatabaseName = $"restored_database-{Guid.NewGuid()}";
using (Backup.RestoreDatabaseFromCloud(store, new RestoreFromS3Configuration
{
DatabaseName = restoredDatabaseName,
ShardRestoreSettings = settings,
Settings = s3Settings
}, timeout: TimeSpan.FromSeconds(60)))
{
var dbRec = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(restoredDatabaseName));
Assert.Equal(3, dbRec.Sharding.Shards.Count);

await Sharding.Backup.CheckData(store, RavenDatabaseMode.Sharded, expectedRevisionsCount: 16, database: restoredDatabaseName);

}
}
}
finally
{
await DeleteObjects(s3Settings);
}

}

private static string GetDirectoryName(string path)
{
Expand Down Expand Up @@ -1831,5 +1930,29 @@ private static async Task DeleteObjects(GoogleCloudSettings settings)
}
}

internal static string GenerateConfigurationScriptForS3(S3Settings settings, out string command)
{
var scriptPath = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".ps1"));
//var s3Settings = s
var s3SettingsString = JsonConvert.SerializeObject(settings);

string script;
if (PlatformDetails.RunningOnPosix)
{
command = "bash";
script = $"#!/bin/bash\r\necho '{s3SettingsString}'";
File.WriteAllText(scriptPath, script);
Process.Start("chmod", $"700 {scriptPath}");
}
else
{
command = "powershell";
script = $"echo '{s3SettingsString}'";
File.WriteAllText(scriptPath, script);
}

return scriptPath;
}

}
}

0 comments on commit eff52b7

Please sign in to comment.