Skip to content

Commit

Permalink
Fix up some migrations and ensure null trimming from old database
Browse files Browse the repository at this point in the history
  • Loading branch information
@Blackburn29
Blackburn29 committed Jan 10, 2024
1 parent ce4351c commit a129d00
Show file tree
Hide file tree
Showing 11 changed files with 1,417 additions and 33 deletions.
8 changes: 4 additions & 4 deletions src/Fragment.NetSlum.Console/Commands/MigrateBbsCommand.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ private void MigrateBbs(ProgressContext ctx)
var mappedThread = new BbsThread
{
Id = (ushort)existingThread.ThreadId,
Title = existingThread.ThreadTitle.AsSpan().ToShiftJisString(),
Title = existingThread.ThreadTitle.AsSpan().ToShiftJisString().TrimNull(),
CategoryId = (ushort)existingThread.CategoryId,
};

Expand Down Expand Up @@ -86,21 +86,21 @@ private void MigrateBbs(ProgressContext ctx)
{
Id = oldPost.PostId,
Thread = mappedThread,
Title = oldPost.Title.AsSpan().ToShiftJisString(),
Title = oldPost.Title.AsSpan().ToShiftJisString().TrimNull(),
PostedBy = _database.Characters.First(c => c.CharacterName == charName),
CreatedAt = oldPost.Date,
PostContent = new BbsPostContent
{
Id = (ushort)(oldContentRecord?.PostBodyId ?? 0),
Content = (oldContentRecord?.PostBody.AsSpan().ToShiftJisString()) ?? "",
Content = oldContentRecord?.PostBody.AsSpan().ToShiftJisString().TrimNull() ?? "",
},
};

var postContent = new BbsPostContent
{
Id = (ushort)(oldContentRecord?.PostBodyId ?? 0),
Post = mappedPost,
Content = oldContentRecord?.PostBody.AsSpan().ToShiftJisString() ?? "",
Content = oldContentRecord?.PostBody.AsSpan().ToShiftJisString().TrimNull() ?? "",
};

_database.Add(postContent);
Expand Down
4 changes: 2 additions & 2 deletions src/Fragment.NetSlum.Console/Commands/MigrateGuildsCommand.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@ private void MigrateGuilds(ProgressContext ctx)
var mappedGuild = new Guild
{
Id = (ushort)existingGuild.GuildId,
Name = existingGuild.GuildName!.AsSpan().ToShiftJisString(),
Comment = existingGuild.GuildComment!.AsSpan().ToShiftJisString(),
Name = existingGuild.GuildName!.AsSpan().ToShiftJisString().TrimNull(),
Comment = existingGuild.GuildComment!.AsSpan().ToShiftJisString().TrimNull(),
Emblem = existingGuild.GuildEmblem!,
Stats = new GuildStats
{
Expand Down
8 changes: 4 additions & 4 deletions src/Fragment.NetSlum.Console/Commands/MigrateMailCommand.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ private void MigrateMail(ProgressContext ctx)
Content = new MailContent
{
Id = oldContent.MailBodyId,
Content = oldContent.MailBody1.AsMemory().ToShiftJisString(),
Content = oldContent.MailBody1.AsMemory().ToShiftJisString().TrimNull(),
MailId = oldContent.MailId,
}
})
Expand All @@ -69,14 +69,14 @@ private void MigrateMail(ProgressContext ctx)
Id = mailMeta.MailId,
CreatedAt = mailMeta.Date,
Recipient = _database.PlayerAccounts.FirstOrDefault(pa => pa.Id == mailMeta.ReceiverAccountId),
RecipientName = mailMeta.ReceiverName.AsSpan().ToShiftJisString(),
RecipientName = mailMeta.ReceiverName.AsSpan().ToShiftJisString().TrimNull(),
Sender = _database.PlayerAccounts.FirstOrDefault(pa => pa.Id == mailMeta.SenderAccountId),
SenderName = mailMeta.SenderName.AsSpan().ToShiftJisString(),
SenderName = mailMeta.SenderName.AsSpan().ToShiftJisString().TrimNull(),
Read = true, // This field does not exist on the current database, default it to true for the migration
Delivered = mailMeta.MailDelivered!.Value,
AvatarId = mappedContent?.AvatarId ?? "",
Content = mappedContent?.Content,
Subject = mailMeta.MailSubject.AsSpan().ToShiftJisString(),
Subject = mailMeta.MailSubject.AsSpan().ToShiftJisString().TrimNull(),
};

_database.Add(mappedMail);
Expand Down
134 changes: 117 additions & 17 deletions src/Fragment.NetSlum.Console/Commands/MigratePlayersCommand.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System;
using System.Globalization;
using System.Linq;
using System.Threading.Tasks;
using Fragment.NetSlum.Core.Constants;
Expand Down Expand Up @@ -28,18 +29,26 @@ public MigratePlayersCommand(FragmentContext database, OldFragmentContext oldDat

public override Task<int> ExecuteAsync(CommandContext context, Settings settings)
{
/*
AnsiConsole.Progress()
.AutoClear(false)
.Columns(new TaskDescriptionColumn(), new ProgressBarColumn(), new PercentageColumn(), new RemainingTimeColumn(),
new SpinnerColumn())
.Start(MigratePlayerRecords);
*/

AnsiConsole.Progress()
.AutoClear(false)
.Columns(new TaskDescriptionColumn(), new ProgressBarColumn(), new PercentageColumn(), new RemainingTimeColumn(),
new SpinnerColumn())
.Start(MigrateCharacterRecords);

AnsiConsole.Progress()
.AutoClear(false)
.Columns(new TaskDescriptionColumn(), new ProgressBarColumn(), new PercentageColumn(), new RemainingTimeColumn(),
new SpinnerColumn())
.Start(MigrateCharacterStatHistory);


return Task.FromResult(0);
}
Expand All @@ -62,7 +71,8 @@ private void MigrateCharacterRecords(ProgressContext ctx)

if (oldPlayerRecord == null)
{
AnsiConsole.MarkupLine("[orange3]Unable to find old player record ID {0} for character {1}. Was the table migrated?[/]", existingCharacter.AccountId!, existingCharacter.CharacterName!.AsSpan().ToShiftJisString());
AnsiConsole.MarkupLine("[orange3]Unable to find old player record ID {0} for character {1}. Was the table migrated?[/]",
existingCharacter.AccountId!, existingCharacter.CharacterName!.AsSpan().ToShiftJisString());
pTask.Increment(1);
continue;
}
Expand All @@ -71,20 +81,22 @@ private void MigrateCharacterRecords(ProgressContext ctx)

if (newPlayerRecord == null)
{
AnsiConsole.MarkupLine("[orange3]Unable to find new player record ID {0} for character {1}. Was the table migrated?[/]", existingCharacter.AccountId!, existingCharacter.CharacterName!.AsSpan().ToShiftJisString());
AnsiConsole.MarkupLine("[orange3]Unable to find new player record ID {0} for character {1}. Was the table migrated?[/]",
existingCharacter.AccountId!, existingCharacter.CharacterName!.AsSpan().ToShiftJisString());
pTask.Increment(1);
continue;
}

var mappedCharacter = new Character
{
Id = existingCharacter.PlayerId,
CharacterName = existingCharacter.CharacterName.AsSpan().ToShiftJisString(),
SaveId = existingCharacter.CharacterSaveId!.TrimNull(),
CharacterName = existingCharacter.CharacterName.AsSpan().ToShiftJisString().TrimNull(),
Class = (CharacterClass)existingCharacter.ClassId!,
CurrentLevel = existingCharacter.CharacterLevel!.Value,
GreetingMessage = existingCharacter.Greeting.AsSpan().ToShiftJisString().Replace("\r\n", "\n"),
GreetingMessage = existingCharacter.Greeting.AsSpan().ToShiftJisString().Replace("\r\n", "\n").TrimNull(),
FullModelId = (uint)existingCharacter.ModelNumber!.Value,
CharacterStats = new()
CharacterStats = new CharacterStats
{
Id = exists?.CharacterStats.Id ?? 0,
CharacterId = existingCharacter.PlayerId,
Expand All @@ -110,6 +122,7 @@ private void MigrateCharacterRecords(ProgressContext ctx)
{
_database.Add(mappedCharacter);
}

_database.SaveChanges();
_database.ChangeTracker.Clear();
}
Expand All @@ -120,10 +133,107 @@ private void MigrateCharacterRecords(ProgressContext ctx)
AnsiConsole.WriteException(e);
continue;
}

pTask.Increment(1);
}
}

private void MigrateCharacterStatHistory(ProgressContext ctx)
{
var existingStats = _oldDatabase.RankingData
.AsNoTracking()
.Where(s => s.AccountId > 0)
.OrderBy(s => s.AccountId);

var characterCount = existingStats.Count() / 5000;
var pTask = ctx.AddTask($"[bold yellow] Generating {characterCount} statistic records[/]", maxValue: characterCount);

var count = 0;
var currentAccountId = -1;
foreach (var stat in existingStats)
{
if (currentAccountId != stat.AccountId && !_database.PlayerAccounts.AsNoTracking().Any(c => c.Id == stat.AccountId))
{
AnsiConsole.WriteLine($"Account ID {stat.AccountId} does not exist");
//pTask.Increment(1);
count += 1;
continue;
}

if (currentAccountId != stat.AccountId)
{
AnsiConsole.WriteLine($"AccountID changed! {currentAccountId} -> {stat.AccountId}");
currentAccountId = stat.AccountId;
}

// The ranking database uses multiple different date formats for whatever reason so we need to perform some jank here to get the
// right format.
var loginTime = DateTime.MinValue;

try
{
loginTime = DateTime.ParseExact(stat.LoginTime, "ddd MMM d HH:mm:ss yyyy", CultureInfo.InvariantCulture);
}
catch (FormatException)
{
try
{
loginTime = DateTime.ParseExact(stat.LoginTime, "ddd MMM d HH:mm:ss yyyy", CultureInfo.InvariantCulture);
}
catch (FormatException)
{
AnsiConsole.WriteLine($"Failed to parse loginTime {stat.LoginTime}");
continue;
}
}

stat.CharacterSaveId = stat.CharacterSaveId.TrimNull();

int? existingCharacterId = _database.Characters.AsNoTracking()
.FirstOrDefault(c => c.SaveSlotId.Equals(stat.CharacterSaveId))?.Id;

if (existingCharacterId == null)
{
AnsiConsole.WriteLine($"No character with name {stat.CharacterName} -- '{stat.CharacterSaveId}'");
count += 1;
continue;
}

var mappedCharacter = new CharacterStatHistory
{
Id = stat.Id,
CharacterId = existingCharacterId!.Value,
CurrentHp = stat.CharacterHp,
CurrentGp = stat.CharacterGp,
CurrentSp = stat.CharacterSp,
AverageFieldLevel = stat.AverageFieldLevel,
OnlineTreasures = (int)stat.GodStatueCounterOnline,
CreatedAt = loginTime,
};

if (_database.CharacterStatHistory.AsNoTracking().Any(sh => sh.Id == mappedCharacter.Id))
{

_database.CharacterStatHistory.Update(mappedCharacter);
}
else
{
_database.CharacterStatHistory.Add(mappedCharacter);
}
count += 1;

if (count % 5000 == 0)
{
_database.SaveChanges();
_database.ChangeTracker.Clear();
pTask.Increment(1);
}
}

_database.SaveChanges();
_database.ChangeTracker.Clear();
}

private void MigratePlayerRecords(ProgressContext ctx)
{
var existingPlayers = _oldDatabase.PlayerAccountIds;
Expand All @@ -150,30 +260,20 @@ private void MigratePlayerRecords(ProgressContext ctx)
var mappedPlayer = new PlayerAccount
{
Id = (ushort)existingPlayer.Id,
SaveId = existingPlayer.Saveid,
SaveId = existingPlayer.Saveid.TrimNull(),
};

_database.Add(mappedPlayer);
pTask.Increment(1);

if (pTask.Value % 100 != 0)
{
continue;
}

AnsiConsole.MarkupLine("[cyan]Persisting 100 records to the database...[/]");
_database.SaveChanges();
_database.ChangeTracker.Clear();
pTask.Increment(1);
}
catch (Exception e)
{
AnsiConsole.WriteException(e, ExceptionFormats.ShortenEverything);
}
}

AnsiConsole.MarkupLine("[cyan]Flushing all remaining records...[/]");
_database.SaveChanges();
_database.ChangeTracker.Clear();
AnsiConsole.MarkupLine("[green]Done updating players![/]");
}
}
10 changes: 10 additions & 0 deletions src/Fragment.NetSlum.Core/Extensions/StringExtensions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,4 +102,14 @@ public static string MaxSubstring(this string str, int size)
{
return str.Length > size ? str[..size] : str;
}

/// <summary>
/// Strips all NUL characters from the given string
/// </summary>
/// <param name="str"></param>
/// <returns></returns>
public static string TrimNull(this string str)
{
return str.Replace("\0", string.Empty);
}
}
2 changes: 2 additions & 0 deletions src/Fragment.NetSlum.Persistence/Entities/Character.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ public class Character : IConfigurableEntity<Character>
[MySqlCollation("cp932_japanese_ci")]
public required string GreetingMessage { get; set; }

[MaxLength(32)]
public string SaveId { get; set; } = default!;
public byte SaveSlotId { get; set; }

public required int CurrentLevel { get; set; }
Expand Down
10 changes: 6 additions & 4 deletions src/Fragment.NetSlum.Persistence/Entities/Deprecated/RankingDatum.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ public class RankingDatum
[Column("ID")]
public int Id { get; set; }

[Column(TypeName = "tinytext")]
[Column("AntiCheatEngineResult", TypeName = "tinytext")]
public string AntiCheatEngineResult { get; set; } = null!;

[Column("loginTime", TypeName = "text")]
Expand All @@ -25,12 +25,13 @@ public class RankingDatum
[Column("CharacterSaveID", TypeName = "text")]
public string CharacterSaveId { get; set; } = null!;

[Column(TypeName = "text")]
[Column("CharacterName", TypeName = "text")]
public string CharacterName { get; set; } = null!;

[Column("CharacterLevel")]
public byte CharacterLevel { get; set; }

[Column(TypeName = "text")]
[Column("CharacterClassName", TypeName = "text")]
public string CharacterClassName { get; set; } = null!;

[Column("CharacterHP")]
Expand All @@ -42,9 +43,10 @@ public class RankingDatum
[Column("CharacterGP", TypeName = "mediumint unsigned")]
public uint CharacterGp { get; set; }

[Column(TypeName = "mediumint unsigned")]
[Column("GodStatueCounterOnline", TypeName = "mediumint unsigned")]
public uint GodStatueCounterOnline { get; set; }

[Column("AverageFieldLevel")]
public byte AverageFieldLevel { get; set; }

[Column("AccountID")]
Expand Down
Loading

0 comments on commit a129d00

Please sign in to comment.