From 1ab558d56134c00f865dd49f91a620acedfe1c39 Mon Sep 17 00:00:00 2001 From: Mike Dickson Date: Wed, 15 Nov 2023 21:52:35 -0500 Subject: [PATCH] Rework the scheme migration to use MEDIUMTEXT for the linksetdata field and to serialize/deserialize to/from JSON. We calculate size based on the size of the serialization which means there is a little bit of overhead for each field. We'll also let things go slightly over 128k to simplify the cost accounting but the field we're using can store well beyond 128k so thats not an issue. --- OpenSim/Data/MySQL/MySQLSimulationData.cs | 2 +- OpenSim/Data/MySQL/OpenSim.Data.MySQL.csproj | 1 + .../MySQL/Resources/RegionStore.migrations | 2 +- OpenSim/Data/Null/OpenSim.Data.Null.csproj | 1 + OpenSim/Data/OpenSim.Data.csproj | 1 + OpenSim/Data/PGSQL/OpenSim.Data.PGSQL.csproj | 1 + OpenSim/Data/PGSQL/PGSQLSimulationData.cs | 2 +- .../PGSQL/Resources/RegionStore.migrations | 2 +- .../Data/SQLite/OpenSim.Data.SQLite.csproj | 1 + .../SQLite/Resources/RegionStore.migrations | 2 +- OpenSim/Data/SQLite/SQLiteSimulationData.cs | 4 +- OpenSim/Framework/VersionInfo.cs | 2 +- .../Framework/Scenes/LinksetDataEntry.cs | 43 ++------- .../Framework/Scenes/SceneObjectPart.cs | 95 ++++--------------- .../Serialization/SceneObjectSerializer.cs | 11 +-- .../OpenSim.Data.MySQL.MoneyData.csproj | 1 + 16 files changed, 44 insertions(+), 127 deletions(-) diff --git a/OpenSim/Data/MySQL/MySQLSimulationData.cs b/OpenSim/Data/MySQL/MySQLSimulationData.cs index 0118c6cfe6f..a054e12782f 100644 --- a/OpenSim/Data/MySQL/MySQLSimulationData.cs +++ b/OpenSim/Data/MySQL/MySQLSimulationData.cs @@ -1206,7 +1206,7 @@ private SceneObjectPart BuildPrim(IDataReader row) if (!(row["linksetdata"] is DBNull)) { - prim.DeserializeLinksetData((byte[])row["linksetdata"]); + prim.DeserializeLinksetData((string)row["linksetdata"]); } return prim; diff --git a/OpenSim/Data/MySQL/OpenSim.Data.MySQL.csproj b/OpenSim/Data/MySQL/OpenSim.Data.MySQL.csproj index 6c298076d00..256377d015d 100644 --- a/OpenSim/Data/MySQL/OpenSim.Data.MySQL.csproj +++ b/OpenSim/Data/MySQL/OpenSim.Data.MySQL.csproj @@ -53,6 +53,7 @@ + diff --git a/OpenSim/Data/MySQL/Resources/RegionStore.migrations b/OpenSim/Data/MySQL/Resources/RegionStore.migrations index 853ef267174..da5fc0c6314 100644 --- a/OpenSim/Data/MySQL/Resources/RegionStore.migrations +++ b/OpenSim/Data/MySQL/Resources/RegionStore.migrations @@ -556,5 +556,5 @@ COMMIT; :VERSION 65 #----- add linkset data storage column BEGIN; -ALTER TABLE `prims` ADD COLUMN `linksetdata` blob default NULL; +ALTER TABLE `prims` ADD COLUMN `linksetdata` MEDIUMTEXT default NULL; COMMIT; \ No newline at end of file diff --git a/OpenSim/Data/Null/OpenSim.Data.Null.csproj b/OpenSim/Data/Null/OpenSim.Data.Null.csproj index a108b966ee7..ffe131b3011 100644 --- a/OpenSim/Data/Null/OpenSim.Data.Null.csproj +++ b/OpenSim/Data/Null/OpenSim.Data.Null.csproj @@ -23,5 +23,6 @@ + \ No newline at end of file diff --git a/OpenSim/Data/OpenSim.Data.csproj b/OpenSim/Data/OpenSim.Data.csproj index 9bc07d0f0c9..544a6182152 100644 --- a/OpenSim/Data/OpenSim.Data.csproj +++ b/OpenSim/Data/OpenSim.Data.csproj @@ -132,5 +132,6 @@ + \ No newline at end of file diff --git a/OpenSim/Data/PGSQL/OpenSim.Data.PGSQL.csproj b/OpenSim/Data/PGSQL/OpenSim.Data.PGSQL.csproj index d03a27fdca7..325c5649018 100644 --- a/OpenSim/Data/PGSQL/OpenSim.Data.PGSQL.csproj +++ b/OpenSim/Data/PGSQL/OpenSim.Data.PGSQL.csproj @@ -55,6 +55,7 @@ + diff --git a/OpenSim/Data/PGSQL/PGSQLSimulationData.cs b/OpenSim/Data/PGSQL/PGSQLSimulationData.cs index f390af2a93f..9d715ac712f 100755 --- a/OpenSim/Data/PGSQL/PGSQLSimulationData.cs +++ b/OpenSim/Data/PGSQL/PGSQLSimulationData.cs @@ -1408,7 +1408,7 @@ private static SceneObjectPart BuildPrim(IDataRecord primRow) if ((primRow["LinksetData"] is DBNull) == false) { - prim.DeserializeLinksetData(((byte[])primRow["linksetdata"])); + prim.DeserializeLinksetData(((string)primRow["linksetdata"])); } return prim; diff --git a/OpenSim/Data/PGSQL/Resources/RegionStore.migrations b/OpenSim/Data/PGSQL/Resources/RegionStore.migrations index 782e1f4a95b..ea172f13783 100644 --- a/OpenSim/Data/PGSQL/Resources/RegionStore.migrations +++ b/OpenSim/Data/PGSQL/Resources/RegionStore.migrations @@ -1264,5 +1264,5 @@ COMMIT; :VERSION 53 #----- add linkset data storage column BEGIN; -ALTER TABLE `prims` ADD COLUMN `linksetdata` bytea NULL; +ALTER TABLE `prims` ADD COLUMN `linksetdata` varchar default NULL; COMMIT; diff --git a/OpenSim/Data/SQLite/OpenSim.Data.SQLite.csproj b/OpenSim/Data/SQLite/OpenSim.Data.SQLite.csproj index f669650a960..a156e7dbdca 100644 --- a/OpenSim/Data/SQLite/OpenSim.Data.SQLite.csproj +++ b/OpenSim/Data/SQLite/OpenSim.Data.SQLite.csproj @@ -48,5 +48,6 @@ + \ No newline at end of file diff --git a/OpenSim/Data/SQLite/Resources/RegionStore.migrations b/OpenSim/Data/SQLite/Resources/RegionStore.migrations index 21453d9a5bb..84e35c7e548 100644 --- a/OpenSim/Data/SQLite/Resources/RegionStore.migrations +++ b/OpenSim/Data/SQLite/Resources/RegionStore.migrations @@ -409,5 +409,5 @@ COMMIT; :VERSION 41 #----- add linkset data storage column BEGIN; -ALTER TABLE `prims` ADD COLUMN `linksetdata` blob default NULL; +ALTER TABLE `prims` ADD COLUMN `linksetdata` TEXT default NULL; COMMIT; \ No newline at end of file diff --git a/OpenSim/Data/SQLite/SQLiteSimulationData.cs b/OpenSim/Data/SQLite/SQLiteSimulationData.cs index bbb7a407800..30b4c4d3c3b 100644 --- a/OpenSim/Data/SQLite/SQLiteSimulationData.cs +++ b/OpenSim/Data/SQLite/SQLiteSimulationData.cs @@ -1251,7 +1251,7 @@ private static DataTable createPrimTable() createCol(prims, "pseudocrc", typeof(int)); createCol(prims, "sopanims", typeof(byte[])); - createCol(prims, "linksetdata", typeof(byte[])); + createCol(prims, "linksetdata", typeof(string)); // Add in contraints prims.PrimaryKey = new DataColumn[] { prims.Columns["UUID"] }; @@ -1814,7 +1814,7 @@ private SceneObjectPart buildPrim(DataRow row) if (!(row["linksetdata"] is DBNull)) { - prim.DeserializeLinksetData((byte[])row["LinksetData"]); + prim.DeserializeLinksetData((string)row["LinksetData"]); } return prim; diff --git a/OpenSim/Framework/VersionInfo.cs b/OpenSim/Framework/VersionInfo.cs index b0764bca611..12af4d47c66 100644 --- a/OpenSim/Framework/VersionInfo.cs +++ b/OpenSim/Framework/VersionInfo.cs @@ -39,7 +39,7 @@ public class VersionInfo { public const string VersionNumber = "0.9.2.2"; public const string AssemblyVersionNumber = "0.9.2.2"; - public const string Release = "8717"; + public const string Release = "8719"; public static string Version { diff --git a/OpenSim/Region/Framework/Scenes/LinksetDataEntry.cs b/OpenSim/Region/Framework/Scenes/LinksetDataEntry.cs index 375e8236c57..68ab7499728 100644 --- a/OpenSim/Region/Framework/Scenes/LinksetDataEntry.cs +++ b/OpenSim/Region/Framework/Scenes/LinksetDataEntry.cs @@ -5,24 +5,20 @@ namespace OpenSim.Region.Framework.Scenes { public class LinksetDataEntry { - public LinksetDataEntry(string value, string pass) + public LinksetDataEntry(string value, string password) { this.Value = value; - this.Password = pass; + this.Password = password; } - private LinksetDataEntry() - { } + public string Value { get; private set; } + public string Password { get; private set; } = string.Empty; - public bool IsProtected + public bool IsProtected() { - get { return (string.IsNullOrEmpty(this.Password) == false); } + return (string.IsNullOrEmpty(this.Password) == false); } - - public string Password { get; private set; } = string.Empty; - - public string Value { get; private set; } - + public bool CheckPassword(string pass) { // A undocumented caveat for LinksetData appears to be that even for unprotected values, if a pass is provided, it is still treated as protected @@ -39,30 +35,5 @@ public string CheckPasswordAndGetValue(string pass) else return string.Empty; } - - public Byte[] Serialize() - { - using (MemoryStream ms = new MemoryStream()) - { - using (BinaryWriter bw = new BinaryWriter(ms)) - { - bw.Write(this.Value); - bw.Write(this.Password); - return ms.ToArray(); - } - } - } - - public static LinksetDataEntry Deserialize(Byte[] inf) - { - LinksetDataEntry pd = new LinksetDataEntry(); - using (BinaryReader br = new BinaryReader(new MemoryStream(inf))) - { - pd.Value = br.ReadString(); - pd.Password = br.ReadString(); - - return pd; - } - } } } \ No newline at end of file diff --git a/OpenSim/Region/Framework/Scenes/SceneObjectPart.cs b/OpenSim/Region/Framework/Scenes/SceneObjectPart.cs index 4dd6bf11f37..73542da3d93 100644 --- a/OpenSim/Region/Framework/Scenes/SceneObjectPart.cs +++ b/OpenSim/Region/Framework/Scenes/SceneObjectPart.cs @@ -33,6 +33,7 @@ using System.Reflection; using System.Threading; using System.Text; +using System.Text.Json; using System.Text.RegularExpressions; using System.Xml; using System.Xml.Serialization; @@ -46,6 +47,7 @@ using PermissionMask = OpenSim.Framework.PermissionMask; using System.Runtime.InteropServices.WindowsRuntime; using System.Data.Entity.ModelConfiguration.Configuration; +using System.Runtime.Serialization.Json; namespace OpenSim.Region.Framework.Scenes { @@ -5864,6 +5866,9 @@ public int AddOrUpdateLinksetDataKey(string key, string value, string pass) if (LinksetData == null) LinksetData = new SortedList(); + if (LinksetDataOverLimit) + return 1; + LinksetDataEntry entry = null; if (LinksetData.TryGetValue(key, out entry) == true) { @@ -5880,23 +5885,11 @@ public int AddOrUpdateLinksetDataKey(string key, string value, string pass) UpdateLinksetDataAccounting(); - // This isnt right. - if (LinksetDataOverLimit) - { - // Abort. - if (entry != null) - LinksetData[key] = entry; - - UpdateLinksetDataAccounting(); - return 1; - } - if (ParentGroup != null) ParentGroup.HasGroupChanged = true; return 0; - } - + } } /// @@ -5945,6 +5938,7 @@ public int DeleteLinksetDataKey(string key, string pass) return 1; LinksetData.Remove(key); + UpdateLinksetDataAccounting(); if (ParentGroup != null) @@ -6064,42 +6058,16 @@ public string[] LinksetDataMultiDelete(string pattern, string pass, out int dele } } - /// - /// Adjust the current used space by cost which may be positive or negative. - /// - /// A positive (if adding) or negative (if removing) value affecting used space. - public void LinksetDataAccountingDelta(int cost) - { - linksetDataBytesUsed += cost; - linksetDataBytesFree = LINKSETDATA_MAX - linksetDataBytesUsed; - } - /// /// Recalculates the amount of memory used by linkset data. + /// Caller should be holding the linksetData lock /// - public void UpdateLinksetDataAccounting() + private void UpdateLinksetDataAccounting() { - lock (linksetDataLock) - { - using (MemoryStream ms = new MemoryStream()) - { - using (BinaryWriter bw = new BinaryWriter(ms)) - { - foreach (var val in LinksetData) - { - bw.Write(Encoding.UTF8.GetBytes(val.Key)); - bw.Write(Encoding.UTF8.GetBytes(val.Value.Value)); - - // For parity, the pass adds 32 bytes regardless of the length. See LL caveats - if (val.Value.IsProtected) - bw.Write(new byte[32]); - } - } + int charCount = JsonSerializer.Serialize>(LinksetData).Length; - linksetDataBytesUsed = ms.ToArray().Length; - linksetDataBytesFree = LINKSETDATA_MAX - linksetDataBytesUsed; - } - } + linksetDataBytesUsed = charCount; + linksetDataBytesFree = LINKSETDATA_MAX - linksetDataBytesUsed; } public const int LINKSETDATA_MAX = 131072; // 128 KB @@ -6134,53 +6102,26 @@ public int LinksetDataKeys } } - public Byte[] SerializeLinksetData() + public string SerializeLinksetData() { if (!IsRoot || LinksetData == null) return null; lock (linksetDataLock) { - using (var ms = new MemoryStream()) - { - using (BinaryWriter bw = new BinaryWriter(ms)) - { - bw.Write(LinksetData.Count); - foreach (KeyValuePair kvp in LinksetData) - { - bw.Write(kvp.Key); - Byte[] prot = kvp.Value.Serialize(); - - bw.Write(prot.Length); - bw.Write(prot); - } - - return ms.ToArray(); - } - } + return JsonSerializer.Serialize>(LinksetData); } } - public void DeserializeLinksetData(Byte[] data) + public void DeserializeLinksetData(string data) { if (data == null || data.Length == 0) return; - using (MemoryStream ms = new MemoryStream(data)) + lock (linksetDataLock) { - using (BinaryReader br = new BinaryReader(ms)) - { - LinksetData = new SortedList(); - - int count = br.ReadInt32(); - while (count > 0) - { - LinksetData.Add(br.ReadString(), LinksetDataEntry.Deserialize(br.ReadBytes(br.ReadInt32()))); - count--; - } - - UpdateLinksetDataAccounting(); - } + LinksetData = JsonSerializer.Deserialize>(data); + UpdateLinksetDataAccounting(); } } diff --git a/OpenSim/Region/Framework/Scenes/Serialization/SceneObjectSerializer.cs b/OpenSim/Region/Framework/Scenes/Serialization/SceneObjectSerializer.cs index 7ed1cee5459..dbe51a891a8 100644 --- a/OpenSim/Region/Framework/Scenes/Serialization/SceneObjectSerializer.cs +++ b/OpenSim/Region/Framework/Scenes/Serialization/SceneObjectSerializer.cs @@ -837,12 +837,11 @@ private static void ProcessLinksetData(SceneObjectPart obj, XmlReader reader) { try { - string b64 = reader.ReadElementContentAsString(); - if (string.IsNullOrEmpty(b64)) + string data = reader.ReadElementContentAsString(); + if (string.IsNullOrEmpty(data)) return; - byte[] bArr = Convert.FromBase64String(b64); - obj.DeserializeLinksetData(bArr); + obj.DeserializeLinksetData(data); } catch { @@ -1689,8 +1688,8 @@ public static void SOPToXml2(XmlTextWriter writer, SceneObjectPart sop, Dictiona } if(Math.Abs(sop.SitActiveRange) > 1e-5) writer.WriteElementString("SitActRange", sop.SitActiveRange.ToString(Culture.FormatProvider)); - - WriteBytes(writer, "LinksetData", sop.SerializeLinksetData()); + + writer.WriteElementString("LinksetData", sop.SerializeLinksetData()); writer.WriteEndElement(); } diff --git a/addon-modules/OpenSim.Data.MySQL.MoneyData/OpenSim.Data.MySQL.MoneyData/OpenSim.Data.MySQL.MoneyData.csproj b/addon-modules/OpenSim.Data.MySQL.MoneyData/OpenSim.Data.MySQL.MoneyData/OpenSim.Data.MySQL.MoneyData.csproj index 2285e1edf4f..1cd26b1d009 100644 --- a/addon-modules/OpenSim.Data.MySQL.MoneyData/OpenSim.Data.MySQL.MoneyData/OpenSim.Data.MySQL.MoneyData.csproj +++ b/addon-modules/OpenSim.Data.MySQL.MoneyData/OpenSim.Data.MySQL.MoneyData/OpenSim.Data.MySQL.MoneyData.csproj @@ -10,6 +10,7 @@ +