Rework the scheme migration to use MEDIUMTEXT for the linksetdata fie…

…ld and to serialize/deserialize to/from JSON. We calculate size based on the size of the serialization which means there is a little bit of overhead for each field. We'll also let things go slightly over 128k to simplify the cost accounting but the field we're using can store well beyond 128k so thats not an issue.

OpenSim/Data/MySQL/MySQLSimulationData.cs

@@ -1206,7 +1206,7 @@ private SceneObjectPart BuildPrim(IDataReader row)
 
             if (!(row["linksetdata"] is DBNull))
             {
-                prim.DeserializeLinksetData((byte[])row["linksetdata"]);
+                prim.DeserializeLinksetData((string)row["linksetdata"]);
             }
 
             return prim;

OpenSim/Data/MySQL/OpenSim.Data.MySQL.csproj

@@ -53,6 +53,7 @@
     <PackageReference Include="log4net" Version="2.0.15" />
     <PackageReference Include="Mono.Addins" Version="1.4.1" />
     <PackageReference Include="MySqlConnector" Version="2.2.6" />
+    <PackageReference Include="System.Text.Json" Version="6.0.9" />
   </ItemGroup>
   <ItemGroup>
     <Folder Include="Properties\" />

OpenSim/Data/MySQL/Resources/RegionStore.migrations

@@ -556,5 +556,5 @@ COMMIT;
 
 :VERSION 65         #----- add linkset data storage column
 BEGIN;
-ALTER TABLE `prims` ADD COLUMN `linksetdata` blob default NULL;
+ALTER TABLE `prims` ADD COLUMN `linksetdata` MEDIUMTEXT default NULL;
 COMMIT;

OpenSim/Data/Null/OpenSim.Data.Null.csproj

@@ -23,5 +23,6 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="log4net" Version="2.0.15" />
+    <PackageReference Include="System.Text.Json" Version="6.0.9" />
   </ItemGroup>
 </Project>

OpenSim/Data/OpenSim.Data.csproj

@@ -132,5 +132,6 @@
     <PackageReference Include="Mono.Addins" Version="1.4.1" />
     <PackageReference Include="MySqlConnector" Version="2.2.6" />
     <PackageReference Include="System.Data.SQLite" Version="1.0.116" />
+    <PackageReference Include="System.Text.Json" Version="6.0.9" />
   </ItemGroup>
 </Project>

OpenSim/Data/PGSQL/OpenSim.Data.PGSQL.csproj

@@ -55,6 +55,7 @@
   <ItemGroup>
     <PackageReference Include="log4net" Version="2.0.15" />
     <PackageReference Include="Mono.Addins" Version="1.4.1" />
+    <PackageReference Include="System.Text.Json" Version="6.0.9" />
   </ItemGroup>
   <ItemGroup>
     <Folder Include="Properties\" />

OpenSim/Data/PGSQL/PGSQLSimulationData.cs

@@ -1408,7 +1408,7 @@ private static SceneObjectPart BuildPrim(IDataRecord primRow)
 
             if ((primRow["LinksetData"] is DBNull) == false)
             {
-                prim.DeserializeLinksetData(((byte[])primRow["linksetdata"]));
+                prim.DeserializeLinksetData(((string)primRow["linksetdata"]));
             }
 
             return prim;

OpenSim/Data/PGSQL/Resources/RegionStore.migrations

@@ -1264,5 +1264,5 @@ COMMIT;
 
 :VERSION 53         #----- add linkset data storage column
 BEGIN;
-ALTER TABLE `prims` ADD COLUMN `linksetdata` bytea NULL;
+ALTER TABLE `prims` ADD COLUMN `linksetdata` varchar default NULL;
 COMMIT;

OpenSim/Data/SQLite/OpenSim.Data.SQLite.csproj

@@ -48,5 +48,6 @@
     <PackageReference Include="log4net" Version="2.0.15" />
     <PackageReference Include="Mono.Addins" Version="1.4.1" />
     <PackageReference Include="System.Data.SQLite" Version="1.0.116" />
+    <PackageReference Include="System.Text.Json" Version="6.0.9" />
   </ItemGroup>
 </Project>

OpenSim/Data/SQLite/Resources/RegionStore.migrations

@@ -409,5 +409,5 @@ COMMIT;
 
 :VERSION 41         #----- add linkset data storage column
 BEGIN;
-ALTER TABLE `prims` ADD COLUMN `linksetdata` blob default NULL;
+ALTER TABLE `prims` ADD COLUMN `linksetdata` TEXT default NULL;
 COMMIT;

OpenSim/Data/SQLite/SQLiteSimulationData.cs

@@ -1251,7 +1251,7 @@ private static DataTable createPrimTable()
             createCol(prims, "pseudocrc", typeof(int));
             createCol(prims, "sopanims", typeof(byte[]));
 
-            createCol(prims, "linksetdata", typeof(byte[]));
+            createCol(prims, "linksetdata", typeof(string));
 
             // Add in contraints
             prims.PrimaryKey = new DataColumn[] { prims.Columns["UUID"] };
@@ -1814,7 +1814,7 @@ private SceneObjectPart buildPrim(DataRow row)
 
             if (!(row["linksetdata"] is DBNull))
             {
-                prim.DeserializeLinksetData((byte[])row["LinksetData"]);
+                prim.DeserializeLinksetData((string)row["LinksetData"]);
             }
 
             return prim;

OpenSim/Framework/VersionInfo.cs

@@ -39,7 +39,7 @@ public class VersionInfo
     {
         public const string VersionNumber = "0.9.2.2";
         public const string AssemblyVersionNumber = "0.9.2.2";
-        public const string Release = "8717";
+        public const string Release = "8719";
 
         public static string Version
         {

OpenSim/Region/Framework/Scenes/LinksetDataEntry.cs

@@ -5,24 +5,20 @@ namespace OpenSim.Region.Framework.Scenes
 {
     public class LinksetDataEntry
     {
-        public LinksetDataEntry(string value, string pass)
+         public LinksetDataEntry(string value, string password)
         {
             this.Value = value;
-            this.Password = pass;
+            this.Password = password;
         }
 
-        private LinksetDataEntry()
-        { }
+        public string Value { get; private set; }
+        public string Password { get; private set; } = string.Empty;
 
-        public bool IsProtected
+        public bool IsProtected()
         {
-            get { return (string.IsNullOrEmpty(this.Password) == false); }
+            return (string.IsNullOrEmpty(this.Password) == false);
         }
-
-        public string Password { get; private set; } = string.Empty;
-
-        public string Value { get; private set; }
-
+
         public bool CheckPassword(string pass)
         {
             // A undocumented caveat for LinksetData appears to be that even for unprotected values, if a pass is provided, it is still treated as protected
@@ -39,30 +35,5 @@ public string CheckPasswordAndGetValue(string pass)
             else 
                 return string.Empty;
         }
-
-        public Byte[] Serialize()
-        {
-            using (MemoryStream ms = new MemoryStream())
-            {
-                using (BinaryWriter bw = new BinaryWriter(ms))
-                {
-                    bw.Write(this.Value);
-                    bw.Write(this.Password);
-                    return ms.ToArray();
-                }
-            }
-        }
-
-        public static LinksetDataEntry Deserialize(Byte[] inf)
-        {
-            LinksetDataEntry pd = new LinksetDataEntry();
-            using (BinaryReader br = new BinaryReader(new MemoryStream(inf)))
-            {
-                pd.Value = br.ReadString();
-                pd.Password = br.ReadString();
-
-                return pd;
-            }
-        }
     }
 }

OpenSim/Region/Framework/Scenes/SceneObjectPart.cs

@@ -33,6 +33,7 @@
 using System.Reflection;
 using System.Threading;
 using System.Text;
+using System.Text.Json;
 using System.Text.RegularExpressions;
 using System.Xml;
 using System.Xml.Serialization;
@@ -46,6 +47,7 @@
 using PermissionMask = OpenSim.Framework.PermissionMask;
 using System.Runtime.InteropServices.WindowsRuntime;
 using System.Data.Entity.ModelConfiguration.Configuration;
+using System.Runtime.Serialization.Json;
 
 namespace OpenSim.Region.Framework.Scenes
 {
@@ -5864,6 +5866,9 @@ public int AddOrUpdateLinksetDataKey(string key, string value, string pass)
                 if (LinksetData == null)
                     LinksetData = new SortedList<string, LinksetDataEntry>();
 
+                if (LinksetDataOverLimit)
+                    return 1;
+
                 LinksetDataEntry entry = null;
                 if (LinksetData.TryGetValue(key, out entry) == true)
                 {
@@ -5880,23 +5885,11 @@ public int AddOrUpdateLinksetDataKey(string key, string value, string pass)
 
                 UpdateLinksetDataAccounting();
 
-                // This isnt right.  
-                if (LinksetDataOverLimit)
-                {
-                    // Abort.
-                    if (entry != null)
-                        LinksetData[key] = entry;
-
-                    UpdateLinksetDataAccounting();
-                    return 1;
-                }
-
                 if (ParentGroup != null)
                     ParentGroup.HasGroupChanged = true;
 
                 return 0;
-            }
-
+            }          
         }
 
         /// <summary>
@@ -5945,6 +5938,7 @@ public int DeleteLinksetDataKey(string key, string pass)
                     return 1;
 
                 LinksetData.Remove(key);
+
                 UpdateLinksetDataAccounting();
 
                 if (ParentGroup != null)
@@ -6064,42 +6058,16 @@ public string[] LinksetDataMultiDelete(string pattern, string pass, out int dele
             }
         }
 
-        /// <summary>
-        /// Adjust the current used space by cost which may be positive or negative.
-        /// </summary>
-        /// <param name="cost">A positive (if adding) or negative (if removing) value affecting used space.</param>
-        public void LinksetDataAccountingDelta(int cost)
-        {
-            linksetDataBytesUsed += cost;
-            linksetDataBytesFree = LINKSETDATA_MAX - linksetDataBytesUsed;
-        }
-
         /// <summary>
         /// Recalculates the amount of memory used by linkset data.
+        /// Caller should be holding the linksetData lock
         /// </summary>
-        public void UpdateLinksetDataAccounting()
+        private void UpdateLinksetDataAccounting()
         {
-            lock (linksetDataLock)
-            {
-                using (MemoryStream ms = new MemoryStream())
-                {
-                    using (BinaryWriter bw = new BinaryWriter(ms))
-                    {
-                        foreach (var val in LinksetData)
-                        {
-                            bw.Write(Encoding.UTF8.GetBytes(val.Key));
-                            bw.Write(Encoding.UTF8.GetBytes(val.Value.Value));
-
-                            // For parity, the pass adds 32 bytes regardless of the length. See LL caveats
-                            if (val.Value.IsProtected) 
-                                bw.Write(new byte[32]);
-                        }
-                    }
+            int charCount = JsonSerializer.Serialize<SortedList<string,LinksetDataEntry>>(LinksetData).Length;
 
-                    linksetDataBytesUsed = ms.ToArray().Length;
-                    linksetDataBytesFree = LINKSETDATA_MAX - linksetDataBytesUsed;
-                }
-            }
+            linksetDataBytesUsed = charCount;
+            linksetDataBytesFree = LINKSETDATA_MAX - linksetDataBytesUsed;
         }
 
         public const int LINKSETDATA_MAX = 131072; // 128 KB
@@ -6134,53 +6102,26 @@ public int LinksetDataKeys
             }
         }
 
-        public Byte[] SerializeLinksetData()
+        public string SerializeLinksetData()
         {
             if (!IsRoot || LinksetData == null)
                 return null;
 
             lock (linksetDataLock)
             {
-                using (var ms = new MemoryStream())
-                {
-                    using (BinaryWriter bw = new BinaryWriter(ms))
-                    {
-                        bw.Write(LinksetData.Count);
-                        foreach (KeyValuePair<String, LinksetDataEntry> kvp in LinksetData)
-                        {
-                            bw.Write(kvp.Key);
-                            Byte[] prot = kvp.Value.Serialize();
-
-                            bw.Write(prot.Length);
-                            bw.Write(prot);
-                        }
-
-                        return ms.ToArray();
-                    }
-                }
+                return JsonSerializer.Serialize<SortedList<string, LinksetDataEntry>>(LinksetData);
             }
         }
 
-        public void DeserializeLinksetData(Byte[] data)
+        public void DeserializeLinksetData(string data)
         {
             if (data == null || data.Length == 0)
                 return;
 
-            using (MemoryStream ms = new MemoryStream(data))
+            lock (linksetDataLock)
             {
-                using (BinaryReader br = new BinaryReader(ms))
-                {
-                    LinksetData = new SortedList<string, LinksetDataEntry>();
-
-                    int count = br.ReadInt32();
-                    while (count > 0)
-                    {
-                        LinksetData.Add(br.ReadString(), LinksetDataEntry.Deserialize(br.ReadBytes(br.ReadInt32())));
-                        count--;
-                    }
-
-                    UpdateLinksetDataAccounting();
-                }
+                LinksetData = JsonSerializer.Deserialize<SortedList<string, LinksetDataEntry>>(data);
+                UpdateLinksetDataAccounting();
             }
         }
 

OpenSim/Region/Framework/Scenes/Serialization/SceneObjectSerializer.cs

@@ -837,12 +837,11 @@ private static void ProcessLinksetData(SceneObjectPart obj, XmlReader reader)
         {
             try
             {
-                string b64 = reader.ReadElementContentAsString();
-                if (string.IsNullOrEmpty(b64))
+                string data = reader.ReadElementContentAsString();
+                if (string.IsNullOrEmpty(data))
                     return;
 
-                byte[] bArr = Convert.FromBase64String(b64);
-                obj.DeserializeLinksetData(bArr);
+                obj.DeserializeLinksetData(data);
             }
             catch 
             {
@@ -1689,8 +1688,8 @@ public static void SOPToXml2(XmlTextWriter writer, SceneObjectPart sop, Dictiona
             }
             if(Math.Abs(sop.SitActiveRange) > 1e-5)
                 writer.WriteElementString("SitActRange", sop.SitActiveRange.ToString(Culture.FormatProvider));
-            
-            WriteBytes(writer, "LinksetData", sop.SerializeLinksetData());    
+
+            writer.WriteElementString("LinksetData", sop.SerializeLinksetData());  
 
             writer.WriteEndElement();
         }

addon-modules/OpenSim.Data.MySQL.MoneyData/OpenSim.Data.MySQL.MoneyData/OpenSim.Data.MySQL.MoneyData.csproj

@@ -10,6 +10,7 @@
     <PackageReference Include="log4net" Version="2.0.15" />
     <PackageReference Include="Mono.Addins" Version="1.4.1" />
     <PackageReference Include="MySqlConnector" Version="2.2.6" />
+    <PackageReference Include="System.Text.Json" Version="6.0.9" />
   </ItemGroup>
   <ItemGroup>
     <Reference Include="Nini">