From ce62eb342f5751b88941ef755b72c31131b13675 Mon Sep 17 00:00:00 2001 From: Taylor Southwick Date: Thu, 17 Sep 2020 17:09:45 -0700 Subject: [PATCH] Abstract file access pattern for DSS provider This is an intial change to abstract file access using IPlateTilePyramid. This interface provides a single method that will take a plate file name along with the desired level and coordinates and will retrieve the associated image. This change includes two implementations of this: - ConfigurationManagerFilePlateTilePyramid allows for a request to a given plate file and returns a stream for the level and coordinates specified. - AzurePlateTilePyramid retrieves the file from Azure blob storage This new access method has been incorporated into the DSS.aspx page and will surface the content via Azure or local storage via a configuration flag (requires a restart of the service). --- WWTMVC5/App_Start/UnityConfig.cs | 18 ++++- WWTMVC5/WWTMVC5.csproj | 10 ++- WWTMVC5/Web.config | 4 ++ WWTWebSiteOnly.sln | 6 ++ WWTWebservices.Azure/AzurePlateTilePyramid.cs | 64 +++++++++++++++++ .../WWTWebservices.Azure.csproj | 19 +++++ ...onfigurationManagerFilePlateTilePyramid.cs | 17 +++++ WWTWebservices/IPlateTilePyramid.cs | 9 +++ WWTWebservices/PlateTilePyramid.cs | 3 +- WWTWebservices/WWTWebservices.csproj | 2 +- src/WWT.Providers/Providers/DSSProvider.cs | 71 +++++++++---------- 11 files changed, 175 insertions(+), 48 deletions(-) create mode 100644 WWTWebservices.Azure/AzurePlateTilePyramid.cs create mode 100644 WWTWebservices.Azure/WWTWebservices.Azure.csproj create mode 100644 WWTWebservices/ConfigurationManagerFilePlateTilePyramid.cs create mode 100644 WWTWebservices/IPlateTilePyramid.cs diff --git a/WWTMVC5/App_Start/UnityConfig.cs b/WWTMVC5/App_Start/UnityConfig.cs index 34055b81..ed1e6d1d 100644 --- a/WWTMVC5/App_Start/UnityConfig.cs +++ b/WWTMVC5/App_Start/UnityConfig.cs @@ -1,8 +1,11 @@ +using Azure.Identity; +using Microsoft.Practices.Unity; using System; +using System.Configuration; using System.Linq; -using Microsoft.Practices.Unity; using WWT.Providers; using WWTWebservices; +using WWTWebservices.Azure; namespace WWTMVC5 { @@ -35,6 +38,7 @@ public static IUnityContainer GetConfiguredContainer() public static void RegisterTypes(IUnityContainer container) { RegisterRequestProviders(container); + RegisterPlateFileProvider(container); // NOTE: To load from web.config uncomment the line below. Make sure to add a Microsoft.Practices.Unity.Configuration to the using statements. // container.LoadConfiguration(); @@ -54,5 +58,17 @@ private static void RegisterRequestProviders(IUnityContainer container) container.RegisterType(type); } } + + private static void RegisterPlateFileProvider(IUnityContainer container) + { + if (ConfigReader.GetSetting("UseAzurePlateFiles")) + { + container.RegisterInstance(new AzurePlateTilePyramid(ConfigurationManager.AppSettings["AzurePlateFileContainer"], new DefaultAzureCredential())); + } + else + { + container.RegisterType(new ContainerControlledLifetimeManager()); + } + } } } diff --git a/WWTMVC5/WWTMVC5.csproj b/WWTMVC5/WWTMVC5.csproj index 393270fa..f275ebb2 100644 --- a/WWTMVC5/WWTMVC5.csproj +++ b/WWTMVC5/WWTMVC5.csproj @@ -1377,6 +1377,10 @@ {ee4a3106-572b-4ef4-9ab5-a22643309a57} WWT.Providers + + {FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9} + WWTWebservices.Azure + {1cf4d986-51ad-43f8-a84a-38b6ecba2172} WWTWebservices @@ -1443,9 +1447,6 @@ 6.1.7600.16394 - - 3.2.3 - 10.0.3 @@ -1461,9 +1462,6 @@ 1.6.0 - - 4.3.0 - 10.0 diff --git a/WWTMVC5/Web.config b/WWTMVC5/Web.config index 95628f7c..f469c31c 100644 --- a/WWTMVC5/Web.config +++ b/WWTMVC5/Web.config @@ -137,6 +137,10 @@ Content-Type: application/x-wt--> + + + + diff --git a/WWTWebSiteOnly.sln b/WWTWebSiteOnly.sln index 9beac5ad..e468c5fe 100644 --- a/WWTWebSiteOnly.sln +++ b/WWTWebSiteOnly.sln @@ -24,6 +24,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution README.md = README.md EndProjectSection EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WWTWebservices.Azure", "WWTWebservices.Azure\WWTWebservices.Azure.csproj", "{FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -46,6 +48,10 @@ Global {31DC6DC8-AF43-41AC-B9B7-7E77E6CD8950}.Debug|Any CPU.Build.0 = Debug|Any CPU {31DC6DC8-AF43-41AC-B9B7-7E77E6CD8950}.Release|Any CPU.ActiveCfg = Release|Any CPU {31DC6DC8-AF43-41AC-B9B7-7E77E6CD8950}.Release|Any CPU.Build.0 = Release|Any CPU + {FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/WWTWebservices.Azure/AzurePlateTilePyramid.cs b/WWTWebservices.Azure/AzurePlateTilePyramid.cs new file mode 100644 index 00000000..54efd571 --- /dev/null +++ b/WWTWebservices.Azure/AzurePlateTilePyramid.cs @@ -0,0 +1,64 @@ +using Azure.Core; +using Azure.Storage.Blobs; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; + +namespace WWTWebservices.Azure +{ + public class AzurePlateTilePyramid : IPlateTilePyramid + { + private readonly Dictionary _plateNameMapping = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + { "dssterrapixel.plate", ("dss", "DSSTerraPixelL{0}X{1}Y{2}.png") } + }; + + private readonly BlobServiceClient _service; + private readonly ConcurrentDictionary _containers; + + public AzurePlateTilePyramid(string storageUri, TokenCredential credentials) + { + _service = new BlobServiceClient(new Uri(storageUri), credentials); + _containers = new ConcurrentDictionary(); + } + + public Stream GetStream(string pathPrefix, string plateName, int level, int x, int y) + { + var container = _containers.GetOrAdd(plateName, p => + { + var name = GetBlobContainerName(plateName); + + return _service.GetBlobContainerClient(name); + }); + + var blobName = GetBlobName(plateName, level, x, y); + var client = container.GetBlobClient(blobName); + var download = client.Download(); + + return download.Value.Content; + } + + private string GetBlobContainerName(string plateName) + { + if (_plateNameMapping.TryGetValue(plateName, out var info)) + { + return info.container; + } + + return Path.GetFileNameWithoutExtension(plateName).ToLowerInvariant(); + } + + private string GetBlobName(string plateName, int level, int x, int y) + { + var blobFormat = "L{0}X{1}Y{2}.png"; + + if (_plateNameMapping.TryGetValue(plateName, out var info)) + { + blobFormat = info.blob; + } + + return string.Format(blobFormat, level, x, y); + } + } +} diff --git a/WWTWebservices.Azure/WWTWebservices.Azure.csproj b/WWTWebservices.Azure/WWTWebservices.Azure.csproj new file mode 100644 index 00000000..dbaa6343 --- /dev/null +++ b/WWTWebservices.Azure/WWTWebservices.Azure.csproj @@ -0,0 +1,19 @@ + + + + net48 + + + + + + + + + + + + + + + diff --git a/WWTWebservices/ConfigurationManagerFilePlateTilePyramid.cs b/WWTWebservices/ConfigurationManagerFilePlateTilePyramid.cs new file mode 100644 index 00000000..8562778f --- /dev/null +++ b/WWTWebservices/ConfigurationManagerFilePlateTilePyramid.cs @@ -0,0 +1,17 @@ +using System.IO; + +namespace WWTWebservices +{ + public class ConfigurationManagerFilePlateTilePyramid : IPlateTilePyramid + { + public Stream GetStream(string pathPrefix, string plateName, int level, int x, int y) + { + if (string.IsNullOrEmpty(pathPrefix)) + { + throw new System.ArgumentException($"'{nameof(pathPrefix)}' cannot be null or empty", nameof(pathPrefix)); + } + + return PlateTilePyramid.GetFileStream(Path.Combine(pathPrefix, plateName), level, x, y); + } + } +} diff --git a/WWTWebservices/IPlateTilePyramid.cs b/WWTWebservices/IPlateTilePyramid.cs new file mode 100644 index 00000000..eec624c8 --- /dev/null +++ b/WWTWebservices/IPlateTilePyramid.cs @@ -0,0 +1,9 @@ +using System.IO; + +namespace WWTWebservices +{ + public interface IPlateTilePyramid + { + Stream GetStream(string pathPrefix, string plateName, int level, int x, int y); + } +} diff --git a/WWTWebservices/PlateTilePyramid.cs b/WWTWebservices/PlateTilePyramid.cs index 62322af3..b29d4949 100644 --- a/WWTWebservices/PlateTilePyramid.cs +++ b/WWTWebservices/PlateTilePyramid.cs @@ -238,14 +238,13 @@ public Stream GetFileStream(int level, int x, int y) static public Stream GetFileStream(string filename, int level, int x, int y) { uint offset = GetFileIndexOffset(level, x, y); - uint length; uint start; MemoryStream ms = null; using (FileStream f = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { f.Seek(offset, SeekOrigin.Begin); - start = GetNodeInfo(f, offset, out length); + start = GetNodeInfo(f, offset, out var length); byte[] buffer = new byte[length]; f.Seek(start, SeekOrigin.Begin); diff --git a/WWTWebservices/WWTWebservices.csproj b/WWTWebservices/WWTWebservices.csproj index 6e56e41d..b0aa1a75 100644 --- a/WWTWebservices/WWTWebservices.csproj +++ b/WWTWebservices/WWTWebservices.csproj @@ -1,6 +1,6 @@  - net45 + net48 Library false true diff --git a/src/WWT.Providers/Providers/DSSProvider.cs b/src/WWT.Providers/Providers/DSSProvider.cs index f464a488..a386eac6 100644 --- a/src/WWT.Providers/Providers/DSSProvider.cs +++ b/src/WWT.Providers/Providers/DSSProvider.cs @@ -1,70 +1,65 @@ using System; using System.Configuration; -using System.IO; using WWTWebservices; namespace WWT.Providers { public class DSSProvider : RequestProvider { - public override void Run(WwtContext context) + private readonly IPlateTilePyramid _plateTile; + + public DSSProvider(IPlateTilePyramid plateTile) { - string wwtTilesDir = ConfigurationManager.AppSettings["WWTTilesDir"]; - string dssTerapixelDir = ConfigurationManager.AppSettings["DssTerapixelDir"]; + _plateTile = plateTile; + } + public override void Run(WwtContext context) + { string query = context.Request.Params["Q"]; string[] values = query.Split(','); int level = Convert.ToInt32(values[0]); int tileX = Convert.ToInt32(values[1]); int tileY = Convert.ToInt32(values[2]); - int octsetlevel = level; - string filename; - - if (level > 12) { context.Response.Write("No image"); context.Response.Close(); - return; } - - if (level < 8) + else if (level < 8) { + string wwtTilesDir = ConfigurationManager.AppSettings["WWTTilesDir"]; + context.Response.ContentType = "image/png"; - Stream s = PlateTilePyramid.GetFileStream(wwtTilesDir + "\\dssterrapixel.plate", level, tileX, tileY); - int length = (int)s.Length; - byte[] data = new byte[length]; - s.Read(data, 0, length); - context.Response.OutputStream.Write(data, 0, length); - context.Response.Flush(); - context.Response.End(); - return; + + using (var s = _plateTile.GetStream(wwtTilesDir, "dssterrapixel.plate", level, tileX, tileY)) + { + s.CopyTo(context.Response.OutputStream); + context.Response.Flush(); + context.Response.End(); + } } else { - int L = level; - int X = tileX; - int Y = tileY; - string mime = "png"; - int powLev5Diff = (int)Math.Pow(2, L - 5); - int X32 = X / powLev5Diff; - int Y32 = Y / powLev5Diff; - filename = string.Format(dssTerapixelDir + @"\DSS{0}L5to12_x{1}_y{2}.plate", mime, X32, Y32); + int powLev5Diff = (int)Math.Pow(2, level - 5); + int X32 = tileX / powLev5Diff; + int Y32 = tileY / powLev5Diff; + + int L5 = level - 5; + int X5 = tileX % powLev5Diff; + int Y5 = tileY % powLev5Diff; - int L5 = L - 5; - int X5 = X % powLev5Diff; - int Y5 = Y % powLev5Diff; context.Response.ContentType = "image/png"; - Stream s = PlateTilePyramid.GetFileStream(filename, L5, X5, Y5); - int length = (int)s.Length; - byte[] data = new byte[length]; - s.Read(data, 0, length); - context.Response.OutputStream.Write(data, 0, length); - context.Response.Flush(); - context.Response.End(); - return; + string dssTerapixelDir = ConfigurationManager.AppSettings["DssTerapixelDir"]; + string filename = $"DSSpngL5to12_x{X32}_y{Y32}.plate"; + + using (var s = _plateTile.GetStream(dssTerapixelDir, filename, L5, X5, Y5)) + { + s.CopyTo(context.Response.OutputStream); + context.Response.Flush(); + context.Response.End(); + } } } }