From 4385e10c9fbbb3b65a5833762d23afa25ff4b030 Mon Sep 17 00:00:00 2001 From: Elisabeth Unger Date: Fri, 14 Oct 2022 11:52:39 +0200 Subject: [PATCH 1/4] Download langtags using etag instead of if-modified-since header Implement #987. +semver:minor --- CHANGELOG.md | 1 + SIL.WritingSystems/Sldr.cs | 91 +++++++++++++++++++++++++++----------- 2 files changed, 67 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21a153eab..3b2afa891 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -174,6 +174,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - [SIL.Core] ConsoleErrorReporter logs exception if available - [SIL.Core, SIL.Windows.Forms] If WinFormsErrorReporter is set as the ErrorReporter, and ErrorReporter.NotifyUserOfProblem(IRepeatNoticePolicy, Exception, String, params object[]) is passed null for the exception, the "Details" button will no longer appear, making this consistent with the no-Exception overload of this method - [SIL.WritingSystems] Changed behavior of IetfLanguageTag to better handle zh-TW. +- [SIL.WritingSystems] Download of langtag.json handled with etag instead of IF-MODIFIED-SINCE Header(#987) ### Fixed diff --git a/SIL.WritingSystems/Sldr.cs b/SIL.WritingSystems/Sldr.cs index bc2e3b878..ec155f237 100644 --- a/SIL.WritingSystems/Sldr.cs +++ b/SIL.WritingSystems/Sldr.cs @@ -451,9 +451,10 @@ public static IReadOnlyKeyedCollection LanguageTags } } - public static void InitializeLanguageTags() + public static void InitializeLanguageTags(bool downloadLangTags = true) { LoadLanguageTagsIfNecessary(); + if (downloadLangTags) LoadLanguageTags(); } /// @@ -470,35 +471,75 @@ private static void LoadLanguageTagsIfNecessary() CreateSldrCacheDirectory(); cachedAllTagsPath = Path.Combine(SldrCachePath, "langtags.json"); + string etagPath; + etagPath = Path.Combine(SldrCachePath, "langtags.json.etag"); var sinceTime = _embeddedAllTagsTime.ToUniversalTime(); if (File.Exists(cachedAllTagsPath)) { var fileTime = File.GetLastWriteTimeUtc(cachedAllTagsPath); if (sinceTime > fileTime) + { // delete the old langtags.json file if a newer embedded one is available. // this can happen if the application is upgraded to use a newer version of SIL.WritingSystems // that has an updated embedded langtags.json file. File.Delete(cachedAllTagsPath); + File.Delete(etagPath); + } else sinceTime = fileTime; } sinceTime += TimeSpan.FromSeconds(1); - try - { - if (_offlineMode) - throw new WebException("Test mode: SLDR offline so accessing cache", WebExceptionStatus.ConnectFailure); + } + _languageTags = new ReadOnlyKeyedCollection(ParseAllTagsJson(cachedAllTagsPath)); + } - // get SLDR langtags.json from the SLDR api compressed - // it will throw WebException or have status HttpStatusCode.NotModified if file is unchanged or not get it - var langtagsUrl = - $"{SldrRepository}index.html?query=langtags&ext=json{StagingParameter}"; - var webRequest = (HttpWebRequest) WebRequest.Create(Uri.EscapeUriString(langtagsUrl)); - webRequest.UserAgent = UserAgent; - webRequest.IfModifiedSince = sinceTime; - webRequest.Timeout = 10000; - webRequest.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate; - using var webResponse = (HttpWebResponse) webRequest.GetResponse(); + public static void LoadLanguageTags() + { + CreateSldrCacheDirectory(); + string cachedAllTagsPath; + cachedAllTagsPath = Path.Combine(SldrCachePath, "langtags.json"); + string etagPath; + etagPath = Path.Combine(SldrCachePath, "langtags.json.etag"); + string etag; + string currentEtag; + try + { + if (_offlineMode) + throw new WebException("Test mode: SLDR offline so accessing cache", WebExceptionStatus.ConnectFailure); + // get SLDR langtags.json from the SLDR api compressed + // it will throw WebException or have status HttpStatusCode.NotModified if file is unchanged or not get it + var langtagsUrl = + $"{SldrRepository}index.html?query=langtags&ext=json{StagingParameter}"; + var webRequest = (HttpWebRequest)WebRequest.Create(Uri.EscapeUriString(langtagsUrl)); + webRequest.UserAgent = UserAgent; + webRequest.Timeout = 10000; + webRequest.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate; + using var webResponse = (HttpWebResponse)webRequest.GetResponse(); + if (File.Exists(etagPath)) + { + etag = File.ReadAllText(etagPath); + currentEtag = webResponse.Headers.Get("Etag"); + if (etag == "") + { + File.WriteAllText(etagPath, currentEtag); + } + else if (!etag.Equals(currentEtag)) + { + File.WriteAllText(etagPath, etag); + webRequest.Headers.Set(etag, "If-None-Match"); + if (webResponse.StatusCode != HttpStatusCode.NotModified) + { + using Stream output = File.OpenWrite(cachedAllTagsPath); + using var input = webResponse.GetResponseStream(); + input.CopyTo(output); + } + } + } + else + { + currentEtag = webResponse.Headers.Get("Etag"); + File.WriteAllText(etagPath, currentEtag); if (webResponse.StatusCode != HttpStatusCode.NotModified) { using Stream output = File.OpenWrite(cachedAllTagsPath); @@ -506,19 +547,19 @@ private static void LoadLanguageTagsIfNecessary() input.CopyTo(output); } } - catch (WebException) - { - } - catch (UnauthorizedAccessException) - { - } - catch (IOException) - { - } } - _languageTags = new ReadOnlyKeyedCollection(ParseAllTagsJson(cachedAllTagsPath)); + catch (WebException) + { + } + catch (UnauthorizedAccessException) + { + } + catch (IOException) + { + } } + private static IKeyedCollection ParseAllTagsJson(string cachedAllTagsPath) { // read in the json file From 6f8b7e92d428b9341074a8735e7c4cffd2172109 Mon Sep 17 00:00:00 2001 From: Elisabeth Unger Date: Thu, 20 Oct 2022 15:54:36 +0200 Subject: [PATCH 2/4] changes langtagDownloads --- SIL.WritingSystems/Sldr.cs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/SIL.WritingSystems/Sldr.cs b/SIL.WritingSystems/Sldr.cs index ec155f237..d22692e0d 100644 --- a/SIL.WritingSystems/Sldr.cs +++ b/SIL.WritingSystems/Sldr.cs @@ -453,8 +453,8 @@ public static IReadOnlyKeyedCollection LanguageTags public static void InitializeLanguageTags(bool downloadLangTags = true) { + if (downloadLangTags) DownloadLanguageTags(); LoadLanguageTagsIfNecessary(); - if (downloadLangTags) LoadLanguageTags(); } /// @@ -488,13 +488,12 @@ private static void LoadLanguageTagsIfNecessary() else sinceTime = fileTime; } - sinceTime += TimeSpan.FromSeconds(1); } _languageTags = new ReadOnlyKeyedCollection(ParseAllTagsJson(cachedAllTagsPath)); } - public static void LoadLanguageTags() + public static void DownloadLanguageTags() { CreateSldrCacheDirectory(); string cachedAllTagsPath; @@ -519,17 +518,15 @@ public static void LoadLanguageTags() if (File.Exists(etagPath)) { etag = File.ReadAllText(etagPath); + webRequest.Headers.Set(etag, "If-None-Match"); currentEtag = webResponse.Headers.Get("Etag"); - if (etag == "") + if (!etag.Equals(currentEtag)) { - File.WriteAllText(etagPath, currentEtag); - } - else if (!etag.Equals(currentEtag)) - { - File.WriteAllText(etagPath, etag); - webRequest.Headers.Set(etag, "If-None-Match"); + //File.WriteAllText(etagPath, currentEtag); + //webRequest.Headers.Set(etag, "If-None-Match"); if (webResponse.StatusCode != HttpStatusCode.NotModified) { + File.WriteAllText(etagPath, currentEtag); using Stream output = File.OpenWrite(cachedAllTagsPath); using var input = webResponse.GetResponseStream(); input.CopyTo(output); From 285df06f1af21c2ec587e5400ebf762e7995253f Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Mon, 14 Nov 2022 19:58:48 +0100 Subject: [PATCH 3/4] Add unit tests for downloading langtags.json --- SIL.WritingSystems.Tests/SldrTests.cs | 117 +++++++++++++++++++++++++- 1 file changed, 114 insertions(+), 3 deletions(-) diff --git a/SIL.WritingSystems.Tests/SldrTests.cs b/SIL.WritingSystems.Tests/SldrTests.cs index db8cd7cc6..e1b24fd96 100644 --- a/SIL.WritingSystems.Tests/SldrTests.cs +++ b/SIL.WritingSystems.Tests/SldrTests.cs @@ -15,9 +15,12 @@ public class SldrTests { private class TestEnvironment : IDisposable { - public TestEnvironment(bool sldrOffline = true, DateTime? embeddedAllTagsTime = null) + public TestEnvironment(bool sldrOffline = true, string sldrCachePath = null, + DateTime? embeddedAllTagsTime = null) { - var sldrCachePath = Sldr.SldrCachePath; + if (string.IsNullOrEmpty(sldrCachePath)) + sldrCachePath = Sldr.DefaultSldrCachePath; + Sldr.Cleanup(); if (embeddedAllTagsTime == null) Sldr.Initialize(sldrOffline, sldrCachePath); @@ -424,7 +427,8 @@ public void MoveTmpToCache_DraftProposed_KeepsUid() [Explicit] public void LanguageTags_OlderEmbeddedLangTags_DownloadsNewLangTags() { - using var testEnv = new TestEnvironment(false, new DateTime(2000, 1, 1, 12, 0, 0)); + using var testEnv = new TestEnvironment(false, null, + new DateTime(2000, 1, 1, 12, 0, 0)); var langTagsPath = Path.Combine(Sldr.SldrCachePath, "langtags.json"); if (File.Exists(langTagsPath)) { @@ -509,5 +513,112 @@ public void GetUnicodeCategoryBasedOnICU_LowSurrogateAtStartOfString_ThrowsArgum Assert.That(() => Sldr.GetUnicodeCategoryBasedOnICU(bogusString, 0), Throws.ArgumentException); } + + [Test] + [Explicit] + public void DownloadLanguageTags_NoPreviousEtagNoFile() + { + // Setup + using var sldrCachePath = new TemporaryFolder(TestContext.CurrentContext.Test.Name); + using var environment = new TestEnvironment(false, sldrCachePath.Path); + var langtagsFile = Path.Combine(sldrCachePath.Path, "langtags.json"); + var langtagsEtagFile = Path.Combine(sldrCachePath.Path, "langtags.json.etag"); + + // Execute + Sldr.DownloadLanguageTags(); + + // Verify + Assert.That(File.Exists(langtagsFile), Is.True); + Assert.That(File.Exists(langtagsEtagFile), Is.True); + Assert.That(File.ReadAllText(langtagsFile), + Contains.Substring("\"full\": \"aa-Latn-ET\"")); + } + + [Test] + [Explicit] + public void DownloadLanguageTags_NoPreviousEtagExistingFile() + { + // Setup + using var sldrCachePath = new TemporaryFolder(TestContext.CurrentContext.Test.Name); + using var environment = new TestEnvironment(false, sldrCachePath.Path); + var langtagsFile = Path.Combine(sldrCachePath.Path, "langtags.json"); + var langtagsEtagFile = Path.Combine(sldrCachePath.Path, "langtags.json.etag"); + Sldr.DownloadLanguageTags(); + File.Delete(langtagsEtagFile); + + // Execute + Sldr.DownloadLanguageTags(); + + // Verify + Assert.That(File.Exists(langtagsFile), Is.True); + Assert.That(File.Exists(langtagsEtagFile), Is.True); + } + + [Test] + [Explicit] + public void DownloadLanguageTags_ExistingEtagNoFile() + { + // Setup + using var sldrCachePath = new TemporaryFolder(TestContext.CurrentContext.Test.Name); + using var environment = new TestEnvironment(false, sldrCachePath.Path); + var langtagsFile = Path.Combine(sldrCachePath.Path, "langtags.json"); + var langtagsEtagFile = Path.Combine(sldrCachePath.Path, "langtags.json.etag"); + Sldr.DownloadLanguageTags(); + File.Delete(langtagsFile); + + // Execute + Sldr.DownloadLanguageTags(); + + // Verify + Assert.That(File.Exists(langtagsFile), Is.True); + Assert.That(File.Exists(langtagsEtagFile), Is.True); + } + + [Test] + [Explicit] + public void DownloadLanguageTags_NotModified() + { + // Setup + using var sldrCachePath = new TemporaryFolder(TestContext.CurrentContext.Test.Name); + using var environment = new TestEnvironment(false, sldrCachePath.Path); + Sldr.DownloadLanguageTags(); + var langtagsFile = Path.Combine(sldrCachePath.Path, "langtags.json"); + var langtagsEtagFile = Path.Combine(sldrCachePath.Path, "langtags.json.etag"); + var langtagsFileDate = new FileInfo(langtagsFile).LastWriteTime; + var etagsFileDate = new FileInfo(langtagsEtagFile).LastWriteTime; + + // Execute + Sldr.DownloadLanguageTags(); + + // Verify + Assert.That(File.Exists(langtagsFile), Is.True); + Assert.That(File.Exists(langtagsEtagFile), Is.True); + Assert.That(new FileInfo(langtagsFile).LastWriteTime, Is.EqualTo(langtagsFileDate)); + Assert.That(new FileInfo(langtagsEtagFile).LastWriteTime, Is.EqualTo(etagsFileDate)); + } + + [Test] + [Explicit] + public void DownloadLanguageTags_Outdated() + { + // Setup + using var sldrCachePath = new TemporaryFolder(TestContext.CurrentContext.Test.Name); + using var environment = new TestEnvironment(false, sldrCachePath.Path); + Sldr.DownloadLanguageTags(); + var langtagsFile = Path.Combine(sldrCachePath.Path, "langtags.json"); + var langtagsEtagFile = Path.Combine(sldrCachePath.Path, "langtags.json.etag"); + File.WriteAllText(langtagsEtagFile, @"""12345678-abcdef"""); + var langtagsFileDate = new FileInfo(langtagsFile).LastWriteTime; + var etagsFileDate = new FileInfo(langtagsEtagFile).LastWriteTime; + + // Execute + Sldr.DownloadLanguageTags(); + + // Verify + Assert.That(File.Exists(langtagsFile), Is.True); + Assert.That(File.Exists(langtagsEtagFile), Is.True); + Assert.That(new FileInfo(langtagsFile).LastWriteTime, Is.GreaterThan(langtagsFileDate)); + Assert.That(new FileInfo(langtagsEtagFile).LastWriteTime, Is.GreaterThan(etagsFileDate)); + } } } From e6e94d6a680773e3eadcab49e77dfe29ef004277 Mon Sep 17 00:00:00 2001 From: Elisabeth Unger Date: Tue, 7 Feb 2023 16:30:12 +0100 Subject: [PATCH 4/4] address code review --- SIL.WritingSystems/Sldr.cs | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/SIL.WritingSystems/Sldr.cs b/SIL.WritingSystems/Sldr.cs index d22692e0d..21d609fba 100644 --- a/SIL.WritingSystems/Sldr.cs +++ b/SIL.WritingSystems/Sldr.cs @@ -496,6 +496,9 @@ private static void LoadLanguageTagsIfNecessary() public static void DownloadLanguageTags() { CreateSldrCacheDirectory(); + if (_offlineMode) + throw new WebException("Test mode: SLDR offline so accessing cache", WebExceptionStatus.ConnectFailure); + string cachedAllTagsPath; cachedAllTagsPath = Path.Combine(SldrCachePath, "langtags.json"); string etagPath; @@ -504,45 +507,40 @@ public static void DownloadLanguageTags() string currentEtag; try { - if (_offlineMode) - throw new WebException("Test mode: SLDR offline so accessing cache", WebExceptionStatus.ConnectFailure); // get SLDR langtags.json from the SLDR api compressed // it will throw WebException or have status HttpStatusCode.NotModified if file is unchanged or not get it var langtagsUrl = $"{SldrRepository}index.html?query=langtags&ext=json{StagingParameter}"; var webRequest = (HttpWebRequest)WebRequest.Create(Uri.EscapeUriString(langtagsUrl)); + webRequest.Headers.Set("If-None-Match", etag); webRequest.UserAgent = UserAgent; webRequest.Timeout = 10000; webRequest.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate; using var webResponse = (HttpWebResponse)webRequest.GetResponse(); - if (File.Exists(etagPath)) + if (webResponse.StatusCode != "OK") + { + return; + } + + if (File.Exists(etagPath) && File.Exists(cachedAllTagsPath)) { etag = File.ReadAllText(etagPath); - webRequest.Headers.Set(etag, "If-None-Match"); currentEtag = webResponse.Headers.Get("Etag"); if (!etag.Equals(currentEtag)) { - //File.WriteAllText(etagPath, currentEtag); - //webRequest.Headers.Set(etag, "If-None-Match"); - if (webResponse.StatusCode != HttpStatusCode.NotModified) - { - File.WriteAllText(etagPath, currentEtag); - using Stream output = File.OpenWrite(cachedAllTagsPath); - using var input = webResponse.GetResponseStream(); - input.CopyTo(output); - } + File.WriteAllText(etagPath, currentEtag); + using Stream output = File.OpenWrite(cachedAllTagsPath); + using var input = webResponse.GetResponseStream(); + input?.CopyTo(output); } } else { currentEtag = webResponse.Headers.Get("Etag"); File.WriteAllText(etagPath, currentEtag); - if (webResponse.StatusCode != HttpStatusCode.NotModified) - { - using Stream output = File.OpenWrite(cachedAllTagsPath); - using var input = webResponse.GetResponseStream(); - input.CopyTo(output); - } + using Stream output = File.OpenWrite(cachedAllTagsPath); + using var input = webResponse.GetResponseStream(); + input?.CopyTo(output); } } catch (WebException)