diff --git a/CHANGELOG.md b/CHANGELOG.md index cd57e66b7..a023e718b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -181,6 +181,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - [SIL.Core] ConsoleErrorReporter logs exception if available - [SIL.Core, SIL.Windows.Forms] If WinFormsErrorReporter is set as the ErrorReporter, and ErrorReporter.NotifyUserOfProblem(IRepeatNoticePolicy, Exception, String, params object[]) is passed null for the exception, the "Details" button will no longer appear, making this consistent with the no-Exception overload of this method - [SIL.WritingSystems] Changed behavior of IetfLanguageTag to better handle zh-TW. +- [SIL.WritingSystems] Download of langtag.json handled with etag instead of IF-MODIFIED-SINCE Header(#987) ### Fixed diff --git a/SIL.WritingSystems.Tests/SldrTests.cs b/SIL.WritingSystems.Tests/SldrTests.cs index db8cd7cc6..e1b24fd96 100644 --- a/SIL.WritingSystems.Tests/SldrTests.cs +++ b/SIL.WritingSystems.Tests/SldrTests.cs @@ -15,9 +15,12 @@ public class SldrTests { private class TestEnvironment : IDisposable { - public TestEnvironment(bool sldrOffline = true, DateTime? embeddedAllTagsTime = null) + public TestEnvironment(bool sldrOffline = true, string sldrCachePath = null, + DateTime? embeddedAllTagsTime = null) { - var sldrCachePath = Sldr.SldrCachePath; + if (string.IsNullOrEmpty(sldrCachePath)) + sldrCachePath = Sldr.DefaultSldrCachePath; + Sldr.Cleanup(); if (embeddedAllTagsTime == null) Sldr.Initialize(sldrOffline, sldrCachePath); @@ -424,7 +427,8 @@ public void MoveTmpToCache_DraftProposed_KeepsUid() [Explicit] public void LanguageTags_OlderEmbeddedLangTags_DownloadsNewLangTags() { - using var testEnv = new TestEnvironment(false, new DateTime(2000, 1, 1, 12, 0, 0)); + using var testEnv = new TestEnvironment(false, null, + new DateTime(2000, 1, 1, 12, 0, 0)); var langTagsPath = Path.Combine(Sldr.SldrCachePath, "langtags.json"); if (File.Exists(langTagsPath)) { @@ -509,5 +513,112 @@ public void GetUnicodeCategoryBasedOnICU_LowSurrogateAtStartOfString_ThrowsArgum Assert.That(() => Sldr.GetUnicodeCategoryBasedOnICU(bogusString, 0), Throws.ArgumentException); } + + [Test] + [Explicit] + public void DownloadLanguageTags_NoPreviousEtagNoFile() + { + // Setup + using var sldrCachePath = new TemporaryFolder(TestContext.CurrentContext.Test.Name); + using var environment = new TestEnvironment(false, sldrCachePath.Path); + var langtagsFile = Path.Combine(sldrCachePath.Path, "langtags.json"); + var langtagsEtagFile = Path.Combine(sldrCachePath.Path, "langtags.json.etag"); + + // Execute + Sldr.DownloadLanguageTags(); + + // Verify + Assert.That(File.Exists(langtagsFile), Is.True); + Assert.That(File.Exists(langtagsEtagFile), Is.True); + Assert.That(File.ReadAllText(langtagsFile), + Contains.Substring("\"full\": \"aa-Latn-ET\"")); + } + + [Test] + [Explicit] + public void DownloadLanguageTags_NoPreviousEtagExistingFile() + { + // Setup + using var sldrCachePath = new TemporaryFolder(TestContext.CurrentContext.Test.Name); + using var environment = new TestEnvironment(false, sldrCachePath.Path); + var langtagsFile = Path.Combine(sldrCachePath.Path, "langtags.json"); + var langtagsEtagFile = Path.Combine(sldrCachePath.Path, "langtags.json.etag"); + Sldr.DownloadLanguageTags(); + File.Delete(langtagsEtagFile); + + // Execute + Sldr.DownloadLanguageTags(); + + // Verify + Assert.That(File.Exists(langtagsFile), Is.True); + Assert.That(File.Exists(langtagsEtagFile), Is.True); + } + + [Test] + [Explicit] + public void DownloadLanguageTags_ExistingEtagNoFile() + { + // Setup + using var sldrCachePath = new TemporaryFolder(TestContext.CurrentContext.Test.Name); + using var environment = new TestEnvironment(false, sldrCachePath.Path); + var langtagsFile = Path.Combine(sldrCachePath.Path, "langtags.json"); + var langtagsEtagFile = Path.Combine(sldrCachePath.Path, "langtags.json.etag"); + Sldr.DownloadLanguageTags(); + File.Delete(langtagsFile); + + // Execute + Sldr.DownloadLanguageTags(); + + // Verify + Assert.That(File.Exists(langtagsFile), Is.True); + Assert.That(File.Exists(langtagsEtagFile), Is.True); + } + + [Test] + [Explicit] + public void DownloadLanguageTags_NotModified() + { + // Setup + using var sldrCachePath = new TemporaryFolder(TestContext.CurrentContext.Test.Name); + using var environment = new TestEnvironment(false, sldrCachePath.Path); + Sldr.DownloadLanguageTags(); + var langtagsFile = Path.Combine(sldrCachePath.Path, "langtags.json"); + var langtagsEtagFile = Path.Combine(sldrCachePath.Path, "langtags.json.etag"); + var langtagsFileDate = new FileInfo(langtagsFile).LastWriteTime; + var etagsFileDate = new FileInfo(langtagsEtagFile).LastWriteTime; + + // Execute + Sldr.DownloadLanguageTags(); + + // Verify + Assert.That(File.Exists(langtagsFile), Is.True); + Assert.That(File.Exists(langtagsEtagFile), Is.True); + Assert.That(new FileInfo(langtagsFile).LastWriteTime, Is.EqualTo(langtagsFileDate)); + Assert.That(new FileInfo(langtagsEtagFile).LastWriteTime, Is.EqualTo(etagsFileDate)); + } + + [Test] + [Explicit] + public void DownloadLanguageTags_Outdated() + { + // Setup + using var sldrCachePath = new TemporaryFolder(TestContext.CurrentContext.Test.Name); + using var environment = new TestEnvironment(false, sldrCachePath.Path); + Sldr.DownloadLanguageTags(); + var langtagsFile = Path.Combine(sldrCachePath.Path, "langtags.json"); + var langtagsEtagFile = Path.Combine(sldrCachePath.Path, "langtags.json.etag"); + File.WriteAllText(langtagsEtagFile, @"""12345678-abcdef"""); + var langtagsFileDate = new FileInfo(langtagsFile).LastWriteTime; + var etagsFileDate = new FileInfo(langtagsEtagFile).LastWriteTime; + + // Execute + Sldr.DownloadLanguageTags(); + + // Verify + Assert.That(File.Exists(langtagsFile), Is.True); + Assert.That(File.Exists(langtagsEtagFile), Is.True); + Assert.That(new FileInfo(langtagsFile).LastWriteTime, Is.GreaterThan(langtagsFileDate)); + Assert.That(new FileInfo(langtagsEtagFile).LastWriteTime, Is.GreaterThan(etagsFileDate)); + } } } diff --git a/SIL.WritingSystems/Sldr.cs b/SIL.WritingSystems/Sldr.cs index f52990cfd..9a52c2827 100644 --- a/SIL.WritingSystems/Sldr.cs +++ b/SIL.WritingSystems/Sldr.cs @@ -451,8 +451,9 @@ public static IReadOnlyKeyedCollection LanguageTags } } - public static void InitializeLanguageTags() + public static void InitializeLanguageTags(bool downloadLangTags = true) { + if (downloadLangTags) DownloadLanguageTags(); LoadLanguageTagsIfNecessary(); } @@ -470,55 +471,90 @@ private static void LoadLanguageTagsIfNecessary() CreateSldrCacheDirectory(); cachedAllTagsPath = Path.Combine(SldrCachePath, "langtags.json"); + string etagPath; + etagPath = Path.Combine(SldrCachePath, "langtags.json.etag"); var sinceTime = _embeddedAllTagsTime.ToUniversalTime(); if (File.Exists(cachedAllTagsPath)) { var fileTime = File.GetLastWriteTimeUtc(cachedAllTagsPath); if (sinceTime > fileTime) + { // delete the old langtags.json file if a newer embedded one is available. // this can happen if the application is upgraded to use a newer version of SIL.WritingSystems // that has an updated embedded langtags.json file. File.Delete(cachedAllTagsPath); + File.Delete(etagPath); + } else sinceTime = fileTime; } - sinceTime += TimeSpan.FromSeconds(1); - try - { - if (_offlineMode) - throw new WebException("Test mode: SLDR offline so accessing cache", WebExceptionStatus.ConnectFailure); + } + _languageTags = new ReadOnlyKeyedCollection(ParseAllTagsJson(cachedAllTagsPath)); + } - // get SLDR langtags.json from the SLDR api compressed - // it will throw WebException or have status HttpStatusCode.NotModified if file is unchanged or not get it - var langtagsUrl = - $"{SldrRepository}index.html?query=langtags&ext=json{StagingParameter}"; - var webRequest = (HttpWebRequest) WebRequest.Create(Uri.EscapeUriString(langtagsUrl)); - webRequest.UserAgent = UserAgent; - webRequest.IfModifiedSince = sinceTime; - webRequest.Timeout = 10000; - webRequest.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate; - using var webResponse = (HttpWebResponse) webRequest.GetResponse(); - if (webResponse.StatusCode != HttpStatusCode.NotModified) + public static void DownloadLanguageTags() + { + CreateSldrCacheDirectory(); + if (_offlineMode) + throw new WebException("Test mode: SLDR offline so accessing cache", WebExceptionStatus.ConnectFailure); + + string cachedAllTagsPath; + cachedAllTagsPath = Path.Combine(SldrCachePath, "langtags.json"); + string etagPath; + etagPath = Path.Combine(SldrCachePath, "langtags.json.etag"); + string etag; + string currentEtag; + try + { + // get SLDR langtags.json from the SLDR api compressed + // it will throw WebException or have status HttpStatusCode.NotModified if file is unchanged or not get it + var langtagsUrl = + $"{SldrRepository}index.html?query=langtags&ext=json{StagingParameter}"; + var webRequest = (HttpWebRequest)WebRequest.Create(Uri.EscapeUriString(langtagsUrl)); + webRequest.Headers.Set("If-None-Match", etag); + webRequest.UserAgent = UserAgent; + webRequest.Timeout = 10000; + webRequest.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate; + using var webResponse = (HttpWebResponse)webRequest.GetResponse(); + if (webResponse.StatusCode != "OK") + { + return; + } + + if (File.Exists(etagPath) && File.Exists(cachedAllTagsPath)) + { + etag = File.ReadAllText(etagPath); + currentEtag = webResponse.Headers.Get("Etag"); + if (!etag.Equals(currentEtag)) { + File.WriteAllText(etagPath, currentEtag); using Stream output = File.OpenWrite(cachedAllTagsPath); using var input = webResponse.GetResponseStream(); - input.CopyTo(output); + input?.CopyTo(output); } } - catch (WebException) - { - } - catch (UnauthorizedAccessException) - { - } - catch (IOException) + else { + currentEtag = webResponse.Headers.Get("Etag"); + File.WriteAllText(etagPath, currentEtag); + using Stream output = File.OpenWrite(cachedAllTagsPath); + using var input = webResponse.GetResponseStream(); + input?.CopyTo(output); } } - _languageTags = new ReadOnlyKeyedCollection(ParseAllTagsJson(cachedAllTagsPath)); + catch (WebException) + { + } + catch (UnauthorizedAccessException) + { + } + catch (IOException) + { + } } + private static IKeyedCollection ParseAllTagsJson(string cachedAllTagsPath) { // read in the json file