Skip to content

Commit

Permalink
Update to latest version of csharpier
Browse files Browse the repository at this point in the history
  • Loading branch information
ddaspit committed Jan 30, 2024
1 parent 21aa862 commit 6fa37e9
Show file tree
Hide file tree
Showing 70 changed files with 431 additions and 442 deletions.
2 changes: 1 addition & 1 deletion .config/dotnet-tools.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"isRoot": true,
"tools": {
"csharpier": {
"version": "0.22.1",
"version": "0.27.2",
"commands": [
"dotnet-csharpier"
]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -107,12 +107,12 @@ public static IMachineBuilder AddClearMLService(this IMachineBuilder builder, st
if (connectionString is null)
throw new InvalidOperationException("ClearML connection string is required");

builder.Services
.AddHttpClient("ClearML")
builder
.Services.AddHttpClient("ClearML")
.ConfigureHttpClient(httpClient => httpClient.BaseAddress = new Uri(connectionString!))
// Add retry policy; fail after approx. 2 + 4 + 8 = 14 seconds
.AddTransientHttpErrorPolicy(
b => b.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)))
.AddTransientHttpErrorPolicy(b =>
b.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)))
);

builder.Services.AddSingleton<IClearMLService, ClearMLService>();
Expand All @@ -121,8 +121,8 @@ public static IMachineBuilder AddClearMLService(this IMachineBuilder builder, st
builder.Services.AddSingleton<IClearMLAuthenticationService, ClearMLAuthenticationService>();
builder.Services.AddHostedService(p => p.GetRequiredService<IClearMLAuthenticationService>());

builder.Services
.AddHttpClient("ClearML-NoRetry")
builder
.Services.AddHttpClient("ClearML-NoRetry")
.ConfigureHttpClient(httpClient => httpClient.BaseAddress = new Uri(connectionString!));
builder.Services.AddSingleton<ClearMLHealthCheck>();

Expand Down Expand Up @@ -160,25 +160,24 @@ public static IMachineBuilder AddMongoHangfireJobClient(
if (connectionString is null)
throw new InvalidOperationException("Hangfire connection string is required");

builder.Services.AddHangfire(
c =>
c.SetDataCompatibilityLevel(CompatibilityLevel.Version_170)
.UseSimpleAssemblyNameTypeSerializer()
.UseRecommendedSerializerSettings()
.UseMongoStorage(
connectionString,
new MongoStorageOptions
builder.Services.AddHangfire(c =>
c.SetDataCompatibilityLevel(CompatibilityLevel.Version_170)
.UseSimpleAssemblyNameTypeSerializer()
.UseRecommendedSerializerSettings()
.UseMongoStorage(
connectionString,
new MongoStorageOptions
{
MigrationOptions = new MongoMigrationOptions
{
MigrationOptions = new MongoMigrationOptions
{
MigrationStrategy = new MigrateMongoMigrationStrategy(),
BackupStrategy = new CollectionMongoBackupStrategy()
},
CheckConnection = true,
CheckQueuedJobsStrategy = CheckQueuedJobsStrategy.TailNotificationsCollection,
}
)
.UseFilter(new AutomaticRetryAttribute { Attempts = 0 })
MigrationStrategy = new MigrateMongoMigrationStrategy(),
BackupStrategy = new CollectionMongoBackupStrategy()
},
CheckConnection = true,
CheckQueuedJobsStrategy = CheckQueuedJobsStrategy.TailNotificationsCollection,
}
)
.UseFilter(new AutomaticRetryAttribute { Attempts = 0 })
);
builder.Services.AddHealthChecks().AddCheck<HangfireHealthCheck>(name: "Hangfire");
return builder;
Expand Down Expand Up @@ -243,8 +242,8 @@ public static IMachineBuilder AddMongoDataAccess(this IMachineBuilder builder, s
{
await c.Indexes.CreateOrUpdateAsync(
new CreateIndexModel<TranslationEngine>(
Builders<TranslationEngine>.IndexKeys
.Ascending(e => e.EngineId)
Builders<TranslationEngine>
.IndexKeys.Ascending(e => e.EngineId)
.Ascending("currentBuild._id")
)
);
Expand Down Expand Up @@ -272,15 +271,18 @@ await c.Indexes.CreateOrUpdateAsync(
return builder;
}

public static IMachineBuilder AddServalPlatformService(this IMachineBuilder builder, string? connectionString = null)
public static IMachineBuilder AddServalPlatformService(
this IMachineBuilder builder,
string? connectionString = null
)
{
connectionString ??= builder.Configuration?.GetConnectionString("Serval");
if (connectionString is null)
throw new InvalidOperationException("Serval connection string is required");

builder.Services.AddScoped<IPlatformService, ServalPlatformService>();
builder.Services
.AddGrpcClient<TranslationPlatformApi.TranslationPlatformApiClient>(o =>
builder
.Services.AddGrpcClient<TranslationPlatformApi.TranslationPlatformApiClient>(o =>
{
o.Address = new Uri(connectionString);
})
Expand Down Expand Up @@ -385,11 +387,11 @@ public static IMachineBuilder AddBuildJobService(this IMachineBuilder builder)
var smtTransferEngineOptions = new SmtTransferEngineOptions();
builder.Configuration.GetSection(SmtTransferEngineOptions.Key).Bind(smtTransferEngineOptions);
string? driveLetter = Path.GetPathRoot(smtTransferEngineOptions.EnginesDir)?[..1];
if(driveLetter is null)
if (driveLetter is null)
throw new InvalidOperationException("SMT Engine directory is required");
// add health check for disk storage capacity
builder.Services
.AddHealthChecks()
builder
.Services.AddHealthChecks()
.AddDiskStorageHealthCheck(
x => x.AddDrive(driveLetter, 1_000), // 1GB
"SMT Engine Storage Capacity",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ public class ClearMLAuthenticationService(
IHttpClientFactory httpClientFactory,
IOptionsMonitor<ClearMLOptions> options,
ILogger<ClearMLAuthenticationService> logger
) : RecurrentTask("ClearML authentication service", services, RefreshPeriod, logger), IClearMLAuthenticationService
) : RecurrentTask("ClearML authentication service", services, RefreshPeriod, logger), IClearMLAuthenticationService
{
private readonly HttpClient _httpClient = httpClientFactory.CreateClient("ClearML");
private readonly IOptionsMonitor<ClearMLOptions> _options = options;
Expand Down Expand Up @@ -40,14 +40,14 @@ protected override async Task DoWorkAsync(IServiceScope scope, CancellationToken
}
catch (Exception e)
{
if (_authToken is ""){
if (_authToken is "")
{
_logger.LogError(e, "Error occurred while aquiring ClearML authentication token for the first time.");
// The ClearML token never was set. We can't continue without it.
throw;
}
else
_logger.LogError(e, "Error occurred while refreshing ClearML authentication token.");

}
}

Expand Down
4 changes: 2 additions & 2 deletions src/SIL.Machine.AspNetCore/Services/ClearMLMonitorService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,8 @@ await _clearMLService.GetTasksByIdAsync(
.UnionBy(await _clearMLService.GetTasksForCurrentQueueAsync(cancellationToken), t => t.Id)
.ToDictionary(t => t.Id);

Dictionary<string, int> queuePositions = tasks.Values
.Where(t => t.Status is ClearMLTaskStatus.Queued or ClearMLTaskStatus.Created)
Dictionary<string, int> queuePositions = tasks
.Values.Where(t => t.Status is ClearMLTaskStatus.Queued or ClearMLTaskStatus.Created)
.OrderBy(t => t.Created)
.Select((t, i) => (Position: i, Task: t))
.ToDictionary(e => e.Task.Name, e => e.Position);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,8 @@ await _locks.UpdateAsync(
RWLock? rwLock = sub.Change.Entity;
if (rwLock is not null && !rwLock.IsAvailableForWriting(lockId))
{
List<DateTime> dateTimes = rwLock.ReaderLocks
.Where(l => l.ExpiresAt.HasValue)
List<DateTime> dateTimes = rwLock
.ReaderLocks.Where(l => l.ExpiresAt.HasValue)
.Select(l => l.ExpiresAt.GetValueOrDefault())
.ToList();
if (rwLock.WriterLock?.ExpiresAt is not null)
Expand Down
14 changes: 12 additions & 2 deletions src/SIL.Machine.AspNetCore/Services/LanguageTagService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,15 @@
public class LanguageTagService : ILanguageTagService
{
private static readonly Dictionary<string, string> StandardLanguages =
new() { { "ar", "arb" }, { "ms", "zsm" }, { "lv", "lvs" }, { "ne", "npi" }, { "sw", "swh" }, { "cmn", "zh" } };
new()
{
{ "ar", "arb" },
{ "ms", "zsm" },
{ "lv", "lvs" },
{ "ne", "npi" },
{ "sw", "swh" },
{ "cmn", "zh" }
};

private readonly Dictionary<string, string> _defaultScripts;

Expand Down Expand Up @@ -41,7 +49,9 @@ private static Dictionary<string, string> InitializeDefaultScripts()
foreach (var t in tags.AsArray().Select(v => (string?)v))
{
if (
t is not null && IetfLanguageTag.TryGetParts(t, out _, out string? s, out _, out _) && s is null
t is not null
&& IetfLanguageTag.TryGetParts(t, out _, out string? s, out _, out _)
&& s is null
)
{
tempDefaultScripts[t] = script;
Expand Down
23 changes: 12 additions & 11 deletions src/SIL.Machine.AspNetCore/Services/NmtPreprocessBuildJob.cs
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,12 @@ CancellationToken cancellationToken

// Log summary of build data
JsonObject _buildPreprocessSummary =
new() { { "Event", "BuildPreprocess" }, { "EngineId", engineId }, { "BuildId", buildId } };
new()
{
{ "Event", "BuildPreprocess" },
{ "EngineId", engineId },
{ "BuildId", buildId }
};
foreach (KeyValuePair<string, int> kvp in counts)
{
_buildPreprocessSummary.Add(kvp.Key, kvp.Value);
Expand Down Expand Up @@ -109,20 +114,16 @@ async IAsyncEnumerable<Pretranslation> ProcessRowsAsync()

var parallelCorpora = new List<IParallelTextCorpus>();

IParallelTextCorpus parallelTextCorpus = sourceCorpora[CorpusType.Text].AlignRows(
targetCorpora[CorpusType.Text],
allSourceRows: true,
allTargetRows: true
);
IParallelTextCorpus parallelTextCorpus = sourceCorpora[CorpusType.Text]
.AlignRows(targetCorpora[CorpusType.Text], allSourceRows: true, allTargetRows: true);
parallelCorpora.Add(parallelTextCorpus);
if (
(bool?)buildOptionsObject?["use_key_terms"]
?? true && sourceCorpora.ContainsKey(CorpusType.Term) && targetCorpora.ContainsKey(CorpusType.Term)
)
{
IParallelTextCorpus parallelKeyTermsCorpus = sourceCorpora[CorpusType.Term].AlignRows(
targetCorpora[CorpusType.Term]
);
IParallelTextCorpus parallelKeyTermsCorpus = sourceCorpora[CorpusType.Term]
.AlignRows(targetCorpora[CorpusType.Term]);
IEnumerable<string> keyTermsTextIds = parallelKeyTermsCorpus.Select(r => r.TextId).Distinct();
if (keyTermsTextIds.Count() == 1)
corpus.TrainOnTextIds.Add(keyTermsTextIds.First()); //Should only be one textId for key terms
Expand All @@ -148,8 +149,8 @@ async IAsyncEnumerable<Pretranslation> ProcessRowsAsync()
{
if (targetCorpora[CorpusType.Text] is ScriptureTextCorpus tstc)
{
refs = row.SourceRefs
.Cast<VerseRef>()
refs = row
.SourceRefs.Cast<VerseRef>()
.Select(srcRef =>
{
var trgRef = srcRef.Clone();
Expand Down
2 changes: 1 addition & 1 deletion src/SIL.Machine.AspNetCore/Services/S3WriteStream.cs
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ protected override void Dispose(bool disposing)
base.Dispose(disposing);
}

public async override ValueTask DisposeAsync()
public override async ValueTask DisposeAsync()
{
if (_uploadResponses.Count == 0)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -202,19 +202,18 @@ private static Serval.Translation.V1.TranslationSources Map(Translation.Translat
{
Values =
{
System.Enum
.GetValues<Translation.TranslationSources>()
System
.Enum.GetValues<Translation.TranslationSources>()
.Where(s => s != Translation.TranslationSources.None && source.HasFlag(s))
.Select(
s =>
s switch
{
Translation.TranslationSources.Smt => TranslationSource.Primary,
Translation.TranslationSources.Nmt => TranslationSource.Primary,
Translation.TranslationSources.Transfer => TranslationSource.Secondary,
Translation.TranslationSources.Prefix => TranslationSource.Human,
_ => TranslationSource.Primary
}
.Select(s =>
s switch
{
Translation.TranslationSources.Smt => TranslationSource.Primary,
Translation.TranslationSources.Nmt => TranslationSource.Primary,
Translation.TranslationSources.Transfer => TranslationSource.Secondary,
Translation.TranslationSources.Prefix => TranslationSource.Human,
_ => TranslationSource.Primary
}
)
}
};
Expand Down
4 changes: 2 additions & 2 deletions src/SIL.Machine.Morphology.HermitCrab.Tool/MorphInfo.cs
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ public MorphInfo(string form, string gloss)

public MorphInfo(Word parse, Annotation<ShapeNode> morph)
{
_form = morph.Children
.Where(a => a.Type() != HCFeatureSystem.Morph)
_form = morph
.Children.Where(a => a.Type() != HCFeatureSystem.Morph)
.Select(a => a.Range.Start)
.ToString(parse.Stratum.CharacterDefinitionTable, false);
_gloss = parse.GetAllomorph(morph).Morpheme.Gloss;
Expand Down
25 changes: 11 additions & 14 deletions src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
#if !SINGLE_THREADED
using System;
using System.Collections.Concurrent;
using System.Threading.Tasks;
#endif
using System.Collections.Generic;
using System.Collections.Generic;
using System.Linq;
using SIL.Machine.Annotations;
using SIL.Machine.FeatureModel;
using SIL.Machine.Rules;
using SIL.ObjectModel;
#if !SINGLE_THREADED
using System;
using System.Collections.Concurrent;
using System.Threading.Tasks;
#endif

namespace SIL.Machine.Morphology.HermitCrab
{
Expand All @@ -23,14 +23,11 @@ public AnalysisAffixTemplateRule(Morpher morpher, AffixTemplate template)
_morpher = morpher;
_template = template;
_rules = new List<IRule<Word, ShapeNode>>(
template.Slots.Select(
slot =>
new RuleBatch<Word, ShapeNode>(
slot.Rules.Select(mr => mr.CompileAnalysisRule(morpher)),
false,
FreezableEqualityComparer<Word>.Default
)
)
template.Slots.Select(slot => new RuleBatch<Word, ShapeNode>(
slot.Rules.Select(mr => mr.CompileAnalysisRule(morpher)),
false,
FreezableEqualityComparer<Word>.Default
))
);
}

Expand Down
4 changes: 2 additions & 2 deletions src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ public AnalysisStratumRule(Morpher morpher, Stratum stratum)
FreezableEqualityComparer<Word>.Default
);
_mrulesRule = null;
IEnumerable<IRule<Word, ShapeNode>> mrules = stratum.MorphologicalRules
.Select(mrule => mrule.CompileAnalysisRule(morpher))
IEnumerable<IRule<Word, ShapeNode>> mrules = stratum
.MorphologicalRules.Select(mrule => mrule.CompileAnalysisRule(morpher))
.Reverse();
switch (stratum.MorphologicalRuleOrder)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -141,9 +141,9 @@ public Shape Segment(string str)
int errorPos;
if (GetShapeNodes(str, out nodes, out errorPos))
{
var shape = new Shape(
begin => new ShapeNode(begin ? HCFeatureSystem.LeftSideAnchor : HCFeatureSystem.RightSideAnchor)
);
var shape = new Shape(begin => new ShapeNode(
begin ? HCFeatureSystem.LeftSideAnchor : HCFeatureSystem.RightSideAnchor
));
shape.AddRange(nodes);
return shape;
}
Expand All @@ -164,9 +164,9 @@ public int TrySegment(string str, out Shape shape)
int errorPos;
if (GetShapeNodes(str, out nodes, out errorPos))
{
shape = new Shape(
begin => new ShapeNode(begin ? HCFeatureSystem.LeftSideAnchor : HCFeatureSystem.RightSideAnchor)
);
shape = new Shape(begin => new ShapeNode(
begin ? HCFeatureSystem.LeftSideAnchor : HCFeatureSystem.RightSideAnchor
));
shape.AddRange(nodes);
return errorPos;
}
Expand Down
Loading

0 comments on commit 6fa37e9

Please sign in to comment.