From def2c21bbf9cb3628e8e6837e9d31445699bfcd5 Mon Sep 17 00:00:00 2001 From: Mark Allen <3417310+maallen@users.noreply.github.com> Date: Mon, 18 Nov 2024 13:40:15 +0000 Subject: [PATCH] Added AI translation quality metrics (#193) * Added AI translation quality metrics * Added unit test to verify metric logging * Reduce default edit distance max * Parameterise version * Use uniform config naming * Use LocalService, drop @Transactional in tests --- pom.xml | 7 + .../box/l10n/mojito/service/tm/TMService.java | 73 ++++++ .../l10n/mojito/service/tm/TMServiceTest.java | 234 ++++++++++++++++++ 3 files changed, 314 insertions(+) diff --git a/pom.xml b/pom.xml index 60dc1473a2..eb3f58acc9 100644 --- a/pom.xml +++ b/pom.xml @@ -38,6 +38,7 @@ true 1.9.21 2.13.5 + 1.12.0 @@ -80,6 +81,12 @@ ${icu4j.version} + + org.apache.commons + commons-text + ${commons.text.version} + + diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/TMService.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/TMService.java index 5bb8e238dc..bfaef4b208 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/TMService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/TMService.java @@ -61,6 +61,7 @@ import com.google.common.base.Preconditions; import com.ibm.icu.text.MessageFormat; import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tags; import io.micrometer.core.instrument.Timer; import jakarta.persistence.EntityManager; import java.io.ByteArrayOutputStream; @@ -79,6 +80,7 @@ import net.sf.okapi.steps.common.FilterEventsWriterStep; import net.sf.okapi.steps.common.RawDocumentToFilterEventsStep; import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.text.similarity.LevenshteinDistance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -150,6 +152,15 @@ public class TMService { @Value("${l10n.tmService.quartz.schedulerName:" + DEFAULT_SCHEDULER_NAME + "}") String schedulerName; + @Value("${l10n.ai.translation.review.similarity.editDistanceMax:50}") + int editDistanceMax; + + @Value("${l10n.ai.translation.review.similarity.highPercentage:90}") + int aiTranslationSimilarityHighPercentage; + + @Value("${l10n.ai.translation.review.similarity.mediumPercentage:70}") + int aiTranslationSimilarityMediumPercentage; + /** * Adds a {@link TMTextUnit} in a {@link TM}. * @@ -608,6 +619,10 @@ public AddTMTextUnitCurrentVariantResult addTMTextUnitCurrentVariantWithResult( boolean overridden = checkOverridden && currentTmTextUnitVariant.getStatus() == TMTextUnitVariant.Status.OVERRIDDEN; + if (currentTmTextUnitVariant.getStatus() == TMTextUnitVariant.Status.MT_REVIEW_NEEDED + && status == TMTextUnitVariant.Status.APPROVED) { + logAiReviewMetrics(content, currentTmTextUnitVariant, localeId); + } boolean updateNeeded = !overridden && isUpdateNeededForTmTextUnitVariant( @@ -653,6 +668,64 @@ && isUpdateNeededForTmTextUnitVariant( return new AddTMTextUnitCurrentVariantResult(!noUpdate, tmTextUnitCurrentVariant); } + private void logAiReviewMetrics( + String reviewedTranslation, TMTextUnitVariant currentTmTextUnitVariant, Long localeId) { + if (currentTmTextUnitVariant.getContent().equals(reviewedTranslation)) { + meterRegistry + .counter( + "AiTranslation.review.similarity.match", + Tags.of("locale", localeService.findById(localeId).getBcp47Tag())) + .increment(); + } else { + // Translation has been updated in review, check similarity of original to new + logSimilarityMetrics(reviewedTranslation, currentTmTextUnitVariant, localeId); + } + } + + private void logSimilarityMetrics( + String reviewedTranslation, TMTextUnitVariant currentTmTextUnitVariant, Long localeId) { + LevenshteinDistance levenshteinDistance = new LevenshteinDistance(editDistanceMax); + int editDistance = + levenshteinDistance.apply(currentTmTextUnitVariant.getContent(), reviewedTranslation); + if (editDistance < 0) { + // Negative edit distance means the edit distance threshold was exceeded, log as low + // similarity + meterRegistry + .counter( + "AiTranslation.review.similarity.low", + Tags.of("locale", localeService.findById(localeId).getBcp47Tag())) + .increment(); + } else { + double similarityPercentage = + calculateSimilarityPercentage( + currentTmTextUnitVariant.getContent(), reviewedTranslation, editDistance); + if (similarityPercentage >= aiTranslationSimilarityHighPercentage) { + meterRegistry + .counter( + "AiTranslation.review.similarity.high", + Tags.of("locale", localeService.findById(localeId).getBcp47Tag())) + .increment(); + } else if (similarityPercentage >= aiTranslationSimilarityMediumPercentage) { + meterRegistry + .counter( + "AiTranslation.review.similarity.medium", + Tags.of("locale", localeService.findById(localeId).getBcp47Tag())) + .increment(); + } else { + meterRegistry + .counter( + "AiTranslation.review.similarity.low", + Tags.of("locale", localeService.findById(localeId).getBcp47Tag())) + .increment(); + } + } + } + + private double calculateSimilarityPercentage(String original, String updated, int editDistance) { + int maxLength = Math.max(original.length(), updated.length()); + return ((double) (maxLength - editDistance) / maxLength) * 100; + } + public AddTMTextUnitCurrentVariantResult addTMTextUnitCurrentVariantWithResult( TMTextUnitCurrentVariant tmTextUnitCurrentVariant, Long tmId, diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java index 0bc2813f02..e50235bc6e 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java @@ -48,6 +48,9 @@ import com.google.common.base.Function; import com.google.common.collect.FluentIterable; import com.google.common.collect.Lists; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -63,6 +66,7 @@ import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; +import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -4808,4 +4812,234 @@ public void testAddTMTextUnitWithOverriddenStatus() throws RepositoryNameAlready assertEquals("this is the newest content", textUnitDTOFromSearch.getTarget()); assertEquals(TMTextUnitVariant.Status.APPROVED, textUnitDTOFromSearch.getStatus()); } + + @Test + public void testMTReviewMetricsLoggingTranslationUpdatedMediumSimilarity() + throws RepositoryNameAlreadyUsedException { + MeterRegistry meterRegistry = Mockito.spy(new SimpleMeterRegistry()); + this.tmService.meterRegistry = meterRegistry; + createTestData(); + + Long textUnitId = + addTextUnitAndCheck( + this.tmId, + this.assetId, + "mtReviewMetricsLogging", + "mt translation content", + "some comment", + "3212c3beb09db681379b7a1ed9f37bfe", + "5f3ca19eb49f50b55326065f4185dadd"); + + Locale targetLocale = this.localeService.findByBcp47Tag("fr-FR"); + + TMTextUnitCurrentVariant tmTextUnitCurrentVariant = + this.tmService.addTMTextUnitCurrentVariant( + textUnitId, + targetLocale.getId(), + "mt translation content", + "some comment", + TMTextUnitVariant.Status.MT_REVIEW_NEEDED, + false); + + this.tmService.addTMTextUnitCurrentVariantWithResult( + tmTextUnitCurrentVariant, + this.tmId, + this.assetId, + textUnitId, + tmTextUnitCurrentVariant.getLocale().getId(), + "mt translation content changed", + "some comment", + TMTextUnitVariant.Status.APPROVED, + true, + JSR310Migration.dateTimeNow(), + null, + false); + + Mockito.verify(meterRegistry, Mockito.times(1)) + .counter("AiTranslation.review.similarity.medium", Tags.of("locale", "fr-FR")); + } + + @Test + public void testMTReviewMetricsLoggingTranslationUpdatedHighSimilarity() + throws RepositoryNameAlreadyUsedException { + MeterRegistry meterRegistry = Mockito.spy(new SimpleMeterRegistry()); + this.tmService.meterRegistry = meterRegistry; + createTestData(); + + Long textUnitId = + addTextUnitAndCheck( + this.tmId, + this.assetId, + "mtReviewMetricsLogging", + "mt translation content", + "some comment", + "3212c3beb09db681379b7a1ed9f37bfe", + "5f3ca19eb49f50b55326065f4185dadd"); + + Locale targetLocale = this.localeService.findByBcp47Tag("fr-FR"); + + TMTextUnitCurrentVariant tmTextUnitCurrentVariant = + this.tmService.addTMTextUnitCurrentVariant( + textUnitId, + targetLocale.getId(), + "mt translation content", + "some comment", + TMTextUnitVariant.Status.MT_REVIEW_NEEDED, + false); + + this.tmService.addTMTextUnitCurrentVariantWithResult( + tmTextUnitCurrentVariant, + this.tmId, + this.assetId, + textUnitId, + tmTextUnitCurrentVariant.getLocale().getId(), + "mt translations content", + "some comment", + TMTextUnitVariant.Status.APPROVED, + true, + JSR310Migration.dateTimeNow(), + null, + false); + + Mockito.verify(meterRegistry, Mockito.times(1)) + .counter("AiTranslation.review.similarity.high", Tags.of("locale", "fr-FR")); + } + + @Test + public void testMTReviewMetricsLoggingTranslationUpdatedLowSimilarity() + throws RepositoryNameAlreadyUsedException { + MeterRegistry meterRegistry = Mockito.spy(new SimpleMeterRegistry()); + this.tmService.meterRegistry = meterRegistry; + createTestData(); + + Long textUnitId = + addTextUnitAndCheck( + this.tmId, + this.assetId, + "mtReviewMetricsLogging", + "mt translation content", + "some comment", + "3212c3beb09db681379b7a1ed9f37bfe", + "5f3ca19eb49f50b55326065f4185dadd"); + + Locale targetLocale = this.localeService.findByBcp47Tag("fr-FR"); + + TMTextUnitCurrentVariant tmTextUnitCurrentVariant = + this.tmService.addTMTextUnitCurrentVariant( + textUnitId, + targetLocale.getId(), + "mt translation content", + "some comment", + TMTextUnitVariant.Status.MT_REVIEW_NEEDED, + false); + + this.tmService.addTMTextUnitCurrentVariantWithResult( + tmTextUnitCurrentVariant, + this.tmId, + this.assetId, + textUnitId, + tmTextUnitCurrentVariant.getLocale().getId(), + "completely different", + "some comment", + TMTextUnitVariant.Status.APPROVED, + true, + JSR310Migration.dateTimeNow(), + null, + false); + + Mockito.verify(meterRegistry, Mockito.times(1)) + .counter("AiTranslation.review.similarity.low", Tags.of("locale", "fr-FR")); + } + + @Test + public void testMTReviewMetricsLoggingTranslationMatch() + throws RepositoryNameAlreadyUsedException { + MeterRegistry meterRegistry = Mockito.spy(new SimpleMeterRegistry()); + this.tmService.meterRegistry = meterRegistry; + createTestData(); + + Long textUnitId = + addTextUnitAndCheck( + this.tmId, + this.assetId, + "mtReviewMetricsLogging", + "mt translation content", + "some comment", + "3212c3beb09db681379b7a1ed9f37bfe", + "5f3ca19eb49f50b55326065f4185dadd"); + + Locale targetLocale = this.localeService.findByBcp47Tag("fr-FR"); + + TMTextUnitCurrentVariant tmTextUnitCurrentVariant = + this.tmService.addTMTextUnitCurrentVariant( + textUnitId, + targetLocale.getId(), + "mt translation content", + "some comment", + TMTextUnitVariant.Status.MT_REVIEW_NEEDED, + false); + + this.tmService.addTMTextUnitCurrentVariantWithResult( + tmTextUnitCurrentVariant, + this.tmId, + this.assetId, + textUnitId, + tmTextUnitCurrentVariant.getLocale().getId(), + "mt translation content", + "some comment", + TMTextUnitVariant.Status.APPROVED, + true, + JSR310Migration.dateTimeNow(), + null, + false); + + Mockito.verify(meterRegistry, Mockito.times(1)) + .counter("AiTranslation.review.similarity.match", Tags.of("locale", "fr-FR")); + } + + @Test + public void testMTReviewMetricsLoggingTranslationNotApproved() + throws RepositoryNameAlreadyUsedException { + MeterRegistry meterRegistry = Mockito.spy(new SimpleMeterRegistry()); + this.tmService.meterRegistry = meterRegistry; + createTestData(); + + Long textUnitId = + addTextUnitAndCheck( + this.tmId, + this.assetId, + "mtReviewMetricsLogging", + "mt translation content", + "some comment", + "3212c3beb09db681379b7a1ed9f37bfe", + "5f3ca19eb49f50b55326065f4185dadd"); + + Locale targetLocale = this.localeService.findByBcp47Tag("fr-FR"); + + TMTextUnitCurrentVariant tmTextUnitCurrentVariant = + this.tmService.addTMTextUnitCurrentVariant( + textUnitId, + targetLocale.getId(), + "mt translation content", + "some comment", + TMTextUnitVariant.Status.MT_REVIEW_NEEDED, + false); + + this.tmService.addTMTextUnitCurrentVariantWithResult( + tmTextUnitCurrentVariant, + this.tmId, + this.assetId, + textUnitId, + tmTextUnitCurrentVariant.getLocale().getId(), + "mt translation content", + "some comment", + TMTextUnitVariant.Status.REVIEW_NEEDED, + true, + JSR310Migration.dateTimeNow(), + null, + false); + + Mockito.verify(meterRegistry, Mockito.times(0)) + .counter("AiTranslation.review.similarity.match", Tags.of("locale", "fr-FR")); + } }