telemetry(amazonq): emit TransformEvent metric from download ZIP (#5038)

* telemetry(amazonq): emit TransformEvent metric from download ZIP * add LOC submitted data point * emit metric only after initial download * fix test * handle case where metrics.json is missing * add try-catch * update api model * remove accidentally-committed file * remove notifys * remove unused import * fix unit test * address rli comments --------- Co-authored-by: David Hasani <[email protected]>
aws · Nov 13, 2024 · 662a251 · 662a251
1 parent c8d82c3
commit 662a251
Show file tree

Hide file tree

Showing 11 changed files with 132 additions and 28 deletions.
diff --git a/...-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/ArtifactHandler.kt b/...-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/ArtifactHandler.kt
@@ -39,6 +39,7 @@ import software.aws.toolkits.jetbrains.services.codemodernizer.model.DownloadFai
 import software.aws.toolkits.jetbrains.services.codemodernizer.model.JobId
 import software.aws.toolkits.jetbrains.services.codemodernizer.model.ParseZipFailureReason
 import software.aws.toolkits.jetbrains.services.codemodernizer.model.UnzipFailureReason
+import software.aws.toolkits.jetbrains.services.codemodernizer.state.CodeModernizerSessionState
 import software.aws.toolkits.jetbrains.services.codemodernizer.utils.getPathToHilArtifactDir
 import software.aws.toolkits.jetbrains.services.codemodernizer.utils.isValidCodeTransformConnection
 import software.aws.toolkits.jetbrains.services.codemodernizer.utils.openTroubleshootingGuideNotificationAction
@@ -199,6 +200,17 @@ class ArtifactHandler(private val project: Project, private val clientAdaptor: G
                     downloadedBuildLogPath[job] = path
                 } else {
                     downloadedArtifacts[job] = path
+                    if (output.artifact is CodeModernizerArtifact && output.artifact.metrics != null) {
+                        output.artifact.metrics.linesOfCodeSubmitted = CodeModernizerSessionState.getInstance(project).getLinesOfCodeSubmitted()
+                        output.artifact.metrics.programmingLanguage = CodeModernizerSessionState.getInstance(project).getTransformationLanguage()
+                        try {
+                            clientAdaptor.sendTransformTelemetryEvent(job, output.artifact.metrics)
+                        } catch (e: Exception) {
+                            // log error, but can still show diff.patch and summary.md
+                            LOG.error { e.message.toString() }
+                            telemetryErrorMessage = "Unexpected error when sending telemetry with metrics ${e.localizedMessage}"
+                        }
+                    }
                 }
                 output
             } catch (e: RuntimeException) {

diff --git a/...mmunity/src/software/aws/toolkits/jetbrains/services/codemodernizer/client/GumbyClient.kt b/...mmunity/src/software/aws/toolkits/jetbrains/services/codemodernizer/client/GumbyClient.kt
@@ -16,6 +16,7 @@ import software.amazon.awssdk.services.codewhispererruntime.model.GetTransformat
 import software.amazon.awssdk.services.codewhispererruntime.model.GetTransformationPlanResponse
 import software.amazon.awssdk.services.codewhispererruntime.model.GetTransformationRequest
 import software.amazon.awssdk.services.codewhispererruntime.model.GetTransformationResponse
+import software.amazon.awssdk.services.codewhispererruntime.model.IdeCategory
 import software.amazon.awssdk.services.codewhispererruntime.model.ResumeTransformationRequest
 import software.amazon.awssdk.services.codewhispererruntime.model.ResumeTransformationResponse
 import software.amazon.awssdk.services.codewhispererruntime.model.StartTransformationRequest
@@ -46,8 +47,11 @@ import software.aws.toolkits.jetbrains.services.amazonq.CONTENT_SHA256
 import software.aws.toolkits.jetbrains.services.amazonq.SERVER_SIDE_ENCRYPTION
 import software.aws.toolkits.jetbrains.services.amazonq.SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID
 import software.aws.toolkits.jetbrains.services.amazonq.clients.AmazonQStreamingClient
+import software.aws.toolkits.jetbrains.services.amazonq.codeWhispererUserContext
+import software.aws.toolkits.jetbrains.services.codemodernizer.model.CodeModernizerMetrics
 import software.aws.toolkits.jetbrains.services.codemodernizer.model.JobId
 import software.aws.toolkits.jetbrains.services.codemodernizer.utils.calculateTotalLatency
+import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.getTelemetryOptOutPreference
 import java.io.File
 import java.net.HttpURLConnection
 import java.time.Instant
@@ -210,6 +214,26 @@ class GumbyClient(private val project: Project) {
             }
     }
 
+    fun sendTransformTelemetryEvent(job: JobId, metrics: CodeModernizerMetrics) {
+        bearerClient().sendTelemetryEvent { requestBuilder ->
+            requestBuilder.telemetryEvent { telemetryEventBuilder ->
+                telemetryEventBuilder.transformEvent {
+                    it.jobId(job.id)
+                    it.timestamp(Instant.now())
+                    it.ideCategory(IdeCategory.JETBRAINS)
+                    it.programmingLanguage { language ->
+                        language.languageName(metrics.programmingLanguage)
+                    }
+                    it.linesOfCodeChanged(metrics.linesOfCodeChanged)
+                    it.charsOfCodeChanged(metrics.charactersOfCodeChanged)
+                    it.linesOfCodeSubmitted(metrics.linesOfCodeSubmitted) // currently unavailable for SQL conversions
+                }
+            }
+            requestBuilder.optOutPreference(getTelemetryOptOutPreference())
+            requestBuilder.userContext(codeWhispererUserContext())
+        }
+    }
+
     companion object {
         private val LOG = getLogger<GumbyClient>()
 

diff --git a/...c/software/aws/toolkits/jetbrains/services/codemodernizer/model/CodeModernizerArtifact.kt b/...c/software/aws/toolkits/jetbrains/services/codemodernizer/model/CodeModernizerArtifact.kt
@@ -5,6 +5,7 @@ package software.aws.toolkits.jetbrains.services.codemodernizer.model
 
 import com.fasterxml.jackson.core.JsonProcessingException
 import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
+import com.fasterxml.jackson.module.kotlin.readValue
 import com.intellij.openapi.util.io.FileUtil.createTempDirectory
 import com.intellij.openapi.vfs.LocalFileSystem
 import com.intellij.openapi.vfs.VirtualFile
@@ -29,17 +30,19 @@ open class CodeModernizerArtifact(
     private val patches: List<VirtualFile>,
     val summary: TransformationSummary,
     val summaryMarkdownFile: File,
+    val metrics: CodeModernizerMetrics?,
 ) : CodeTransformDownloadArtifact {
     val patch: VirtualFile
         get() = patches.first()
 
     companion object {
-        private const val maxSupportedVersion = 1.0
+        private const val MAX_SUPPORTED_VERSION = 1.0
         private val tempDir = createTempDirectory("codeTransformArtifacts", null)
-        private const val manifestPathInZip = "manifest.json"
-        private const val summaryNameInZip = "summary.md"
+        private const val MANIFEST_FILE_NAME = "manifest.json"
+        private const val SUMMARY_FILE_NAME = "summary.md"
+        private const val METRICS_FILE_NAME = "metrics.json"
         val LOG = getLogger<CodeModernizerArtifact>()
-        private val MAPPER = jacksonObjectMapper()
+        val MAPPER = jacksonObjectMapper()
 
         /**
          * Extracts the file at [zipPath] and uses its contents to produce a [CodeModernizerArtifact].
@@ -53,39 +56,40 @@ open class CodeModernizerArtifact(
                     throw RuntimeException("Could not unzip artifact")
                 }
                 val manifest = loadManifest()
-                if (manifest.version > maxSupportedVersion) {
+                if (manifest.version > MAX_SUPPORTED_VERSION) {
                     // If not supported we can still try to use it, i.e. the versions should largely be backwards compatible
-                    LOG.warn { "Unsupported version: ${manifest.version}" }
+                    LOG.warn { "Unsupported manifest.json version: ${manifest.version}" }
                 }
                 val patches = extractPatches(manifest)
                 val summary = extractSummary(manifest)
                 val summaryMarkdownFile = getSummaryFile(manifest)
+                val metrics = loadMetrics(manifest)
                 if (patches.size != 1) throw RuntimeException("Expected 1 patch, but found ${patches.size}")
-                return CodeModernizerArtifact(zipPath, manifest, patches, summary, summaryMarkdownFile)
+                return CodeModernizerArtifact(zipPath, manifest, patches, summary, summaryMarkdownFile, metrics)
             }
             throw RuntimeException("Could not find artifact")
         }
 
         private fun extractSummary(manifest: CodeModernizerManifest): TransformationSummary {
-            val summaryFile = tempDir.toPath().resolve(manifest.summaryRoot).resolve(summaryNameInZip).toFile()
+            val summaryFile = tempDir.toPath().resolve(manifest.summaryRoot).resolve(SUMMARY_FILE_NAME).toFile()
             if (!summaryFile.exists() || summaryFile.isDirectory) {
                 throw RuntimeException("The summary in the downloaded zip had an unknown format")
             }
             return TransformationSummary(summaryFile.readText())
         }
 
-        private fun getSummaryFile(manifest: CodeModernizerManifest) = tempDir.toPath().resolve(manifest.summaryRoot).resolve(summaryNameInZip).toFile()
+        private fun getSummaryFile(manifest: CodeModernizerManifest) = tempDir.toPath().resolve(manifest.summaryRoot).resolve(SUMMARY_FILE_NAME).toFile()
 
         /**
          * Attempts to load the manifest from the zip file. Throws an exception if the manifest is not found or cannot be serialized.
          */
         private fun loadManifest(): CodeModernizerManifest {
             val manifestFile =
                 tempDir.listFiles()
-                    ?.firstOrNull { it.name.endsWith(manifestPathInZip) }
+                    ?.firstOrNull { it.name.endsWith(MANIFEST_FILE_NAME) }
                     ?: throw RuntimeException("Could not find manifest")
             try {
-                val manifest = MAPPER.readValue(manifestFile, CodeModernizerManifest::class.java)
+                val manifest = MAPPER.readValue<CodeModernizerManifest>(manifestFile)
                 if (manifest.version == 0.0F) {
                     throw RuntimeException(
                         "Unable to deserialize the manifest",
@@ -97,6 +101,19 @@ open class CodeModernizerArtifact(
             }
         }
 
+        private fun loadMetrics(manifest: CodeModernizerManifest): CodeModernizerMetrics? {
+            try {
+                val metricsFile =
+                    tempDir.resolve(manifest.metricsRoot).listFiles()
+                        ?.firstOrNull { it.name.endsWith(METRICS_FILE_NAME) }
+                        ?: throw RuntimeException("Could not find metrics.json")
+                return MAPPER.readValue<CodeModernizerMetrics>(metricsFile)
+            } catch (exception: Exception) {
+                // if metrics.json not present or parsing fails, can still show diff.patch and summary.md
+                return null
+            }
+        }
+
         @OptIn(ExperimentalPathApi::class)
         private fun extractPatches(manifest: CodeModernizerManifest): List<VirtualFile> {
             val fileSystem = LocalFileSystem.getInstance()

diff --git a/...c/software/aws/toolkits/jetbrains/services/codemodernizer/model/CodeModernizerManifest.kt b/...c/software/aws/toolkits/jetbrains/services/codemodernizer/model/CodeModernizerManifest.kt
@@ -6,4 +6,4 @@ package software.aws.toolkits.jetbrains.services.codemodernizer.model
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties
 
 @JsonIgnoreProperties(ignoreUnknown = true)
-data class CodeModernizerManifest(val version: Float, val patchesRoot: String, val artifactsRoot: String, val summaryRoot: String)
+data class CodeModernizerManifest(val version: Float, val patchesRoot: String, val artifactsRoot: String, val summaryRoot: String, val metricsRoot: String)
diff --git a/...rc/software/aws/toolkits/jetbrains/services/codemodernizer/model/CodeModernizerMetrics.kt b/...rc/software/aws/toolkits/jetbrains/services/codemodernizer/model/CodeModernizerMetrics.kt
@@ -0,0 +1,14 @@
+// Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package software.aws.toolkits.jetbrains.services.codemodernizer.model
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+data class CodeModernizerMetrics(
+    val linesOfCodeChanged: Int?,
+    val charactersOfCodeChanged: Int?,
+    var linesOfCodeSubmitted: Int?,
+    var programmingLanguage: String?,
+)
diff --git a/.../software/aws/toolkits/jetbrains/services/codemodernizer/plan/CodeModernizerPlanEditor.kt b/.../software/aws/toolkits/jetbrains/services/codemodernizer/plan/CodeModernizerPlanEditor.kt
@@ -3,7 +3,6 @@
 
 package software.aws.toolkits.jetbrains.services.codemodernizer.plan
 
-import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import com.fasterxml.jackson.module.kotlin.readValue
 import com.intellij.ide.BrowserUtil
 import com.intellij.openapi.fileEditor.FileEditor
@@ -22,11 +21,14 @@ import software.aws.toolkits.jetbrains.services.codemodernizer.constants.APPENDI
 import software.aws.toolkits.jetbrains.services.codemodernizer.constants.CodeModernizerUIConstants
 import software.aws.toolkits.jetbrains.services.codemodernizer.constants.JOB_STATISTICS_TABLE_KEY
 import software.aws.toolkits.jetbrains.services.codemodernizer.constants.LOC_THRESHOLD
+import software.aws.toolkits.jetbrains.services.codemodernizer.model.CodeModernizerArtifact.Companion.MAPPER
 import software.aws.toolkits.jetbrains.services.codemodernizer.model.PlanTable
 import software.aws.toolkits.jetbrains.services.codemodernizer.plan.CodeModernizerPlanEditorProvider.Companion.MIGRATION_PLAN_KEY
 import software.aws.toolkits.jetbrains.services.codemodernizer.utils.getAuthType
 import software.aws.toolkits.jetbrains.services.codemodernizer.utils.getBillingText
+import software.aws.toolkits.jetbrains.services.codemodernizer.utils.getLinesOfCodeSubmitted
 import software.aws.toolkits.jetbrains.services.codemodernizer.utils.getTableMapping
+import software.aws.toolkits.jetbrains.services.codemodernizer.utils.parseTableMapping
 import software.aws.toolkits.jetbrains.services.codewhisperer.layout.CodeWhispererLayoutConfig.addHorizontalGlue
 import software.aws.toolkits.resources.message
 import software.aws.toolkits.telemetry.CredentialSourceId
@@ -59,13 +61,7 @@ import javax.swing.table.DefaultTableModel
 
 class CodeModernizerPlanEditor(val project: Project, private val virtualFile: VirtualFile) : UserDataHolderBase(), FileEditor {
     val plan = virtualFile.getUserData(MIGRATION_PLAN_KEY) ?: throw RuntimeException("Migration plan not found")
-    private val tableMapping =
-        if (!plan.transformationSteps()[0].progressUpdates().isNullOrEmpty()) {
-            getTableMapping(plan.transformationSteps()[0].progressUpdates())
-        } else {
-            throw RuntimeException("GetPlan response missing step 0 progress updates with table data")
-        }
-    private val mapper = jacksonObjectMapper()
+    private val tableMapping = getTableMapping(plan.transformationSteps()[0].progressUpdates())
 
     // to-do: convert to UI DSL
     private val contentPanel =
@@ -79,8 +75,8 @@ class CodeModernizerPlanEditor(val project: Project, private val virtualFile: Vi
                     // key "0" reserved for job statistics table
                     // comes from "name" field of each progressUpdate in step zero of plan
                     if (JOB_STATISTICS_TABLE_KEY in tableMapping) {
-                        val planTable = mapper.readValue(tableMapping[JOB_STATISTICS_TABLE_KEY], PlanTable::class.java)
-                        val linesOfCode = planTable.rows.find { it.name == "linesOfCode" }?.value?.toInt()
+                        val planTable = parseTableMapping(tableMapping)
+                        val linesOfCode = planTable?.let { getLinesOfCodeSubmitted(it) }
                         if (linesOfCode != null && linesOfCode > LOC_THRESHOLD && getAuthType(project) == CredentialSourceId.IamIdentityCenter) {
                             val billingText = getBillingText(linesOfCode)
                             val billingTextComponent =
@@ -102,15 +98,15 @@ class CodeModernizerPlanEditor(val project: Project, private val virtualFile: Vi
                             add(billingTextComponent, CodeModernizerUIConstants.transformationPlanPlaneConstraint)
                         }
                         add(
-                            transformationPlanInfo(planTable),
+                            planTable?.let { transformationPlanInfo(it) },
                             CodeModernizerUIConstants.transformationPlanPlaneConstraint,
                         )
                     }
                     add(transformationPlanPanel(plan), CodeModernizerUIConstants.transformationPlanPlaneConstraint)
                     // key "-1" reserved for appendix table
                     if (APPENDIX_TABLE_KEY in tableMapping) {
                         add(
-                            transformationPlanAppendix(mapper.readValue(tableMapping[APPENDIX_TABLE_KEY], PlanTable::class.java)),
+                            tableMapping[APPENDIX_TABLE_KEY]?.let { MAPPER.readValue<PlanTable>(it) }?.let { transformationPlanAppendix(it) },
                             CodeModernizerUIConstants.transformationPlanPlaneConstraint,
                         )
                     }
@@ -400,7 +396,7 @@ class CodeModernizerPlanEditor(val project: Project, private val virtualFile: Vi
         val table = tableMapping[step.id()]
 
         val parsedTable = table?.let {
-            mapper.readValue<PlanTable>(it)
+            MAPPER.readValue<PlanTable>(it)
         }
 
         val renderedStepTable = parsedTable?.let {

diff --git a/...ftware/aws/toolkits/jetbrains/services/codemodernizer/state/CodeModernizerSessionState.kt b/...ftware/aws/toolkits/jetbrains/services/codemodernizer/state/CodeModernizerSessionState.kt
@@ -12,6 +12,9 @@ import software.aws.toolkits.jetbrains.services.codemodernizer.model.CodeModerni
 import software.aws.toolkits.jetbrains.services.codemodernizer.model.CodeModernizerSessionContext
 import software.aws.toolkits.jetbrains.services.codemodernizer.model.JobHistoryItem
 import software.aws.toolkits.jetbrains.services.codemodernizer.model.JobId
+import software.aws.toolkits.jetbrains.services.codemodernizer.utils.getLinesOfCodeSubmitted
+import software.aws.toolkits.jetbrains.services.codemodernizer.utils.getTableMapping
+import software.aws.toolkits.jetbrains.services.codemodernizer.utils.parseTableMapping
 import java.time.Duration
 import java.time.Instant
 import kotlin.io.path.Path
@@ -55,4 +58,14 @@ class CodeModernizerSessionState {
     }
 
     fun getJobHistory(): Array<JobHistoryItem> = previousJobHistory.values.toTypedArray()
+
+    // LOC submitted only available for Java upgrades
+    fun getLinesOfCodeSubmitted(): Int? {
+        val tableMapping = transformationPlan?.transformationSteps()?.get(0)?.let { getTableMapping(it.progressUpdates()) }
+        val planTable = tableMapping?.let { parseTableMapping(it) }
+        return planTable?.let { getLinesOfCodeSubmitted(it) }
+    }
+
+    // we only create a transformationPlan for Java upgrades
+    fun getTransformationLanguage(): String = if (transformationPlan != null) "JAVA" else "SQL"
 }
diff --git a/...rc/software/aws/toolkits/jetbrains/services/codemodernizer/utils/CodeTransformApiUtils.kt b/...rc/software/aws/toolkits/jetbrains/services/codemodernizer/utils/CodeTransformApiUtils.kt
@@ -3,6 +3,7 @@
 
 package software.aws.toolkits.jetbrains.services.codemodernizer.utils
 
+import com.fasterxml.jackson.module.kotlin.readValue
 import com.intellij.openapi.project.Project
 import com.intellij.serviceContainer.AlreadyDisposedException
 import kotlinx.coroutines.delay
@@ -23,7 +24,10 @@ import software.aws.toolkits.core.utils.Waiters.waitUntil
 import software.aws.toolkits.jetbrains.services.codemodernizer.CodeTransformTelemetryManager
 import software.aws.toolkits.jetbrains.services.codemodernizer.client.GumbyClient
 import software.aws.toolkits.jetbrains.services.codemodernizer.constants.BILLING_RATE
+import software.aws.toolkits.jetbrains.services.codemodernizer.constants.JOB_STATISTICS_TABLE_KEY
+import software.aws.toolkits.jetbrains.services.codemodernizer.model.CodeModernizerArtifact.Companion.MAPPER
 import software.aws.toolkits.jetbrains.services.codemodernizer.model.JobId
+import software.aws.toolkits.jetbrains.services.codemodernizer.model.PlanTable
 import software.aws.toolkits.resources.message
 import java.time.Duration
 import java.util.Locale
@@ -128,11 +132,23 @@ suspend fun JobId.pollTransformationStatusAndPlan(
 }
 
 // "name" holds the ID of the corresponding plan step (where table will go) and "description" holds the plan data
-fun getTableMapping(stepZeroProgressUpdates: List<TransformationProgressUpdate>) = stepZeroProgressUpdates.associate {
-    it.name() to it.description()
+fun getTableMapping(stepZeroProgressUpdates: List<TransformationProgressUpdate>): Map<String, String> {
+    if (stepZeroProgressUpdates.isNotEmpty()) {
+        return stepZeroProgressUpdates.associate {
+            it.name() to it.description()
+        }
+    } else {
+        error("GetPlan response missing step 0 progress updates with table data")
+    }
 }
 
+fun parseTableMapping(tableMapping: Map<String, String>): PlanTable? =
+    tableMapping[JOB_STATISTICS_TABLE_KEY]?.let { MAPPER.readValue<PlanTable>(it) }
+
 fun getBillingText(linesOfCode: Int): String {
     val estimatedCost = String.format(Locale.US, "%.2f", linesOfCode.times(BILLING_RATE))
     return message("codemodernizer.migration_plan.header.billing_text", linesOfCode, BILLING_RATE, estimatedCost)
 }
+
+fun getLinesOfCodeSubmitted(planTable: PlanTable) =
+    planTable.rows.find { it.name == "linesOfCode" }?.value?.toInt()
diff --git a/plugins/amazonq/codetransform/jetbrains-community/tst-resources/codemodernizer/simple.zip b/plugins/amazonq/codetransform/jetbrains-community/tst-resources/codemodernizer/simple.zip