Adding error codes as attributes to the Recipe Engine and Driver metr…

…ics (#6205)

# Description
Adding error codes as attributes to the Recipe Engine and Driver metrics

## Type of change
- This pull request is a minor refactor, code cleanup, test improvement,
or other maintenance task and doesn't change the functionality of Radius
(issue link optional).

Fixes: #issue_number

## Auto-generated summary

<!--
copilot:all
-->
### <samp>🤖 Generated by Copilot at c046059</samp>

### Summary
📊🚚🐛

<!--
1. 📊 This emoji represents the improvement of the metric recording and
reporting by using a more specific and consistent constant.
2. 🚚 This emoji represents the addition of new error codes for different
types of failures in the recipes package, which makes them more
descriptive and reusable.
3. 🐛 This emoji represents the bug fix in the test case by using the
correct error message.
-->
This pull request improves the error handling and reporting in the
`recipes` package by using consistent and descriptive error codes and
metrics. It also updates the test cases and the `bicep` and `terraform`
drivers to use the new error codes and metrics.

> _`engine.go` changes_
> _better errors and metrics_
> _autumn of bugs ends_

### Walkthrough
* Add more descriptive and consistent error codes for recipe failures
([link](https://github.com/radius-project/radius/pull/6205/files?diff=unified&w=0#diff-4811c2c3581e1effba7ad029717fa82713a8e3a58b848b3845cfaf7d6a44fbacR37-R42),
[link](https://github.com/radius-project/radius/pull/6205/files?diff=unified&w=0#diff-06702c6df95f9efe532b70614e1341244f575e5cbc030db57af268ecbfcd8c59L72-R79),
[link](https://github.com/radius-project/radius/pull/6205/files?diff=unified&w=0#diff-06702c6df95f9efe532b70614e1341244f575e5cbc030db57af268ecbfcd8c59L117-R126),
[link](https://github.com/radius-project/radius/pull/6205/files?diff=unified&w=0#diff-3fdb9931c1f2ddfc5b14d87f47b4124f2bdc1da187219fa1ad9cbe79fc326528L276-R276),
[link](https://github.com/radius-project/radius/pull/6205/files?diff=unified&w=0#diff-3fdb9931c1f2ddfc5b14d87f47b4124f2bdc1da187219fa1ad9cbe79fc326528L438-R438))
* Use error codes from `GetRecipeErrorDetails` to record metrics in
`engine.go`
([link](https://github.com/radius-project/radius/pull/6205/files?diff=unified&w=0#diff-06702c6df95f9efe532b70614e1341244f575e5cbc030db57af268ecbfcd8c59L58-R64),
[link](https://github.com/radius-project/radius/pull/6205/files?diff=unified&w=0#diff-06702c6df95f9efe532b70614e1341244f575e5cbc030db57af268ecbfcd8c59L103-R111))
* Use `RecipeDownloadFailed` instead of `FailedOperationState` to record
recipe download metrics in `bicep.go` and `execute.go`
([link](https://github.com/radius-project/radius/pull/6205/files?diff=unified&w=0#diff-f12e37706fa75e9e04b8dfa01a45cf9bf2a0463e1962605b2420d0cb343922a0L81-R81),
[link](https://github.com/radius-project/radius/pull/6205/files?diff=unified&w=0#diff-41ec31c0ceb8bd50c0329b55a8d27aeba69648059faef4e785302155f4e97482L295-R295))

grafana/radius-resource-provider-dashboard.json

@@ -1922,7 +1922,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "(sum(rate(recipe_operation_duration_count{operation_state=\"failed\"}[$__rate_interval])) by (recipe_template_path, operation_type)) / (sum(rate(recipe_operation_duration_count[$__rate_interval])) by (recipe_template_path, operation_type))",
+          "expr": "(sum(rate(recipe_operation_duration_count{operation_state!=\"success\"}[$__rate_interval])) by (recipe_template_path, operation_type)) / (sum(rate(recipe_operation_duration_count[$__rate_interval])) by (recipe_template_path, operation_type))",
           "format": "time_series",
           "instant": false,
           "interval": "",

pkg/recipes/driver/bicep.go

@@ -78,7 +78,7 @@ func (d *bicepDriver) Execute(ctx context.Context, opts ExecuteOptions) (*recipe
 	err := util.ReadFromRegistry(ctx, opts.Definition.TemplatePath, &recipeData)
 	if err != nil {
 		metrics.DefaultRecipeEngineMetrics.RecordRecipeDownloadDuration(ctx, downloadStartTime,
-			metrics.NewRecipeAttributes(metrics.RecipeEngineOperationDownloadRecipe, opts.Recipe.Name, &opts.Definition, metrics.FailedOperationState))
+			metrics.NewRecipeAttributes(metrics.RecipeEngineOperationDownloadRecipe, opts.Recipe.Name, &opts.Definition, recipes.RecipeDownloadFailed))
 		return nil, recipes.NewRecipeError(recipes.RecipeDownloadFailed, err.Error(), recipes.GetRecipeErrorDetails(err))
 	}
 	metrics.DefaultRecipeEngineMetrics.RecordRecipeDownloadDuration(ctx, downloadStartTime,

pkg/recipes/engine/engine.go

@@ -55,6 +55,9 @@ func (e *engine) Execute(ctx context.Context, recipe recipes.ResourceMetadata, p
 	recipeOutput, definition, err := e.executeCore(ctx, recipe, prevState)
 	if err != nil {
 		result = metrics.FailedOperationState
+		if recipes.GetRecipeErrorDetails(err) != nil {
+			result = recipes.GetRecipeErrorDetails(err).Code
+		}
 	}
 
 	metrics.DefaultRecipeEngineMetrics.RecordRecipeOperationDuration(ctx, executionStart,
@@ -100,6 +103,9 @@ func (e *engine) Delete(ctx context.Context, recipe recipes.ResourceMetadata, ou
 	definition, err := e.deleteCore(ctx, recipe, outputResources)
 	if err != nil {
 		result = metrics.FailedOperationState
+		if recipes.GetRecipeErrorDetails(err) != nil {
+			result = recipes.GetRecipeErrorDetails(err).Code
+		}
 	}
 
 	metrics.DefaultRecipeEngineMetrics.RecordRecipeOperationDuration(ctx, deletionStart,

pkg/recipes/terraform/execute.go

@@ -292,7 +292,7 @@ func downloadAndInspect(ctx context.Context, workingDir string, execPath string,
 	if err := downloadModule(ctx, workingDir, execPath); err != nil {
 		metrics.DefaultRecipeEngineMetrics.RecordRecipeDownloadDuration(ctx, downloadStartTime,
 			metrics.NewRecipeAttributes(metrics.RecipeEngineOperationDownloadRecipe, options.EnvRecipe.Name,
-				options.EnvRecipe, metrics.FailedOperationState))
+				options.EnvRecipe, recipes.RecipeDownloadFailed))
 		return nil, recipes.NewRecipeError(recipes.RecipeDownloadFailed, err.Error(), recipes.GetRecipeErrorDetails(err))
 	}
 	metrics.DefaultRecipeEngineMetrics.RecordRecipeDownloadDuration(ctx, downloadStartTime,