Allow adding a Search Attribute to fail, but warn. (#75)

* Allow adding a Search Attribute to fail, but warn.

This allows running Omes against Cloud which does not permit search attributes to be added via the WorkflowService API.

Also make cleanup-scenario job IDs unique so that cleanup can be run multiples times on the same namespace.

cmd/cleanup_scenario.go

@@ -7,6 +7,7 @@ import (
 	"os/user"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"github.com/temporalio/omes/cmd/cmdoptions"
@@ -68,7 +69,7 @@ func (c *scenarioCleaner) run(ctx context.Context) error {
 	client := c.clientOptions.MustDial(metrics, c.logger)
 	defer client.Close()
 	taskQueue := loadgen.TaskQueueForRun(c.scenario, c.runID)
-	jobID := "omes-cleanup-" + taskQueue
+	jobID := "omes-cleanup-" + taskQueue + "-" + uuid.New().String()
 	username, hostname := "anonymous", "unknown"
 	if user, err := user.Current(); err == nil {
 		username = user.Name

scenarios/throughput_stress.go

@@ -34,14 +34,23 @@ func (t *tpsExecutor) Run(ctx context.Context, info loadgen.ScenarioInfo) error
 	attribMap := map[string]enums.IndexedValueType{
 		ThroughputStressScenarioIdSearchAttribute: enums.INDEXED_VALUE_TYPE_KEYWORD,
 	}
+
 	_, err := info.Client.OperatorService().AddSearchAttributes(ctx,
 		&operatorservice.AddSearchAttributesRequest{
 			Namespace:        info.Namespace,
 			SearchAttributes: attribMap,
 		})
-	var svcErr *serviceerror.AlreadyExists
-	if !errors.As(err, &svcErr) {
-		return fmt.Errorf("failed adding search attribute: %w", err)
+	var deniedErr *serviceerror.PermissionDenied
+	var alreadyErr *serviceerror.AlreadyExists
+
+	if errors.As(err, &deniedErr) {
+		info.Logger.Warnf("Failed to add Search Attribute %s: %v", ThroughputStressScenarioIdSearchAttribute, err)
+	} else if !errors.As(err, &alreadyErr) {
+		info.Logger.Info("Search Attribute %s already exists", ThroughputStressScenarioIdSearchAttribute)
+
+		return err
+	} else {
+		info.Logger.Info("Search Attribute %s added", ThroughputStressScenarioIdSearchAttribute)
 	}
 
 	// Complain if there are already existing workflows with the provided run id