Update Integration API and use custom tags on structs for better read…

…ability (#8321) Update API and use custom tags on structs Signed-off-by: Matthias Wessendorf <[email protected]>
knative · Nov 19, 2024 · 63d4da5 · 63d4da5
1 parent 7abb04d
commit 63d4da5
Show file tree

Hide file tree

Showing 8 changed files with 83 additions and 57 deletions.
diff --git a/config/core/resources/integrationsource.yaml b/config/core/resources/integrationsource.yaml
@@ -104,7 +104,7 @@ spec:
                     s3:
                       type: object
                       properties:
-                        bucketNameOrArn:
+                        arn:
                           type: string
                           title: Bucket Name
                           description: The S3 Bucket name or Amazon Resource Name (ARN).
@@ -199,7 +199,7 @@ spec:
                     sqs:
                       type: object
                       properties:
-                        queueNameOrArn:
+                        arn:
                           type: string
                           title: Queue Name
                           description: The SQS Queue Name or ARN
@@ -217,7 +217,7 @@ spec:
                           title: Autocreate Queue
                           description: Setting the autocreation of the SQS queue.
                           default: false
-                        amazonAWSHost:
+                        host:
                           type: string
                           title: AWS Host
                           description: The hostname of the Amazon AWS cloud.
@@ -287,7 +287,7 @@ spec:
                           description: The duration (in seconds) that the received messages are
                             hidden from subsequent retrieve requests after being retrieved by
                             a ReceiveMessage request.
-                    ddb-streams:
+                    ddbStreams:
                       type: object
                       properties:
                         table:

diff --git a/docs/eventing-api.md b/docs/eventing-api.md
@@ -7577,7 +7577,7 @@ AWSCommon
 </tr>
 <tr>
 <td>
-<code>bucketNameOrArn</code><br/>
+<code>arn</code><br/>
 <em>
 string
 </em>
@@ -7594,7 +7594,7 @@ bool
 </em>
 </td>
 <td>
-<p>S3 Bucket name or ARN</p>
+<p>S3 ARN</p>
 </td>
 </tr>
 <tr>
@@ -7741,7 +7741,7 @@ AWSCommon
 </tr>
 <tr>
 <td>
-<code>queueNameOrArn</code><br/>
+<code>arn</code><br/>
 <em>
 string
 </em>
@@ -7758,7 +7758,7 @@ bool
 </em>
 </td>
 <td>
-<p>SQS Queue name or ARN</p>
+<p>SQS ARN</p>
 </td>
 </tr>
 <tr>
@@ -7774,7 +7774,7 @@ bool
 </tr>
 <tr>
 <td>
-<code>amazonAWSHost</code><br/>
+<code>host</code><br/>
 <em>
 string
 </em>
@@ -7956,7 +7956,7 @@ AWSSQS
 </tr>
 <tr>
 <td>
-<code>ddb-streams</code><br/>
+<code>ddbStreams</code><br/>
 <em>
 <a href="#sources.knative.dev/v1alpha1.AWSDDBStreams">
 AWSDDBStreams

diff --git a/pkg/apis/sources/v1alpha1/integration_types.go b/pkg/apis/sources/v1alpha1/integration_types.go
@@ -79,33 +79,33 @@ type AWSCommon struct {
 
 type AWSS3 struct {
 	AWSCommon               `json:",inline"` // Embeds AWSCommon to inherit its fields in JSON
-	BucketNameOrArn         string           `json:"bucketNameOrArn,omitempty"`         // S3 Bucket name or ARN
-	DeleteAfterRead         bool             `json:"deleteAfterRead" default:"true"`    // Auto-delete objects after reading
-	MoveAfterRead           bool             `json:"moveAfterRead" default:"false"`     // Move objects after reading
-	DestinationBucket       string           `json:"destinationBucket,omitempty"`       // Destination bucket for moved objects
-	DestinationBucketPrefix string           `json:"destinationBucketPrefix,omitempty"` // Prefix for moved objects
-	DestinationBucketSuffix string           `json:"destinationBucketSuffix,omitempty"` // Suffix for moved objects
-	AutoCreateBucket        bool             `json:"autoCreateBucket" default:"false"`  // Auto-create S3 bucket
-	Prefix                  string           `json:"prefix,omitempty"`                  // S3 bucket prefix for search
-	IgnoreBody              bool             `json:"ignoreBody" default:"false"`        // Ignore object body
-	ForcePathStyle          bool             `json:"forcePathStyle" default:"false"`    // Force path style for bucket access
-	Delay                   int              `json:"delay" default:"500"`               // Delay between polls in milliseconds
-	MaxMessagesPerPoll      int              `json:"maxMessagesPerPoll" default:"10"`   // Max messages to poll per request
+	Arn                     string           `json:"arn,omitempty" camel:"CAMEL_KAMELET_AWS_S3_SOURCE_BUCKETNAMEORARN"` // S3 ARN
+	DeleteAfterRead         bool             `json:"deleteAfterRead" default:"true"`                                    // Auto-delete objects after reading
+	MoveAfterRead           bool             `json:"moveAfterRead" default:"false"`                                     // Move objects after reading
+	DestinationBucket       string           `json:"destinationBucket,omitempty"`                                       // Destination bucket for moved objects
+	DestinationBucketPrefix string           `json:"destinationBucketPrefix,omitempty"`                                 // Prefix for moved objects
+	DestinationBucketSuffix string           `json:"destinationBucketSuffix,omitempty"`                                 // Suffix for moved objects
+	AutoCreateBucket        bool             `json:"autoCreateBucket" default:"false"`                                  // Auto-create S3 bucket
+	Prefix                  string           `json:"prefix,omitempty"`                                                  // S3 bucket prefix for search
+	IgnoreBody              bool             `json:"ignoreBody" default:"false"`                                        // Ignore object body
+	ForcePathStyle          bool             `json:"forcePathStyle" default:"false"`                                    // Force path style for bucket access
+	Delay                   int              `json:"delay" default:"500"`                                               // Delay between polls in milliseconds
+	MaxMessagesPerPoll      int              `json:"maxMessagesPerPoll" default:"10"`                                   // Max messages to poll per request
 }
 
 type AWSSQS struct {
 	AWSCommon          `json:",inline"` // Embeds AWSCommon to inherit its fields in JSON
-	QueueNameOrArn     string           `json:"queueNameOrArn,omitempty"`              // SQS Queue name or ARN
-	DeleteAfterRead    bool             `json:"deleteAfterRead" default:"true"`        // Auto-delete messages after reading
-	AutoCreateQueue    bool             `json:"autoCreateQueue" default:"false"`       // Auto-create SQS queue
-	AmazonAWSHost      string           `json:"amazonAWSHost" default:"amazonaws.com"` // AWS host
-	Protocol           string           `json:"protocol" default:"https"`              // Communication protocol (http/https)
-	QueueURL           string           `json:"queueURL,omitempty"`                    // Full SQS queue URL
-	Greedy             bool             `json:"greedy" default:"false"`                // Greedy scheduler
-	Delay              int              `json:"delay" default:"500"`                   // Delay between polls in milliseconds
-	MaxMessagesPerPoll int              `json:"maxMessagesPerPoll" default:"1"`        // Max messages to return (1-10)
-	WaitTimeSeconds    int              `json:"waitTimeSeconds,omitempty"`             // Wait time for messages
-	VisibilityTimeout  int              `json:"visibilityTimeout,omitempty"`           // Visibility timeout in seconds
+	Arn                string           `json:"arn,omitempty" camel:"CAMEL_KAMELET_AWS_SQS_SOURCE_QUEUENAMEORARN"`               // SQS ARN
+	DeleteAfterRead    bool             `json:"deleteAfterRead" default:"true"`                                                  // Auto-delete messages after reading
+	AutoCreateQueue    bool             `json:"autoCreateQueue" default:"false"`                                                 // Auto-create SQS queue
+	Host               string           `json:"host" camel:"CAMEL_KAMELET_AWS_SQS_SOURCE_AMAZONAWSHOST" default:"amazonaws.com"` // AWS host
+	Protocol           string           `json:"protocol" default:"https"`                                                        // Communication protocol (http/https)
+	QueueURL           string           `json:"queueURL,omitempty"`                                                              // Full SQS queue URL
+	Greedy             bool             `json:"greedy" default:"false"`                                                          // Greedy scheduler
+	Delay              int              `json:"delay" default:"500"`                                                             // Delay between polls in milliseconds
+	MaxMessagesPerPoll int              `json:"maxMessagesPerPoll" default:"1"`                                                  // Max messages to return (1-10)
+	WaitTimeSeconds    int              `json:"waitTimeSeconds,omitempty"`                                                       // Wait time for messages
+	VisibilityTimeout  int              `json:"visibilityTimeout,omitempty"`                                                     // Visibility timeout in seconds
 }
 
 type AWSDDBStreams struct {
@@ -116,9 +116,9 @@ type AWSDDBStreams struct {
 }
 
 type Aws struct {
-	S3         *AWSS3         `json:"s3,omitempty"`          // S3 source configuration
-	SQS        *AWSSQS        `json:"sqs,omitempty"`         // SQS source configuration
-	DDBStreams *AWSDDBStreams `json:"ddb-streams,omitempty"` // DynamoDB Streams source configuration
+	S3         *AWSS3         `json:"s3,omitempty"`         // S3 source configuration
+	SQS        *AWSSQS        `json:"sqs,omitempty"`        // SQS source configuration
+	DDBStreams *AWSDDBStreams `json:"ddbStreams,omitempty"` // DynamoDB Streams source configuration
 	Auth       *Auth          `json:"auth,omitempty"`
 }
 

diff --git a/pkg/apis/sources/v1alpha1/integration_types_test.go b/pkg/apis/sources/v1alpha1/integration_types_test.go
@@ -57,7 +57,7 @@ func TestAWS(t *testing.T) {
 		AWSCommon: AWSCommon{
 			Region: "eu-north-1",
 		},
-		BucketNameOrArn: "example-bucket",
+		Arn: "example-bucket",
 	}
 
 	if s3.Region != "eu-north-1" {
@@ -68,7 +68,7 @@ func TestAWS(t *testing.T) {
 		AWSCommon: AWSCommon{
 			Region: "eu-north-1",
 		},
-		QueueNameOrArn: "example-queue",
+		Arn: "example-queue",
 	}
 
 	if sqs.Region != "eu-north-1" {

diff --git a/pkg/apis/sources/v1alpha1/integration_validation.go b/pkg/apis/sources/v1alpha1/integration_validation.go
@@ -65,8 +65,8 @@ func (spec *IntegrationSourceSpec) Validate(ctx context.Context) *apis.FieldErro
 
 		// Additional validation for AWS S3 required fields
 		if spec.Aws.S3 != nil {
-			if spec.Aws.S3.BucketNameOrArn == "" {
-				errs = errs.Also(apis.ErrMissingField("aws.s3.bucketNameOrArn"))
+			if spec.Aws.S3.Arn == "" {
+				errs = errs.Also(apis.ErrMissingField("aws.s3.arn"))
 			}
 			if spec.Aws.S3.Region == "" {
 				errs = errs.Also(apis.ErrMissingField("aws.s3.region"))
@@ -75,7 +75,7 @@ func (spec *IntegrationSourceSpec) Validate(ctx context.Context) *apis.FieldErro
 
 		// Additional validation for AWS SQS required fields
 		if spec.Aws.SQS != nil {
-			if spec.Aws.SQS.QueueNameOrArn == "" {
+			if spec.Aws.SQS.Arn == "" {
 				errs = errs.Also(apis.ErrMissingField("aws.sqs.queueNameOrArn"))
 			}
 			if spec.Aws.SQS.Region == "" {

diff --git a/pkg/apis/sources/v1alpha1/integration_validation_test.go b/pkg/apis/sources/v1alpha1/integration_validation_test.go
@@ -49,7 +49,7 @@ func TestIntegrationSourceSpecValidation(t *testing.T) {
 						AWSCommon: AWSCommon{
 							Region: "us-east-1",
 						},
-						BucketNameOrArn: "example-bucket",
+						Arn: "example-bucket",
 					},
 					Auth: &Auth{
 						Secret: &Secret{
@@ -70,7 +70,7 @@ func TestIntegrationSourceSpecValidation(t *testing.T) {
 						AWSCommon: AWSCommon{
 							Region: "us-east-1",
 						},
-						QueueNameOrArn: "example-queue",
+						Arn: "example-queue",
 					},
 					Auth: &Auth{
 						Secret: &Secret{
@@ -117,7 +117,7 @@ func TestIntegrationSourceSpecValidation(t *testing.T) {
 						AWSCommon: AWSCommon{
 							Region: "us-east-1",
 						},
-						BucketNameOrArn: "example-bucket",
+						Arn: "example-bucket",
 					},
 				},
 			},
@@ -131,13 +131,13 @@ func TestIntegrationSourceSpecValidation(t *testing.T) {
 						AWSCommon: AWSCommon{
 							Region: "us-east-1",
 						},
-						BucketNameOrArn: "example-bucket",
+						Arn: "example-bucket",
 					},
 					SQS: &AWSSQS{
 						AWSCommon: AWSCommon{
 							Region: "us-east-1",
 						},
-						QueueNameOrArn: "example-queue",
+						Arn: "example-queue",
 					},
 					Auth: &Auth{
 						Secret: &Secret{
@@ -203,7 +203,7 @@ func TestIntegrationSourceSpecValidation(t *testing.T) {
 						AWSCommon: AWSCommon{
 							Region: "us-east-1",
 						},
-						BucketNameOrArn: "example-bucket",
+						Arn: "example-bucket",
 					},
 				},
 			},
@@ -214,7 +214,7 @@ func TestIntegrationSourceSpecValidation(t *testing.T) {
 			spec: IntegrationSourceSpec{
 				Aws: &Aws{
 					S3: &AWSS3{
-						BucketNameOrArn: "example-bucket",
+						Arn: "example-bucket",
 					},
 					Auth: &Auth{
 						Secret: &Secret{

diff --git a/pkg/reconciler/integrationsource/resources/containersource.go b/pkg/reconciler/integrationsource/resources/containersource.go
@@ -89,17 +89,18 @@ func generateEnvVarsFromStruct(prefix string, s interface{}) []corev1.EnvVar {
 			continue
 		}
 
-		// Extract the JSON tag or fall back to the Go field name
-		jsonTag := fieldType.Tag.Get("json")
-		tagName := strings.Split(jsonTag, ",")[0]
-
-		// fallback to Go field name if no JSON tag
-		if tagName == "" || tagName == "-" {
-			tagName = fieldType.Name
+		// First, check for the custom 'camel' tag
+		envVarName := fieldType.Tag.Get("camel")
+		if envVarName == "" {
+			// If 'camel' tag is not present, fall back to the 'json' tag or Go field name
+			jsonTag := fieldType.Tag.Get("json")
+			tagName := strings.Split(jsonTag, ",")[0]
+			if tagName == "" || tagName == "-" {
+				tagName = fieldType.Name
+			}
+			envVarName = fmt.Sprintf("%s_%s", prefix, strings.ToUpper(tagName))
 		}
 
-		envVarName := fmt.Sprintf("%s_%s", prefix, strings.ToUpper(tagName))
-
 		if field.Kind() == reflect.Ptr {
 			if field.IsNil() {
 				continue

diff --git a/pkg/reconciler/integrationsource/resources/containersource_test.go b/pkg/reconciler/integrationsource/resources/containersource_test.go
@@ -129,3 +129,28 @@ func TestGenerateEnvVarsFromStruct(t *testing.T) {
 		t.Errorf("generateEnvVarsFromStruct() mismatch (-want +got):\n%s", diff)
 	}
 }
+
+func TestGenerateEnvVarsFromStruct_S3WithCamelTag(t *testing.T) {
+	type AWSS3 struct {
+		Arn    string `json:"arn,omitempty" camel:"CAMEL_KAMELET_AWS_S3_SOURCE_BUCKETNAMEORARN"`
+		Region string `json:"region,omitempty"`
+	}
+
+	prefix := "CAMEL_KAMELET_AWS_S3_SOURCE"
+	input := AWSS3{
+		Arn:    "arn:aws:s3:::example-bucket",
+		Region: "us-west-2",
+	}
+
+	// Expected environment variables including SSL settings and camel tag for Arn
+	want := []corev1.EnvVar{
+		{Name: "CAMEL_KAMELET_AWS_S3_SOURCE_BUCKETNAMEORARN", Value: "arn:aws:s3:::example-bucket"},
+		{Name: "CAMEL_KAMELET_AWS_S3_SOURCE_REGION", Value: "us-west-2"},
+	}
+
+	got := generateEnvVarsFromStruct(prefix, input)
+
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("generateEnvVarsFromStruct_S3WithCamelTag() mismatch (-want +got):\n%s", diff)
+	}
+}