From 63d4da59c6cd7ce80be98581bb24a0abb0111f08 Mon Sep 17 00:00:00 2001 From: Matthias Wessendorf Date: Tue, 19 Nov 2024 10:00:00 +0100 Subject: [PATCH] Update Integration API and use custom tags on structs for better readability (#8321) Update API and use custom tags on structs Signed-off-by: Matthias Wessendorf --- config/core/resources/integrationsource.yaml | 8 +-- docs/eventing-api.md | 12 ++--- .../sources/v1alpha1/integration_types.go | 52 +++++++++---------- .../v1alpha1/integration_types_test.go | 4 +- .../v1alpha1/integration_validation.go | 6 +-- .../v1alpha1/integration_validation_test.go | 14 ++--- .../resources/containersource.go | 19 +++---- .../resources/containersource_test.go | 25 +++++++++ 8 files changed, 83 insertions(+), 57 deletions(-) diff --git a/config/core/resources/integrationsource.yaml b/config/core/resources/integrationsource.yaml index b671eaf5478..a114426d248 100644 --- a/config/core/resources/integrationsource.yaml +++ b/config/core/resources/integrationsource.yaml @@ -104,7 +104,7 @@ spec: s3: type: object properties: - bucketNameOrArn: + arn: type: string title: Bucket Name description: The S3 Bucket name or Amazon Resource Name (ARN). @@ -199,7 +199,7 @@ spec: sqs: type: object properties: - queueNameOrArn: + arn: type: string title: Queue Name description: The SQS Queue Name or ARN @@ -217,7 +217,7 @@ spec: title: Autocreate Queue description: Setting the autocreation of the SQS queue. default: false - amazonAWSHost: + host: type: string title: AWS Host description: The hostname of the Amazon AWS cloud. @@ -287,7 +287,7 @@ spec: description: The duration (in seconds) that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. - ddb-streams: + ddbStreams: type: object properties: table: diff --git a/docs/eventing-api.md b/docs/eventing-api.md index 79a98c5983d..41220eef825 100644 --- a/docs/eventing-api.md +++ b/docs/eventing-api.md @@ -7577,7 +7577,7 @@ AWSCommon -bucketNameOrArn
+arn
string @@ -7594,7 +7594,7 @@ bool -

S3 Bucket name or ARN

+

S3 ARN

@@ -7741,7 +7741,7 @@ AWSCommon -queueNameOrArn
+arn
string @@ -7758,7 +7758,7 @@ bool -

SQS Queue name or ARN

+

SQS ARN

@@ -7774,7 +7774,7 @@ bool -amazonAWSHost
+host
string @@ -7956,7 +7956,7 @@ AWSSQS -ddb-streams
+ddbStreams
AWSDDBStreams diff --git a/pkg/apis/sources/v1alpha1/integration_types.go b/pkg/apis/sources/v1alpha1/integration_types.go index 15b4e60a2b0..1db79bc8096 100644 --- a/pkg/apis/sources/v1alpha1/integration_types.go +++ b/pkg/apis/sources/v1alpha1/integration_types.go @@ -79,33 +79,33 @@ type AWSCommon struct { type AWSS3 struct { AWSCommon `json:",inline"` // Embeds AWSCommon to inherit its fields in JSON - BucketNameOrArn string `json:"bucketNameOrArn,omitempty"` // S3 Bucket name or ARN - DeleteAfterRead bool `json:"deleteAfterRead" default:"true"` // Auto-delete objects after reading - MoveAfterRead bool `json:"moveAfterRead" default:"false"` // Move objects after reading - DestinationBucket string `json:"destinationBucket,omitempty"` // Destination bucket for moved objects - DestinationBucketPrefix string `json:"destinationBucketPrefix,omitempty"` // Prefix for moved objects - DestinationBucketSuffix string `json:"destinationBucketSuffix,omitempty"` // Suffix for moved objects - AutoCreateBucket bool `json:"autoCreateBucket" default:"false"` // Auto-create S3 bucket - Prefix string `json:"prefix,omitempty"` // S3 bucket prefix for search - IgnoreBody bool `json:"ignoreBody" default:"false"` // Ignore object body - ForcePathStyle bool `json:"forcePathStyle" default:"false"` // Force path style for bucket access - Delay int `json:"delay" default:"500"` // Delay between polls in milliseconds - MaxMessagesPerPoll int `json:"maxMessagesPerPoll" default:"10"` // Max messages to poll per request + Arn string `json:"arn,omitempty" camel:"CAMEL_KAMELET_AWS_S3_SOURCE_BUCKETNAMEORARN"` // S3 ARN + DeleteAfterRead bool `json:"deleteAfterRead" default:"true"` // Auto-delete objects after reading + MoveAfterRead bool `json:"moveAfterRead" default:"false"` // Move objects after reading + DestinationBucket string `json:"destinationBucket,omitempty"` // Destination bucket for moved objects + DestinationBucketPrefix string `json:"destinationBucketPrefix,omitempty"` // Prefix for moved objects + DestinationBucketSuffix string `json:"destinationBucketSuffix,omitempty"` // Suffix for moved objects + AutoCreateBucket bool `json:"autoCreateBucket" default:"false"` // Auto-create S3 bucket + Prefix string `json:"prefix,omitempty"` // S3 bucket prefix for search + IgnoreBody bool `json:"ignoreBody" default:"false"` // Ignore object body + ForcePathStyle bool `json:"forcePathStyle" default:"false"` // Force path style for bucket access + Delay int `json:"delay" default:"500"` // Delay between polls in milliseconds + MaxMessagesPerPoll int `json:"maxMessagesPerPoll" default:"10"` // Max messages to poll per request } type AWSSQS struct { AWSCommon `json:",inline"` // Embeds AWSCommon to inherit its fields in JSON - QueueNameOrArn string `json:"queueNameOrArn,omitempty"` // SQS Queue name or ARN - DeleteAfterRead bool `json:"deleteAfterRead" default:"true"` // Auto-delete messages after reading - AutoCreateQueue bool `json:"autoCreateQueue" default:"false"` // Auto-create SQS queue - AmazonAWSHost string `json:"amazonAWSHost" default:"amazonaws.com"` // AWS host - Protocol string `json:"protocol" default:"https"` // Communication protocol (http/https) - QueueURL string `json:"queueURL,omitempty"` // Full SQS queue URL - Greedy bool `json:"greedy" default:"false"` // Greedy scheduler - Delay int `json:"delay" default:"500"` // Delay between polls in milliseconds - MaxMessagesPerPoll int `json:"maxMessagesPerPoll" default:"1"` // Max messages to return (1-10) - WaitTimeSeconds int `json:"waitTimeSeconds,omitempty"` // Wait time for messages - VisibilityTimeout int `json:"visibilityTimeout,omitempty"` // Visibility timeout in seconds + Arn string `json:"arn,omitempty" camel:"CAMEL_KAMELET_AWS_SQS_SOURCE_QUEUENAMEORARN"` // SQS ARN + DeleteAfterRead bool `json:"deleteAfterRead" default:"true"` // Auto-delete messages after reading + AutoCreateQueue bool `json:"autoCreateQueue" default:"false"` // Auto-create SQS queue + Host string `json:"host" camel:"CAMEL_KAMELET_AWS_SQS_SOURCE_AMAZONAWSHOST" default:"amazonaws.com"` // AWS host + Protocol string `json:"protocol" default:"https"` // Communication protocol (http/https) + QueueURL string `json:"queueURL,omitempty"` // Full SQS queue URL + Greedy bool `json:"greedy" default:"false"` // Greedy scheduler + Delay int `json:"delay" default:"500"` // Delay between polls in milliseconds + MaxMessagesPerPoll int `json:"maxMessagesPerPoll" default:"1"` // Max messages to return (1-10) + WaitTimeSeconds int `json:"waitTimeSeconds,omitempty"` // Wait time for messages + VisibilityTimeout int `json:"visibilityTimeout,omitempty"` // Visibility timeout in seconds } type AWSDDBStreams struct { @@ -116,9 +116,9 @@ type AWSDDBStreams struct { } type Aws struct { - S3 *AWSS3 `json:"s3,omitempty"` // S3 source configuration - SQS *AWSSQS `json:"sqs,omitempty"` // SQS source configuration - DDBStreams *AWSDDBStreams `json:"ddb-streams,omitempty"` // DynamoDB Streams source configuration + S3 *AWSS3 `json:"s3,omitempty"` // S3 source configuration + SQS *AWSSQS `json:"sqs,omitempty"` // SQS source configuration + DDBStreams *AWSDDBStreams `json:"ddbStreams,omitempty"` // DynamoDB Streams source configuration Auth *Auth `json:"auth,omitempty"` } diff --git a/pkg/apis/sources/v1alpha1/integration_types_test.go b/pkg/apis/sources/v1alpha1/integration_types_test.go index 83b2ff0ffc5..7a0d9f35479 100644 --- a/pkg/apis/sources/v1alpha1/integration_types_test.go +++ b/pkg/apis/sources/v1alpha1/integration_types_test.go @@ -57,7 +57,7 @@ func TestAWS(t *testing.T) { AWSCommon: AWSCommon{ Region: "eu-north-1", }, - BucketNameOrArn: "example-bucket", + Arn: "example-bucket", } if s3.Region != "eu-north-1" { @@ -68,7 +68,7 @@ func TestAWS(t *testing.T) { AWSCommon: AWSCommon{ Region: "eu-north-1", }, - QueueNameOrArn: "example-queue", + Arn: "example-queue", } if sqs.Region != "eu-north-1" { diff --git a/pkg/apis/sources/v1alpha1/integration_validation.go b/pkg/apis/sources/v1alpha1/integration_validation.go index 4df223eb5dd..3b412998460 100644 --- a/pkg/apis/sources/v1alpha1/integration_validation.go +++ b/pkg/apis/sources/v1alpha1/integration_validation.go @@ -65,8 +65,8 @@ func (spec *IntegrationSourceSpec) Validate(ctx context.Context) *apis.FieldErro // Additional validation for AWS S3 required fields if spec.Aws.S3 != nil { - if spec.Aws.S3.BucketNameOrArn == "" { - errs = errs.Also(apis.ErrMissingField("aws.s3.bucketNameOrArn")) + if spec.Aws.S3.Arn == "" { + errs = errs.Also(apis.ErrMissingField("aws.s3.arn")) } if spec.Aws.S3.Region == "" { errs = errs.Also(apis.ErrMissingField("aws.s3.region")) @@ -75,7 +75,7 @@ func (spec *IntegrationSourceSpec) Validate(ctx context.Context) *apis.FieldErro // Additional validation for AWS SQS required fields if spec.Aws.SQS != nil { - if spec.Aws.SQS.QueueNameOrArn == "" { + if spec.Aws.SQS.Arn == "" { errs = errs.Also(apis.ErrMissingField("aws.sqs.queueNameOrArn")) } if spec.Aws.SQS.Region == "" { diff --git a/pkg/apis/sources/v1alpha1/integration_validation_test.go b/pkg/apis/sources/v1alpha1/integration_validation_test.go index 6e0071703d1..c2a932af3d2 100644 --- a/pkg/apis/sources/v1alpha1/integration_validation_test.go +++ b/pkg/apis/sources/v1alpha1/integration_validation_test.go @@ -49,7 +49,7 @@ func TestIntegrationSourceSpecValidation(t *testing.T) { AWSCommon: AWSCommon{ Region: "us-east-1", }, - BucketNameOrArn: "example-bucket", + Arn: "example-bucket", }, Auth: &Auth{ Secret: &Secret{ @@ -70,7 +70,7 @@ func TestIntegrationSourceSpecValidation(t *testing.T) { AWSCommon: AWSCommon{ Region: "us-east-1", }, - QueueNameOrArn: "example-queue", + Arn: "example-queue", }, Auth: &Auth{ Secret: &Secret{ @@ -117,7 +117,7 @@ func TestIntegrationSourceSpecValidation(t *testing.T) { AWSCommon: AWSCommon{ Region: "us-east-1", }, - BucketNameOrArn: "example-bucket", + Arn: "example-bucket", }, }, }, @@ -131,13 +131,13 @@ func TestIntegrationSourceSpecValidation(t *testing.T) { AWSCommon: AWSCommon{ Region: "us-east-1", }, - BucketNameOrArn: "example-bucket", + Arn: "example-bucket", }, SQS: &AWSSQS{ AWSCommon: AWSCommon{ Region: "us-east-1", }, - QueueNameOrArn: "example-queue", + Arn: "example-queue", }, Auth: &Auth{ Secret: &Secret{ @@ -203,7 +203,7 @@ func TestIntegrationSourceSpecValidation(t *testing.T) { AWSCommon: AWSCommon{ Region: "us-east-1", }, - BucketNameOrArn: "example-bucket", + Arn: "example-bucket", }, }, }, @@ -214,7 +214,7 @@ func TestIntegrationSourceSpecValidation(t *testing.T) { spec: IntegrationSourceSpec{ Aws: &Aws{ S3: &AWSS3{ - BucketNameOrArn: "example-bucket", + Arn: "example-bucket", }, Auth: &Auth{ Secret: &Secret{ diff --git a/pkg/reconciler/integrationsource/resources/containersource.go b/pkg/reconciler/integrationsource/resources/containersource.go index 95fb2a32aa1..8e55bcee889 100644 --- a/pkg/reconciler/integrationsource/resources/containersource.go +++ b/pkg/reconciler/integrationsource/resources/containersource.go @@ -89,17 +89,18 @@ func generateEnvVarsFromStruct(prefix string, s interface{}) []corev1.EnvVar { continue } - // Extract the JSON tag or fall back to the Go field name - jsonTag := fieldType.Tag.Get("json") - tagName := strings.Split(jsonTag, ",")[0] - - // fallback to Go field name if no JSON tag - if tagName == "" || tagName == "-" { - tagName = fieldType.Name + // First, check for the custom 'camel' tag + envVarName := fieldType.Tag.Get("camel") + if envVarName == "" { + // If 'camel' tag is not present, fall back to the 'json' tag or Go field name + jsonTag := fieldType.Tag.Get("json") + tagName := strings.Split(jsonTag, ",")[0] + if tagName == "" || tagName == "-" { + tagName = fieldType.Name + } + envVarName = fmt.Sprintf("%s_%s", prefix, strings.ToUpper(tagName)) } - envVarName := fmt.Sprintf("%s_%s", prefix, strings.ToUpper(tagName)) - if field.Kind() == reflect.Ptr { if field.IsNil() { continue diff --git a/pkg/reconciler/integrationsource/resources/containersource_test.go b/pkg/reconciler/integrationsource/resources/containersource_test.go index 669b4eeaabc..b512fdd784a 100644 --- a/pkg/reconciler/integrationsource/resources/containersource_test.go +++ b/pkg/reconciler/integrationsource/resources/containersource_test.go @@ -129,3 +129,28 @@ func TestGenerateEnvVarsFromStruct(t *testing.T) { t.Errorf("generateEnvVarsFromStruct() mismatch (-want +got):\n%s", diff) } } + +func TestGenerateEnvVarsFromStruct_S3WithCamelTag(t *testing.T) { + type AWSS3 struct { + Arn string `json:"arn,omitempty" camel:"CAMEL_KAMELET_AWS_S3_SOURCE_BUCKETNAMEORARN"` + Region string `json:"region,omitempty"` + } + + prefix := "CAMEL_KAMELET_AWS_S3_SOURCE" + input := AWSS3{ + Arn: "arn:aws:s3:::example-bucket", + Region: "us-west-2", + } + + // Expected environment variables including SSL settings and camel tag for Arn + want := []corev1.EnvVar{ + {Name: "CAMEL_KAMELET_AWS_S3_SOURCE_BUCKETNAMEORARN", Value: "arn:aws:s3:::example-bucket"}, + {Name: "CAMEL_KAMELET_AWS_S3_SOURCE_REGION", Value: "us-west-2"}, + } + + got := generateEnvVarsFromStruct(prefix, input) + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("generateEnvVarsFromStruct_S3WithCamelTag() mismatch (-want +got):\n%s", diff) + } +}