From 6f892460147675cf686ed330aeaf6aaa66effb96 Mon Sep 17 00:00:00 2001 From: Rich Unger Date: Thu, 21 Nov 2024 17:25:53 -0800 Subject: [PATCH 1/3] Add tag support for sqs and dynamo --- src/main/java/com/airbnb/billow/AWSDatabase.java | 16 ++++++++++++++-- src/main/java/com/airbnb/billow/DynamoTable.java | 13 ++++++++++++- src/main/java/com/airbnb/billow/SQSQueue.java | 8 +++++++- 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/airbnb/billow/AWSDatabase.java b/src/main/java/com/airbnb/billow/AWSDatabase.java index af40732..f0e1424 100644 --- a/src/main/java/com/airbnb/billow/AWSDatabase.java +++ b/src/main/java/com/airbnb/billow/AWSDatabase.java @@ -253,11 +253,16 @@ public class AWSDatabase { String lastModifiedTimestamp = map.get(SQSQueue.ATTR_LAST_MODIFIED_TIMESTAMP); String queueArn = map.get(SQSQueue.ATTR_QUEUE_ARN); + com.amazonaws.services.sqs.model.ListQueueTagsRequest tagsRequest = + new com.amazonaws.services.sqs.model.ListQueueTagsRequest().withQueueUrl(url); + com.amazonaws.services.sqs.model.ListQueueTagsResult tagsResult = + client.listQueueTags(tagsRequest); + SQSQueue queue = new SQSQueue(url, Long.valueOf(approximateNumberOfMessagesDelayed), Long.valueOf(receiveMessageWaitTimeSeconds), Long.valueOf(createdTimestamp), Long.valueOf(delaySeconds), Long.valueOf(messageRetentionPeriod), Long.valueOf(maximumMessageSize), Long.valueOf(visibilityTimeout), Long.valueOf(approximateNumberOfMessages), - Long.valueOf(lastModifiedTimestamp), queueArn); + Long.valueOf(lastModifiedTimestamp), queueArn, tagsResult.getTags()); sqsQueueBuilder.putAll(regionName, queue); cnt++; @@ -282,7 +287,14 @@ public class AWSDatabase { int cnt = 0; while (iterator.hasNext()) { Table table = iterator.next(); - dynamoTableBuilder.putAll(regionName, new DynamoTable(table)); + + com.amazonaws.services.dynamodbv2.model.ListTagsOfResourceRequest tagsRequest = + new com.amazonaws.services.dynamodbv2.model.ListTagsOfResourceRequest().withResourceArn( + table.getDescription().getTableArn()); + com.amazonaws.services.dynamodbv2.model.ListTagsOfResourceResult tagsResult = + client.listTagsOfResource(tagsRequest); + + dynamoTableBuilder.putAll(regionName, new DynamoTable(table, tagsResult.getTags())); cnt++; } diff --git a/src/main/java/com/airbnb/billow/DynamoTable.java b/src/main/java/com/airbnb/billow/DynamoTable.java index a288103..69f56ff 100644 --- a/src/main/java/com/airbnb/billow/DynamoTable.java +++ b/src/main/java/com/airbnb/billow/DynamoTable.java @@ -1,8 +1,12 @@ package com.airbnb.billow; import com.amazonaws.services.dynamodbv2.model.GlobalSecondaryIndexDescription; +import com.amazonaws.services.dynamodbv2.model.Tag; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; + import lombok.Getter; import com.amazonaws.services.dynamodbv2.document.Table; @@ -39,8 +43,10 @@ public class DynamoTable { private final String provisionedThroughput; @Getter private final List globalSecondaryIndexes; + @Getter + private final Map tags; - public DynamoTable(Table table) { + public DynamoTable(Table table, List tags) { table.describe(); tableName = table.getTableName(); attributeDefinitions = table.getDescription().getAttributeDefinitions().toString(); @@ -56,6 +62,11 @@ public DynamoTable(Table table) { provisionedThroughput = table.getDescription().getProvisionedThroughput().toString(); globalSecondaryIndexes = new ArrayList<>(); + this.tags = new HashMap<>(tags.size()); + for(Tag tag : tags) { + this.tags.put(tag.getKey(), tag.getValue()); + } + if (table.getDescription().getGlobalSecondaryIndexes() != null) { for (GlobalSecondaryIndexDescription gsiDesc : table .getDescription() diff --git a/src/main/java/com/airbnb/billow/SQSQueue.java b/src/main/java/com/airbnb/billow/SQSQueue.java index d42433e..ef9bfef 100644 --- a/src/main/java/com/airbnb/billow/SQSQueue.java +++ b/src/main/java/com/airbnb/billow/SQSQueue.java @@ -4,6 +4,8 @@ import com.fasterxml.jackson.annotation.JsonFilter; +import java.util.Map; + @JsonFilter(SQSQueue.QUEUE_FILTER) public class SQSQueue { public static final String QUEUE_FILTER = "QueueFilter"; @@ -41,6 +43,8 @@ public class SQSQueue { private final Long lastModifiedTimestamp; @Getter private final String queueArn; + @Getter + private final Map tags; @@ -54,7 +58,8 @@ public SQSQueue(String url, Long visibilityTimeout, Long approximateNumberOfMessages, Long lastModifiedTimestamp, - String queueArn) { + String queueArn, + Map tags) { this.url = url; this.approximateNumberOfMessagesDelayed = approximateNumberOfMessagesDelayed; this.receiveMessageWaitTimeSeconds = receiveMessageWaitTimeSeconds; @@ -66,5 +71,6 @@ public SQSQueue(String url, this.approximateNumberOfMessages = approximateNumberOfMessages; this.lastModifiedTimestamp = lastModifiedTimestamp; this.queueArn = queueArn; + this.tags = tags; } } From cae7956af44faeed4f5647689adff2947868f154 Mon Sep 17 00:00:00 2001 From: Rich Unger Date: Wed, 11 Dec 2024 14:34:20 -0800 Subject: [PATCH 2/3] bump version, bump java version --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 6a702d0..d770938 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.airbnb.billow billow - 2.26 + 2.27 @@ -207,8 +207,8 @@ maven-compiler-plugin ${maven-compiler-plugin.version} - 1.7 - 1.7 + 1.8 + 1.8 From 48b6da08953a1a950f2159d6802894b303e75c0d Mon Sep 17 00:00:00 2001 From: Rich Unger Date: Wed, 18 Dec 2024 09:59:44 -0800 Subject: [PATCH 3/3] don't use getDescription() before describe() is called --- src/main/java/com/airbnb/billow/AWSDatabase.java | 9 +-------- src/main/java/com/airbnb/billow/DynamoTable.java | 13 ++++++++++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/airbnb/billow/AWSDatabase.java b/src/main/java/com/airbnb/billow/AWSDatabase.java index f0e1424..88b18a8 100644 --- a/src/main/java/com/airbnb/billow/AWSDatabase.java +++ b/src/main/java/com/airbnb/billow/AWSDatabase.java @@ -287,14 +287,7 @@ public class AWSDatabase { int cnt = 0; while (iterator.hasNext()) { Table table = iterator.next(); - - com.amazonaws.services.dynamodbv2.model.ListTagsOfResourceRequest tagsRequest = - new com.amazonaws.services.dynamodbv2.model.ListTagsOfResourceRequest().withResourceArn( - table.getDescription().getTableArn()); - com.amazonaws.services.dynamodbv2.model.ListTagsOfResourceResult tagsResult = - client.listTagsOfResource(tagsRequest); - - dynamoTableBuilder.putAll(regionName, new DynamoTable(table, tagsResult.getTags())); + dynamoTableBuilder.putAll(regionName, new DynamoTable(table, client)); cnt++; } diff --git a/src/main/java/com/airbnb/billow/DynamoTable.java b/src/main/java/com/airbnb/billow/DynamoTable.java index 69f56ff..f1d3dd2 100644 --- a/src/main/java/com/airbnb/billow/DynamoTable.java +++ b/src/main/java/com/airbnb/billow/DynamoTable.java @@ -9,6 +9,7 @@ import lombok.Getter; +import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; import com.amazonaws.services.dynamodbv2.document.Table; import com.fasterxml.jackson.annotation.JsonFilter; import org.joda.time.DateTime; @@ -46,7 +47,7 @@ public class DynamoTable { @Getter private final Map tags; - public DynamoTable(Table table, List tags) { + public DynamoTable(Table table, AmazonDynamoDBClient client) { table.describe(); tableName = table.getTableName(); attributeDefinitions = table.getDescription().getAttributeDefinitions().toString(); @@ -62,8 +63,14 @@ public DynamoTable(Table table, List tags) { provisionedThroughput = table.getDescription().getProvisionedThroughput().toString(); globalSecondaryIndexes = new ArrayList<>(); - this.tags = new HashMap<>(tags.size()); - for(Tag tag : tags) { + com.amazonaws.services.dynamodbv2.model.ListTagsOfResourceRequest tagsRequest = + new com.amazonaws.services.dynamodbv2.model.ListTagsOfResourceRequest().withResourceArn( + table.getDescription().getTableArn()); + com.amazonaws.services.dynamodbv2.model.ListTagsOfResourceResult tagsResult = + client.listTagsOfResource(tagsRequest); + + this.tags = new HashMap<>(tagsResult.getTags().size()); + for(Tag tag : tagsResult.getTags()) { this.tags.put(tag.getKey(), tag.getValue()); }