Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add tag support for sqs and dynamo #67

Merged
merged 4 commits into from
Dec 18, 2024
Merged

Add tag support for sqs and dynamo #67

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6f89246
Add tag support for sqs and dynamo
Nov 22, 2024
cae7956
bump version, bump java version
Dec 11, 2024
315e151
Merge branch 'master' into tagsForSqsAndDynamo
Dec 17, 2024
48b6da0
don't use getDescription() before describe() is called
Dec 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

<groupId>com.airbnb.billow</groupId>
<artifactId>billow</artifactId>
<version>2.26</version>
<version>2.27</version>

<licenses>
<license>
Expand Down Expand Up @@ -207,8 +207,8 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>${maven-compiler-plugin.version}</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
</plugins>
Expand Down
9 changes: 7 additions & 2 deletions src/main/java/com/airbnb/billow/AWSDatabase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,11 +253,16 @@ public class AWSDatabase {
String lastModifiedTimestamp = map.get(SQSQueue.ATTR_LAST_MODIFIED_TIMESTAMP);
String queueArn = map.get(SQSQueue.ATTR_QUEUE_ARN);

com.amazonaws.services.sqs.model.ListQueueTagsRequest tagsRequest =
new com.amazonaws.services.sqs.model.ListQueueTagsRequest().withQueueUrl(url);
com.amazonaws.services.sqs.model.ListQueueTagsResult tagsResult =
client.listQueueTags(tagsRequest);

Comment on lines +256 to +260
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What happens if these requests get rate limited/error out? Would they retry/skip continue..

Copy link
Contributor Author

@richunger richunger Nov 22, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's a great question. I don't see how that's handled anywhere in billow at all, not just in tag requests elsewhere, but in requesting the resources themselves.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would suppose we'd see the failure count tick up (we should have an alert on that), and then set the refreshRate in reference.conf from 5m to something higher.

SQSQueue queue = new SQSQueue(url, Long.valueOf(approximateNumberOfMessagesDelayed),
Long.valueOf(receiveMessageWaitTimeSeconds), Long.valueOf(createdTimestamp),
Long.valueOf(delaySeconds), Long.valueOf(messageRetentionPeriod), Long.valueOf(maximumMessageSize),
Long.valueOf(visibilityTimeout), Long.valueOf(approximateNumberOfMessages),
Long.valueOf(lastModifiedTimestamp), queueArn);
Long.valueOf(lastModifiedTimestamp), queueArn, tagsResult.getTags());

sqsQueueBuilder.putAll(regionName, queue);
cnt++;
Expand All @@ -282,7 +287,7 @@ public class AWSDatabase {
int cnt = 0;
while (iterator.hasNext()) {
Table table = iterator.next();
dynamoTableBuilder.putAll(regionName, new DynamoTable(table));
dynamoTableBuilder.putAll(regionName, new DynamoTable(table, client));
cnt++;
}

Expand Down
20 changes: 19 additions & 1 deletion src/main/java/com/airbnb/billow/DynamoTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,15 @@
package com.airbnb.billow;

import com.amazonaws.services.dynamodbv2.model.GlobalSecondaryIndexDescription;
import com.amazonaws.services.dynamodbv2.model.Tag;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import lombok.Getter;

import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient;
import com.amazonaws.services.dynamodbv2.document.Table;
import com.fasterxml.jackson.annotation.JsonFilter;
import org.joda.time.DateTime;
Expand Down Expand Up @@ -39,8 +44,10 @@ public class DynamoTable {
private final String provisionedThroughput;
@Getter
private final List<DynamoGSI> globalSecondaryIndexes;
@Getter
private final Map<String, String> tags;

public DynamoTable(Table table) {
public DynamoTable(Table table, AmazonDynamoDBClient client) {
table.describe();
tableName = table.getTableName();
attributeDefinitions = table.getDescription().getAttributeDefinitions().toString();
Expand All @@ -56,6 +63,17 @@ public DynamoTable(Table table) {
provisionedThroughput = table.getDescription().getProvisionedThroughput().toString();
globalSecondaryIndexes = new ArrayList<>();

com.amazonaws.services.dynamodbv2.model.ListTagsOfResourceRequest tagsRequest =
new com.amazonaws.services.dynamodbv2.model.ListTagsOfResourceRequest().withResourceArn(
table.getDescription().getTableArn());
com.amazonaws.services.dynamodbv2.model.ListTagsOfResourceResult tagsResult =
client.listTagsOfResource(tagsRequest);

this.tags = new HashMap<>(tagsResult.getTags().size());
for(Tag tag : tagsResult.getTags()) {
this.tags.put(tag.getKey(), tag.getValue());
}

if (table.getDescription().getGlobalSecondaryIndexes() != null) {
for (GlobalSecondaryIndexDescription gsiDesc : table
.getDescription()
Expand Down
8 changes: 7 additions & 1 deletion src/main/java/com/airbnb/billow/SQSQueue.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@

import com.fasterxml.jackson.annotation.JsonFilter;

import java.util.Map;

@JsonFilter(SQSQueue.QUEUE_FILTER)
public class SQSQueue {
public static final String QUEUE_FILTER = "QueueFilter";
Expand Down Expand Up @@ -41,6 +43,8 @@ public class SQSQueue {
private final Long lastModifiedTimestamp;
@Getter
private final String queueArn;
@Getter
private final Map<String, String> tags;



Expand All @@ -54,7 +58,8 @@ public SQSQueue(String url,
Long visibilityTimeout,
Long approximateNumberOfMessages,
Long lastModifiedTimestamp,
String queueArn) {
String queueArn,
Map<String, String> tags) {
this.url = url;
this.approximateNumberOfMessagesDelayed = approximateNumberOfMessagesDelayed;
this.receiveMessageWaitTimeSeconds = receiveMessageWaitTimeSeconds;
Expand All @@ -66,5 +71,6 @@ public SQSQueue(String url,
this.approximateNumberOfMessages = approximateNumberOfMessages;
this.lastModifiedTimestamp = lastModifiedTimestamp;
this.queueArn = queueArn;
this.tags = tags;
}
}