From 4c7c2681eed9c407400df1e7f11de3944a0a5a05 Mon Sep 17 00:00:00 2001 From: Ben Muschol Date: Tue, 23 Mar 2021 11:07:03 -0400 Subject: [PATCH 1/6] Some initial refactoring --- .../java/com/airbnb/billow/AWSDatabase.java | 166 ++++++++++-------- 1 file changed, 89 insertions(+), 77 deletions(-) diff --git a/src/main/java/com/airbnb/billow/AWSDatabase.java b/src/main/java/com/airbnb/billow/AWSDatabase.java index 5d16a60..685917d 100644 --- a/src/main/java/com/airbnb/billow/AWSDatabase.java +++ b/src/main/java/com/airbnb/billow/AWSDatabase.java @@ -77,6 +77,7 @@ public class AWSDatabase { private String awsAccountNumber; private String awsARNPartition; + AWSDatabase(final Map ec2Clients, final Map rdsClients, final Map dynamoClients, @@ -89,15 +90,6 @@ public class AWSDatabase { timestamp = System.currentTimeMillis(); log.info("Building AWS DB with timestamp {}", timestamp); - log.info("Getting EC2 instances"); - final ImmutableMultimap.Builder ec2InstanceBuilder = new ImmutableMultimap.Builder<>(); - final ImmutableMultimap.Builder dynamoTableBuilder = new ImmutableMultimap.Builder<>(); - final ImmutableMultimap.Builder sqsQueueBuilder = new ImmutableMultimap.Builder<>(); - final ImmutableMultimap.Builder elasticacheClusterBuilder = - new ImmutableMultimap.Builder<>(); - final ImmutableMultimap.Builder elasticsearchClusterBuilder = - new ImmutableMultimap.Builder<>(); - if (configAWSAccountNumber == null) { awsAccountNumber = ""; } else { @@ -112,11 +104,43 @@ public class AWSDatabase { awsARNPartition = configAWSARNPartition; } - /* - * IAM keys - * Put this in the beginning to populate the awsAccountNumber. - */ + this.iamUsers = loadIAM(iamClient); + this.elasticacheClusters = loadElasticache(elasticacheClients); + this.elasticsearchClusters = loadElasticsearch(elasticsearchClients); + this.sqsQueues = loadSQS(sqsClients); + this.dynamoTables = loadDynamo(dynamoClients); + this.ec2Instances = loadEC2Instances(ec2Clients); + this.ec2SGs = loadEC2SGs(ec2Clients); + this.rdsInstances = loadRDS(rdsClients); + + log.info("Done building AWS DB"); + } + private AWSDatabase(ImmutableMultimap ec2Instances, + ImmutableMultimap dynamoTables, + ImmutableMultimap rdsInstances, + ImmutableMultimap ec2SGs, + ImmutableMultimap sqsQueues, + ImmutableMultimap elasticacheClusters, + ImmutableMultimap elasticsearchClusters, + ImmutableList iamUsers, + long timestamp, + String awsAccountNumber, + String awsARNPartition) { + this.ec2Instances = ec2Instances; + this.dynamoTables = dynamoTables; + this.rdsInstances = rdsInstances; + this.ec2SGs = ec2SGs; + this.sqsQueues = sqsQueues; + this.elasticacheClusters = elasticacheClusters; + this.elasticsearchClusters = elasticsearchClusters; + this.iamUsers = iamUsers; + this.timestamp = timestamp; + this.awsAccountNumber = awsAccountNumber; + this.awsARNPartition = awsARNPartition; + } + + private ImmutableList loadIAM(final AmazonIdentityManagement iamClient) { log.info("Getting IAM keys"); final ImmutableList.Builder usersBuilder = new ImmutableList.Builder<>(); @@ -141,13 +165,11 @@ public class AWSDatabase { } listUsersRequest.setMarker(listUsersResult.getMarker()); } while (listUsersResult.isTruncated()); - this.iamUsers = usersBuilder.build(); - - - /* - * ElasticCache - */ + return usersBuilder.build(); + } + private ImmutableMultimap loadElasticache(final Map elasticacheClients) { + ImmutableMultimap.Builder elasticacheClusterBuilder = ImmutableMultimap.builder(); for (Map.Entry clientPair : elasticacheClients.entrySet()) { final String regionName = clientPair.getKey(); final AmazonElastiCacheClient client = clientPair.getValue(); @@ -181,11 +203,11 @@ public class AWSDatabase { for (CacheCluster cluster : describeCacheClustersResult.getCacheClusters()) { com.amazonaws.services.elasticache.model.ListTagsForResourceRequest tagsRequest = - new com.amazonaws.services.elasticache.model.ListTagsForResourceRequest() - .withResourceName(elasticacheARN(awsARNPartition, regionName, awsAccountNumber, cluster)); + new com.amazonaws.services.elasticache.model.ListTagsForResourceRequest() + .withResourceName(elasticacheARN(awsARNPartition, regionName, awsAccountNumber, cluster)); com.amazonaws.services.elasticache.model.ListTagsForResourceResult tagsResult = - client.listTagsForResource(tagsRequest); + client.listTagsForResource(tagsRequest); elasticacheClusterBuilder.putAll(regionName, new ElasticacheCluster(cluster, clusterIdToNodeGroupMember.get(cluster.getCacheClusterId()), tagsResult.getTagList())); cntClusters++; } @@ -194,15 +216,14 @@ public class AWSDatabase { describeCacheClustersRequest.setMarker(describeCacheClustersResult.getMarker()); } while (describeCacheClustersResult.getMarker() != null); - - } - this.elasticacheClusters = elasticacheClusterBuilder.build(); - /* - * Elasticsearch - */ + return elasticacheClusterBuilder.build(); + } + private ImmutableMultimap loadElasticsearch(final Map elasticsearchClients) { + final ImmutableMultimap.Builder elasticsearchClusterBuilder = + new ImmutableMultimap.Builder<>(); for (Map.Entry clientPair : elasticsearchClients.entrySet()) { final String regionName = clientPair.getKey(); final AWSElasticsearchClient client = clientPair.getValue(); @@ -224,12 +245,11 @@ public class AWSDatabase { log.debug("Found {} Elasticsearch domains in {}", domainInfoList.size(), regionName); } - this.elasticsearchClusters = elasticsearchClusterBuilder.build(); - - /* - * SQS Queues - */ + return elasticsearchClusterBuilder.build(); + } + private ImmutableMultimap loadSQS(final Map sqsClients) { + final ImmutableMultimap.Builder sqsQueueBuilder = new ImmutableMultimap.Builder<>(); for (Map.Entry clientPair : sqsClients.entrySet()) { final String regionName = clientPair.getKey(); final AmazonSQSClient client = clientPair.getValue(); @@ -254,10 +274,10 @@ public class AWSDatabase { String queueArn = map.get(SQSQueue.ATTR_QUEUE_ARN); SQSQueue queue = new SQSQueue(url, Long.valueOf(approximateNumberOfMessagesDelayed), - Long.valueOf(receiveMessageWaitTimeSeconds), Long.valueOf(createdTimestamp), - Long.valueOf(delaySeconds), Long.valueOf(messageRetentionPeriod), Long.valueOf(maximumMessageSize), - Long.valueOf(visibilityTimeout), Long.valueOf(approximateNumberOfMessages), - Long.valueOf(lastModifiedTimestamp), queueArn); + Long.valueOf(receiveMessageWaitTimeSeconds), Long.valueOf(createdTimestamp), + Long.valueOf(delaySeconds), Long.valueOf(messageRetentionPeriod), Long.valueOf(maximumMessageSize), + Long.valueOf(visibilityTimeout), Long.valueOf(approximateNumberOfMessages), + Long.valueOf(lastModifiedTimestamp), queueArn); sqsQueueBuilder.putAll(regionName, queue); cnt++; @@ -265,12 +285,11 @@ public class AWSDatabase { log.debug("Found {} queues in {}", cnt, regionName); } - this.sqsQueues = sqsQueueBuilder.build(); - - /* - * DynamoDB Tables - */ + return sqsQueueBuilder.build(); + } + private ImmutableMultimap loadDynamo(final Map dynamoClients) { + final ImmutableMultimap.Builder dynamoTableBuilder = new ImmutableMultimap.Builder<>(); for (Map.Entry clientPair : dynamoClients.entrySet()) { final String regionName = clientPair.getKey(); final AmazonDynamoDBClient client = clientPair.getValue(); @@ -288,12 +307,12 @@ public class AWSDatabase { log.debug("Found {} dynamodbs in {}", cnt, regionName); } - this.dynamoTables = dynamoTableBuilder.build(); - - /* - * EC2 Instances - */ + return dynamoTableBuilder.build(); + } + private ImmutableMultimap loadEC2Instances(final Map ec2Clients) { + final ImmutableMultimap.Builder ec2InstanceBuilder = new ImmutableMultimap.Builder<>(); + log.info("Getting EC2 instances"); for (Map.Entry clientPair : ec2Clients.entrySet()) { final String regionName = clientPair.getKey(); final AmazonEC2Client client = clientPair.getValue(); @@ -307,12 +326,10 @@ public class AWSDatabase { ec2InstanceBuilder.putAll(regionName, new EC2Instance(instance)); } } - this.ec2Instances = ec2InstanceBuilder.build(); - - /* - * EC2 security groups - */ + return ec2InstanceBuilder.build(); + } + private ImmutableMultimap loadEC2SGs(final Map ec2Clients) { log.info("Getting EC2 security groups"); final ImmutableMultimap.Builder ec2SGbuilder = new ImmutableMultimap.Builder(); for (Map.Entry clientPair : ec2Clients.entrySet()) { @@ -322,13 +339,10 @@ public class AWSDatabase { log.debug("Found {} security groups in {}", securityGroups.size(), regionName); ec2SGbuilder.putAll(regionName, securityGroups); } - this.ec2SGs = ec2SGbuilder.build(); - - - /* - * RDS Instances - */ + return ec2SGbuilder.build(); + } + private ImmutableMultimap loadRDS(final Map rdsClients) { log.info("Getting RDS instances and clusters"); final ImmutableMultimap.Builder rdsBuilder = new ImmutableMultimap.Builder(); @@ -374,32 +388,30 @@ public class AWSDatabase { List snapshots = new ArrayList<>(); // Get snapshot for masters only. if (RDSInstance.checkIfMaster(instance, instanceIdToCluster.get(instance.getDBInstanceIdentifier()))) { - if ("aurora".equals(instance.getEngine()) || "aurora-mysql".equals(instance.getEngine())) { - DescribeDBClusterSnapshotsRequest snapshotsRequest = new DescribeDBClusterSnapshotsRequest() - .withDBClusterIdentifier(instance.getDBClusterIdentifier()); - DescribeDBClusterSnapshotsResult snapshotsResult = client.describeDBClusterSnapshots(snapshotsRequest); - for (DBClusterSnapshot s : snapshotsResult.getDBClusterSnapshots()) { - snapshots.add(s.getDBClusterSnapshotIdentifier()); - } - } else { - DescribeDBSnapshotsRequest snapshotsRequest = new DescribeDBSnapshotsRequest() - .withDBInstanceIdentifier(instance.getDBInstanceIdentifier()); - DescribeDBSnapshotsResult snapshotsResult = client.describeDBSnapshots(snapshotsRequest); - for (DBSnapshot s : snapshotsResult.getDBSnapshots()) { - snapshots.add(s.getDBSnapshotIdentifier()); - } - } + if ("aurora".equals(instance.getEngine()) || "aurora-mysql".equals(instance.getEngine())) { + DescribeDBClusterSnapshotsRequest snapshotsRequest = new DescribeDBClusterSnapshotsRequest() + .withDBClusterIdentifier(instance.getDBClusterIdentifier()); + DescribeDBClusterSnapshotsResult snapshotsResult = client.describeDBClusterSnapshots(snapshotsRequest); + for (DBClusterSnapshot s : snapshotsResult.getDBClusterSnapshots()) { + snapshots.add(s.getDBClusterSnapshotIdentifier()); + } + } else { + DescribeDBSnapshotsRequest snapshotsRequest = new DescribeDBSnapshotsRequest() + .withDBInstanceIdentifier(instance.getDBInstanceIdentifier()); + DescribeDBSnapshotsResult snapshotsResult = client.describeDBSnapshots(snapshotsRequest); + for (DBSnapshot s : snapshotsResult.getDBSnapshots()) { + snapshots.add(s.getDBSnapshotIdentifier()); + } + } } rdsBuilder.putAll(regionName, new RDSInstance(instance, - instanceIdToCluster.get(instance.getDBInstanceIdentifier()), tagsResult.getTagList(), snapshots)); + instanceIdToCluster.get(instance.getDBInstanceIdentifier()), tagsResult.getTagList(), snapshots)); } rdsRequest.setMarker(result.getMarker()); } while (result.getMarker() != null); } - this.rdsInstances = rdsBuilder.build(); - - log.info("Done building AWS DB"); + return rdsBuilder.build(); } public long getAgeInMs() { From 76a3ce8ff4c82b5c77c67c51f2ac0cbad4d4ada2 Mon Sep 17 00:00:00 2001 From: Ben Muschol Date: Tue, 23 Mar 2021 13:53:56 -0400 Subject: [PATCH 2/6] WIP: Incremental jobs for more frequent updates --- pom.xml | 4 +- .../java/com/airbnb/billow/AWSDatabase.java | 174 ++++++++++++------ .../com/airbnb/billow/AWSDatabaseHolder.java | 78 +++++++- src/main/java/com/airbnb/billow/Main.java | 42 +++-- .../com/airbnb/billow/TimestampedData.java | 18 ++ .../BaseAWSDatabaseHolderRefreshJob.java} | 8 +- .../airbnb/billow/jobs/DynamoRefreshJob.java | 12 ++ .../billow/jobs/Ec2InstanceRefreshJob.java | 12 ++ .../airbnb/billow/jobs/Ec2SGRefreshJob.java | 12 ++ .../billow/jobs/ElasticacheRefreshJob.java | 12 ++ .../billow/jobs/ElasticsearchRefreshJob.java | 12 ++ .../com/airbnb/billow/jobs/IamRefreshJob.java | 12 ++ .../com/airbnb/billow/jobs/RdsRefreshJob.java | 12 ++ .../com/airbnb/billow/jobs/SqsRefreshJob.java | 12 ++ 14 files changed, 338 insertions(+), 82 deletions(-) create mode 100644 src/main/java/com/airbnb/billow/TimestampedData.java rename src/main/java/com/airbnb/billow/{AWSDatabaseHolderRefreshJob.java => jobs/BaseAWSDatabaseHolderRefreshJob.java} (85%) create mode 100644 src/main/java/com/airbnb/billow/jobs/DynamoRefreshJob.java create mode 100644 src/main/java/com/airbnb/billow/jobs/Ec2InstanceRefreshJob.java create mode 100644 src/main/java/com/airbnb/billow/jobs/Ec2SGRefreshJob.java create mode 100644 src/main/java/com/airbnb/billow/jobs/ElasticacheRefreshJob.java create mode 100644 src/main/java/com/airbnb/billow/jobs/ElasticsearchRefreshJob.java create mode 100644 src/main/java/com/airbnb/billow/jobs/IamRefreshJob.java create mode 100644 src/main/java/com/airbnb/billow/jobs/RdsRefreshJob.java create mode 100644 src/main/java/com/airbnb/billow/jobs/SqsRefreshJob.java diff --git a/pom.xml b/pom.xml index 52b2f18..1d3be4f 100644 --- a/pom.xml +++ b/pom.xml @@ -207,8 +207,8 @@ maven-compiler-plugin ${maven-compiler-plugin.version} - 1.7 - 1.7 + 8 + 8 diff --git a/src/main/java/com/airbnb/billow/AWSDatabase.java b/src/main/java/com/airbnb/billow/AWSDatabase.java index 685917d..9f53a82 100644 --- a/src/main/java/com/airbnb/billow/AWSDatabase.java +++ b/src/main/java/com/airbnb/billow/AWSDatabase.java @@ -7,14 +7,12 @@ import com.amazonaws.services.elasticache.model.ReplicationGroup; import com.amazonaws.services.elasticsearch.model.DescribeElasticsearchDomainRequest; import com.amazonaws.services.elasticsearch.model.DescribeElasticsearchDomainResult; -import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; +import lombok.ToString; import lombok.extern.slf4j.Slf4j; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; import com.amazonaws.services.dynamodbv2.document.DynamoDB; @@ -63,17 +61,18 @@ import com.google.common.collect.ImmutableMultimap; @Slf4j -@Data +@ToString +@EqualsAndHashCode +@RequiredArgsConstructor public class AWSDatabase { - private final ImmutableMultimap ec2Instances; - private final ImmutableMultimap dynamoTables; - private final ImmutableMultimap rdsInstances; - private final ImmutableMultimap ec2SGs; - private final ImmutableMultimap sqsQueues; - private final ImmutableMultimap elasticacheClusters; - private final ImmutableMultimap elasticsearchClusters; - private final ImmutableList iamUsers; - private final long timestamp; + private TimestampedData> ec2Instances; + private TimestampedData> dynamoTables; + private TimestampedData> rdsInstances; + private TimestampedData> ec2SGs; + private TimestampedData> sqsQueues; + private TimestampedData> elasticacheClusters; + private TimestampedData> elasticsearchClusters; + private TimestampedData> iamUsers; private String awsAccountNumber; private String awsARNPartition; @@ -87,9 +86,6 @@ public class AWSDatabase { final AmazonIdentityManagement iamClient, final String configAWSAccountNumber, final String configAWSARNPartition) { - timestamp = System.currentTimeMillis(); - log.info("Building AWS DB with timestamp {}", timestamp); - if (configAWSAccountNumber == null) { awsAccountNumber = ""; } else { @@ -104,46 +100,118 @@ public class AWSDatabase { awsARNPartition = configAWSARNPartition; } - this.iamUsers = loadIAM(iamClient); - this.elasticacheClusters = loadElasticache(elasticacheClients); - this.elasticsearchClusters = loadElasticsearch(elasticsearchClients); - this.sqsQueues = loadSQS(sqsClients); - this.dynamoTables = loadDynamo(dynamoClients); - this.ec2Instances = loadEC2Instances(ec2Clients); - this.ec2SGs = loadEC2SGs(ec2Clients); - this.rdsInstances = loadRDS(rdsClients); - + refreshIamUsers(iamClient); + refreshElasticacheClusters(elasticacheClients); + refreshElasticsearchClusters(elasticsearchClients); + refreshSqsQueues(sqsClients); + refreshDynamoTables(dynamoClients); + refreshEc2Instances(ec2Clients); + refreshEc2SGs(ec2Clients); + refreshRdsInstances(rdsClients); log.info("Done building AWS DB"); } - private AWSDatabase(ImmutableMultimap ec2Instances, - ImmutableMultimap dynamoTables, - ImmutableMultimap rdsInstances, - ImmutableMultimap ec2SGs, - ImmutableMultimap sqsQueues, - ImmutableMultimap elasticacheClusters, - ImmutableMultimap elasticsearchClusters, - ImmutableList iamUsers, - long timestamp, - String awsAccountNumber, - String awsARNPartition) { - this.ec2Instances = ec2Instances; - this.dynamoTables = dynamoTables; - this.rdsInstances = rdsInstances; - this.ec2SGs = ec2SGs; - this.sqsQueues = sqsQueues; - this.elasticacheClusters = elasticacheClusters; - this.elasticsearchClusters = elasticsearchClusters; - this.iamUsers = iamUsers; - this.timestamp = timestamp; - this.awsAccountNumber = awsAccountNumber; - this.awsARNPartition = awsARNPartition; + /** + * @return the earliest timestamp of all data in the DB + */ + public long getTimestamp() { + return Collections.min(Arrays.asList( + ec2Instances.getTimestamp(), + dynamoTables.getTimestamp(), + rdsInstances.getTimestamp(), + ec2SGs.getTimestamp(), + sqsQueues.getTimestamp(), + elasticsearchClusters.getTimestamp(), + elasticsearchClusters.getTimestamp(), + iamUsers.getTimestamp() + )); + } + + public long getAgeInMs() { + return System.currentTimeMillis() - getTimestamp(); + } + + /** + * Public getters for AWS data to hide the TimestampedData wrapped class from users of this class + */ + + public ImmutableMultimap getEc2Instances() { + return ec2Instances.getData(); + } + + public ImmutableMultimap getDynamoTables() { + return dynamoTables.getData(); + } + + public ImmutableMultimap getRdsInstances() { + return rdsInstances.getData(); + } + + public ImmutableMultimap getEc2SGs() { + return ec2SGs.getData(); + } + + public ImmutableMultimap getSqsQueues() { + return sqsQueues.getData(); + } + + public ImmutableMultimap getElasticacheClusters() { + return elasticacheClusters.getData(); + } + + public ImmutableList getIamUsers() { + return iamUsers.getData(); + } + + public ImmutableMultimap getElasticsearchClusters() { + return elasticsearchClusters.getData(); } + /** + * Public API for updating data after initial build + */ + + public void refreshEc2Instances(final Map ec2Clients) { + this.ec2Instances = TimestampedData.withTimestamp(() -> loadEC2Instances(ec2Clients)); + } + + public void refreshDynamoTables(final Map dynamoClients) { + this.dynamoTables = TimestampedData.withTimestamp(() -> loadDynamo(dynamoClients)); + } + + public void refreshRdsInstances(final Map rdsClients) { + this.rdsInstances = TimestampedData.withTimestamp(() -> loadRDS(rdsClients)); + } + + public void refreshEc2SGs(final Map ec2Clients) { + this.ec2SGs = TimestampedData.withTimestamp(() -> loadEC2SGs(ec2Clients)); + } + + public void refreshSqsQueues(final Map sqsClients) { + this.sqsQueues = TimestampedData.withTimestamp(() -> loadSQS(sqsClients)); + } + + public void refreshElasticacheClusters(Map elasticacheClients) { + this.elasticacheClusters = TimestampedData.withTimestamp(() -> loadElasticache(elasticacheClients)); + } + + public void refreshIamUsers(final AmazonIdentityManagement iamClient) { + this.iamUsers = TimestampedData.withTimestamp(() -> loadIAM(iamClient)); + } + + public void refreshElasticsearchClusters(final Map elasticsearchClients) { + this.elasticsearchClusters = TimestampedData.withTimestamp(() -> loadElasticsearch(elasticsearchClients)); + } + + /** + * Private helper methods for loading data from the AWS API + */ private ImmutableList loadIAM(final AmazonIdentityManagement iamClient) { log.info("Getting IAM keys"); final ImmutableList.Builder usersBuilder = new ImmutableList.Builder<>(); + return usersBuilder.build(); + /* final ListUsersRequest listUsersRequest = new ListUsersRequest(); ListUsersResult listUsersResult; do { @@ -166,6 +234,8 @@ private ImmutableList loadIAM(final AmazonIdentityManagement ia listUsersRequest.setMarker(listUsersResult.getMarker()); } while (listUsersResult.isTruncated()); return usersBuilder.build(); + + */ } private ImmutableMultimap loadElasticache(final Map elasticacheClients) { @@ -414,10 +484,6 @@ private ImmutableMultimap loadRDS(final Map { + private final T data; + private final long timestamp; + + public static TimestampedData withTimestamp(Supplier supplier) { + long timestamp = System.currentTimeMillis(); + return new TimestampedData(supplier.get(), timestamp); + } +} \ No newline at end of file diff --git a/src/main/java/com/airbnb/billow/AWSDatabaseHolderRefreshJob.java b/src/main/java/com/airbnb/billow/jobs/BaseAWSDatabaseHolderRefreshJob.java similarity index 85% rename from src/main/java/com/airbnb/billow/AWSDatabaseHolderRefreshJob.java rename to src/main/java/com/airbnb/billow/jobs/BaseAWSDatabaseHolderRefreshJob.java index fb83019..41bc81b 100644 --- a/src/main/java/com/airbnb/billow/AWSDatabaseHolderRefreshJob.java +++ b/src/main/java/com/airbnb/billow/jobs/BaseAWSDatabaseHolderRefreshJob.java @@ -1,17 +1,17 @@ -package com.airbnb.billow; +package com.airbnb.billow.jobs; +import com.airbnb.billow.AWSDatabaseHolder; import com.codahale.metrics.Counter; import org.quartz.Job; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; import org.quartz.SchedulerException; -public class AWSDatabaseHolderRefreshJob implements Job { +public abstract class BaseAWSDatabaseHolderRefreshJob implements Job { public static final String DB_KEY = "db"; public static final String FAILURE_COUNTER_KEY = "failure_counter"; public static final String START_COUNTER_KEY = "start_counter"; public static final String SUCCESS_COUNTER_KEY = "success_counter"; - public static final String NAME = "dbRefresh"; @Override public void execute(JobExecutionContext context) throws JobExecutionException { @@ -25,6 +25,8 @@ public void execute(JobExecutionContext context) throws JobExecutionException { } } + abstract void refresh(AWSDatabaseHolder dbHolder); + private void increment(String counterName, JobExecutionContext context) { try { ((Counter) context.getScheduler().getContext().get(counterName)).inc(); diff --git a/src/main/java/com/airbnb/billow/jobs/DynamoRefreshJob.java b/src/main/java/com/airbnb/billow/jobs/DynamoRefreshJob.java new file mode 100644 index 0000000..d477537 --- /dev/null +++ b/src/main/java/com/airbnb/billow/jobs/DynamoRefreshJob.java @@ -0,0 +1,12 @@ +package com.airbnb.billow.jobs; + +import com.airbnb.billow.AWSDatabaseHolder; + +public class DynamoRefreshJob extends BaseAWSDatabaseHolderRefreshJob { + public static final String NAME = "dynamo_job"; + + @Override + void refresh(AWSDatabaseHolder dbHolder) { + dbHolder.refreshDynamoTables(); + } +} diff --git a/src/main/java/com/airbnb/billow/jobs/Ec2InstanceRefreshJob.java b/src/main/java/com/airbnb/billow/jobs/Ec2InstanceRefreshJob.java new file mode 100644 index 0000000..3edc7ab --- /dev/null +++ b/src/main/java/com/airbnb/billow/jobs/Ec2InstanceRefreshJob.java @@ -0,0 +1,12 @@ +package com.airbnb.billow.jobs; + +import com.airbnb.billow.AWSDatabaseHolder; + +public class Ec2InstanceRefreshJob extends BaseAWSDatabaseHolderRefreshJob { + public static final String NAME = "ec2_instance_job"; + + @Override + void refresh(AWSDatabaseHolder dbHolder) { + dbHolder.refreshEc2Instances(); + } +} diff --git a/src/main/java/com/airbnb/billow/jobs/Ec2SGRefreshJob.java b/src/main/java/com/airbnb/billow/jobs/Ec2SGRefreshJob.java new file mode 100644 index 0000000..015018d --- /dev/null +++ b/src/main/java/com/airbnb/billow/jobs/Ec2SGRefreshJob.java @@ -0,0 +1,12 @@ +package com.airbnb.billow.jobs; + +import com.airbnb.billow.AWSDatabaseHolder; + +public class Ec2SGRefreshJob extends BaseAWSDatabaseHolderRefreshJob { + public static final String NAME = "ec2_sg_job"; + + @Override + void refresh(AWSDatabaseHolder dbHolder) { + dbHolder.refreshEc2SGs(); + } +} diff --git a/src/main/java/com/airbnb/billow/jobs/ElasticacheRefreshJob.java b/src/main/java/com/airbnb/billow/jobs/ElasticacheRefreshJob.java new file mode 100644 index 0000000..a14e2b9 --- /dev/null +++ b/src/main/java/com/airbnb/billow/jobs/ElasticacheRefreshJob.java @@ -0,0 +1,12 @@ +package com.airbnb.billow.jobs; + +import com.airbnb.billow.AWSDatabaseHolder; + +public class ElasticacheRefreshJob extends BaseAWSDatabaseHolderRefreshJob { + public static final String NAME = "elasticache_job"; + + @Override + void refresh(AWSDatabaseHolder dbHolder) { + dbHolder.refreshElasticacheClusters(); + } +} diff --git a/src/main/java/com/airbnb/billow/jobs/ElasticsearchRefreshJob.java b/src/main/java/com/airbnb/billow/jobs/ElasticsearchRefreshJob.java new file mode 100644 index 0000000..f1557d8 --- /dev/null +++ b/src/main/java/com/airbnb/billow/jobs/ElasticsearchRefreshJob.java @@ -0,0 +1,12 @@ +package com.airbnb.billow.jobs; + +import com.airbnb.billow.AWSDatabaseHolder; + +public class ElasticsearchRefreshJob extends BaseAWSDatabaseHolderRefreshJob { + public static final String NAME = "elasticsearch_job"; + + @Override + void refresh(AWSDatabaseHolder dbHolder) { + dbHolder.refreshElasticsearchClusters(); + } +} diff --git a/src/main/java/com/airbnb/billow/jobs/IamRefreshJob.java b/src/main/java/com/airbnb/billow/jobs/IamRefreshJob.java new file mode 100644 index 0000000..f869e42 --- /dev/null +++ b/src/main/java/com/airbnb/billow/jobs/IamRefreshJob.java @@ -0,0 +1,12 @@ +package com.airbnb.billow.jobs; + +import com.airbnb.billow.AWSDatabaseHolder; + +public class IamRefreshJob extends BaseAWSDatabaseHolderRefreshJob { + public static final String NAME = "iam_job"; + + @Override + void refresh(AWSDatabaseHolder dbHolder) { + dbHolder.refreshIamUsers(); + } +} diff --git a/src/main/java/com/airbnb/billow/jobs/RdsRefreshJob.java b/src/main/java/com/airbnb/billow/jobs/RdsRefreshJob.java new file mode 100644 index 0000000..fcbe475 --- /dev/null +++ b/src/main/java/com/airbnb/billow/jobs/RdsRefreshJob.java @@ -0,0 +1,12 @@ +package com.airbnb.billow.jobs; + +import com.airbnb.billow.AWSDatabaseHolder; + +public class RdsRefreshJob extends BaseAWSDatabaseHolderRefreshJob{ + public static final String NAME = "rds_job"; + + @Override + void refresh(AWSDatabaseHolder dbHolder) { + dbHolder.refreshRdsInstances(); + } +} diff --git a/src/main/java/com/airbnb/billow/jobs/SqsRefreshJob.java b/src/main/java/com/airbnb/billow/jobs/SqsRefreshJob.java new file mode 100644 index 0000000..f10172a --- /dev/null +++ b/src/main/java/com/airbnb/billow/jobs/SqsRefreshJob.java @@ -0,0 +1,12 @@ +package com.airbnb.billow.jobs; + +import com.airbnb.billow.AWSDatabaseHolder; + +public class SqsRefreshJob extends BaseAWSDatabaseHolderRefreshJob { + public static final String NAME = "sqs_job"; + + @Override + void refresh(AWSDatabaseHolder dbHolder) { + dbHolder.refreshSqsQueues(); + } +} From 635193d27a5cc5a3e637f549fccad5e13324f467 Mon Sep 17 00:00:00 2001 From: Ben Muschol Date: Tue, 23 Mar 2021 15:24:20 -0400 Subject: [PATCH 3/6] bump version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1d3be4f..91844e1 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.airbnb.billow billow - 2.21 + 2.22 From 4bbcb871be24bef75b07e989596abc5ff9b43b33 Mon Sep 17 00:00:00 2001 From: Ben Muschol Date: Wed, 24 Mar 2021 13:25:19 -0400 Subject: [PATCH 4/6] Fix IAM --- src/main/java/com/airbnb/billow/AWSDatabase.java | 7 ++----- src/main/java/com/airbnb/billow/AWSDatabaseHolder.java | 1 + 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/airbnb/billow/AWSDatabase.java b/src/main/java/com/airbnb/billow/AWSDatabase.java index 9f53a82..ed0e938 100644 --- a/src/main/java/com/airbnb/billow/AWSDatabase.java +++ b/src/main/java/com/airbnb/billow/AWSDatabase.java @@ -87,6 +87,7 @@ public class AWSDatabase { final String configAWSAccountNumber, final String configAWSARNPartition) { if (configAWSAccountNumber == null) { + log.info("No AWS account number given"); awsAccountNumber = ""; } else { log.info("using account number '{}' from config", configAWSAccountNumber); @@ -209,10 +210,8 @@ public void refreshElasticsearchClusters(final Map loadIAM(final AmazonIdentityManagement iamClient) { log.info("Getting IAM keys"); final ImmutableList.Builder usersBuilder = new ImmutableList.Builder<>(); - return usersBuilder.build(); - /* - final ListUsersRequest listUsersRequest = new ListUsersRequest(); + ListUsersRequest listUsersRequest = new ListUsersRequest(); ListUsersResult listUsersResult; do { log.debug("Performing IAM request: {}", listUsersRequest); @@ -234,8 +233,6 @@ private ImmutableList loadIAM(final AmazonIdentityManagement ia listUsersRequest.setMarker(listUsersResult.getMarker()); } while (listUsersResult.isTruncated()); return usersBuilder.build(); - - */ } private ImmutableMultimap loadElasticache(final Map elasticacheClients) { diff --git a/src/main/java/com/airbnb/billow/AWSDatabaseHolder.java b/src/main/java/com/airbnb/billow/AWSDatabaseHolder.java index 9f6f367..438f7d7 100644 --- a/src/main/java/com/airbnb/billow/AWSDatabaseHolder.java +++ b/src/main/java/com/airbnb/billow/AWSDatabaseHolder.java @@ -129,6 +129,7 @@ public AWSDatabaseHolder(Config config) { * Build a fresh version of the DB completely from scratch */ public void rebuild() { + log.info(String.format("Using AWS Account number: '%s'", awsAccountNumber)); current = new AWSDatabase( ec2Clients, rdsClients, From 06756f21fb891ef6be717b386b6bdcfe65d413ef Mon Sep 17 00:00:00 2001 From: Ben Muschol Date: Wed, 24 Mar 2021 17:11:07 -0400 Subject: [PATCH 5/6] Bump version again --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 91844e1..e89ee7b 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.airbnb.billow billow - 2.22 + 2.23 From 8f250385611e2908ba29a92e1a63da4bc6aa43af Mon Sep 17 00:00:00 2001 From: Ben Muschol Date: Thu, 25 Mar 2021 16:31:38 -0400 Subject: [PATCH 6/6] More logging --- pom.xml | 2 +- .../airbnb/billow/jobs/BaseAWSDatabaseHolderRefreshJob.java | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e89ee7b..c2fdc75 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.airbnb.billow billow - 2.23 + 2.24 diff --git a/src/main/java/com/airbnb/billow/jobs/BaseAWSDatabaseHolderRefreshJob.java b/src/main/java/com/airbnb/billow/jobs/BaseAWSDatabaseHolderRefreshJob.java index 41bc81b..571fcce 100644 --- a/src/main/java/com/airbnb/billow/jobs/BaseAWSDatabaseHolderRefreshJob.java +++ b/src/main/java/com/airbnb/billow/jobs/BaseAWSDatabaseHolderRefreshJob.java @@ -2,11 +2,13 @@ import com.airbnb.billow.AWSDatabaseHolder; import com.codahale.metrics.Counter; +import lombok.extern.slf4j.Slf4j; import org.quartz.Job; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; import org.quartz.SchedulerException; +@Slf4j public abstract class BaseAWSDatabaseHolderRefreshJob implements Job { public static final String DB_KEY = "db"; public static final String FAILURE_COUNTER_KEY = "failure_counter"; @@ -18,8 +20,10 @@ public void execute(JobExecutionContext context) throws JobExecutionException { try { increment(START_COUNTER_KEY, context); ((AWSDatabaseHolder) context.getScheduler().getContext().get(DB_KEY)).rebuild(); + log.info("[job success] {} completed", this.getClass().toString()); increment(SUCCESS_COUNTER_KEY, context); } catch (SchedulerException e) { + log.error("[job failure] {} completed", this.getClass().toString()); increment(FAILURE_COUNTER_KEY, context); throw new JobExecutionException(e); }