Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore(cleanup): refactoring down to single accounts across all services #935

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
Open

chore(cleanup): refactoring down to single accounts across all services #935

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
6b1c440
chore(cleanup): refactoring down to single accounts across all services
jasonmcintosh Feb 28, 2023
92f996c
chore(refactor): Dedupe graphite
jasonmcintosh Feb 28, 2023
fcd21ba
Merge branch 'master' into refactorRoundB
jasonmcintosh Feb 28, 2023
a0e732c
chore(refactor): Final set of accounts to cleanup
jasonmcintosh Feb 28, 2023
44cdc7c
chore(refactor): Final set of accounts to cleanup
jasonmcintosh Feb 28, 2023
c16169e
Merge branch 'refactorRoundB' of github.com:armory-io/kayenta into re…
jasonmcintosh Feb 28, 2023
e1ca8cf
Merge branch 'master' into refactorRoundB
jasonmcintosh Mar 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 6 additions & 24 deletions kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/config/AtlasConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,6 @@
import com.netflix.kayenta.atlas.backends.BackendUpdater;
import com.netflix.kayenta.atlas.backends.BackendUpdaterService;
import com.netflix.kayenta.atlas.metrics.AtlasMetricsService;
import com.netflix.kayenta.atlas.security.AtlasCredentials;
import com.netflix.kayenta.atlas.security.AtlasNamedAccountCredentials;
import com.netflix.kayenta.metrics.MetricsService;
import com.netflix.kayenta.security.AccountCredentials;
import com.netflix.kayenta.security.AccountCredentialsRepository;
Expand All @@ -34,7 +32,6 @@
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.util.CollectionUtils;

@Configuration
@ConditionalOnProperty("kayenta.atlas.enabled")
Expand Down Expand Up @@ -63,42 +60,27 @@ AtlasConfigurationProperties atlasConfigurationProperties() {
MetricsService atlasMetricsService(
AtlasConfigurationProperties atlasConfigurationProperties,
AccountCredentialsRepository accountCredentialsRepository) {

AtlasMetricsService.AtlasMetricsServiceBuilder atlasMetricsServiceBuilder =
AtlasMetricsService.builder();

for (AtlasManagedAccount atlasManagedAccount : atlasConfigurationProperties.getAccounts()) {
String name = atlasManagedAccount.getName();
List<AccountCredentials.Type> supportedTypes = atlasManagedAccount.getSupportedTypes();
String backendsJsonUriPrefix = atlasManagedAccount.getBackendsJsonBaseUrl();

log.info("Registering Atlas account {} with supported types {}.", name, supportedTypes);

AtlasCredentials atlasCredentials = AtlasCredentials.builder().build();

BackendUpdater backendUpdater = BackendUpdater.builder().uri(backendsJsonUriPrefix).build();
AtlasStorageUpdater atlasStorageUpdater =
AtlasStorageUpdater.builder().uri(backendsJsonUriPrefix).build();
AtlasNamedAccountCredentials.AtlasNamedAccountCredentialsBuilder
atlasNamedAccountCredentialsBuilder =
AtlasNamedAccountCredentials.builder()
.name(name)
.credentials(atlasCredentials)
.fetchId(atlasManagedAccount.getFetchId())
.recommendedLocations(atlasManagedAccount.getRecommendedLocations())
.atlasStorageUpdater(atlasStorageUpdater)
.backendUpdater(backendUpdater);

if (!CollectionUtils.isEmpty(supportedTypes)) {
atlasNamedAccountCredentialsBuilder.supportedTypes(supportedTypes);
}
atlasManagedAccount.setAtlasStorageUpdater(atlasStorageUpdater);
atlasManagedAccount.setBackendUpdater(backendUpdater);
accountCredentialsRepository.save(atlasManagedAccount);

AtlasNamedAccountCredentials atlasNamedAccountCredentials =
atlasNamedAccountCredentialsBuilder.build();
accountCredentialsRepository.save(name, atlasNamedAccountCredentials);
backendUpdaterService.add(backendUpdater);
atlasStorageUpdaterService.add(atlasStorageUpdater);
atlasMetricsServiceBuilder.accountName(name);

backendUpdaterService.add(atlasNamedAccountCredentials.getBackendUpdater());
atlasStorageUpdaterService.add(atlasNamedAccountCredentials.getAtlasStorageUpdater());
}

AtlasMetricsService atlasMetricsService = atlasMetricsServiceBuilder.build();
Expand Down
26 changes: 20 additions & 6 deletions kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/config/AtlasManagedAccount.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,21 +16,35 @@

package com.netflix.kayenta.atlas.config;

import com.fasterxml.jackson.annotation.JsonIgnore;
import com.netflix.kayenta.atlas.backends.AtlasStorageUpdater;
import com.netflix.kayenta.atlas.backends.BackendUpdater;
import com.netflix.kayenta.security.AccountCredentials;
import java.util.ArrayList;
import java.util.List;
import javax.validation.constraints.NotNull;
import lombok.Data;

@Data
public class AtlasManagedAccount {

@NotNull private String name;

private List<AccountCredentials.Type> supportedTypes;
public class AtlasManagedAccount extends AccountCredentials<AtlasManagedAccount> {

@NotNull private String backendsJsonBaseUrl;

private String fetchId;

private List<String> recommendedLocations;
private List<String> recommendedLocations = new ArrayList<>();

@Override
public List<String> getLocations() {
return getBackendUpdater().getBackendDatabase().getLocations();
}

@JsonIgnore private transient BackendUpdater backendUpdater;
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I know it might be a little early to ask, but if you intend on making these credentials storable, then it would also make sense for them to work similarly through the API. Simply supporting JPA isn't enough; we'd need a way to marshal and unmarshal these through the REST API so that they can be specified from somewhere other than a config file in the first place.


@JsonIgnore private transient AtlasStorageUpdater atlasStorageUpdater;

@Override
public String getType() {
return "atlas";
}
}
6 changes: 3 additions & 3 deletions kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/metrics/AtlasMetricsService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,11 @@
import com.netflix.kayenta.atlas.backends.AtlasStorageDatabase;
import com.netflix.kayenta.atlas.backends.BackendDatabase;
import com.netflix.kayenta.atlas.canary.AtlasCanaryScope;
import com.netflix.kayenta.atlas.config.AtlasManagedAccount;
import com.netflix.kayenta.atlas.config.AtlasSSEConverter;
import com.netflix.kayenta.atlas.model.AtlasResults;
import com.netflix.kayenta.atlas.model.AtlasResultsHelper;
import com.netflix.kayenta.atlas.model.Backend;
import com.netflix.kayenta.atlas.security.AtlasNamedAccountCredentials;
import com.netflix.kayenta.atlas.service.AtlasRemoteService;
import com.netflix.kayenta.canary.CanaryConfig;
import com.netflix.kayenta.canary.CanaryMetricConfig;
Expand Down Expand Up @@ -83,7 +83,7 @@ public boolean servicesAccount(String accountName) {
return accountNames.contains(accountName);
}

private AtlasNamedAccountCredentials getCredentials(String accountName) {
private AtlasManagedAccount getCredentials(String accountName) {
return accountCredentialsRepository.getRequiredOne(accountName);
}

Expand All @@ -107,7 +107,7 @@ public List<MetricSet> queryMetrics(
}

AtlasCanaryScope atlasCanaryScope = (AtlasCanaryScope) canaryScope;
AtlasNamedAccountCredentials credentials = getCredentials(accountName);
AtlasManagedAccount credentials = getCredentials(accountName);
BackendDatabase backendDatabase = credentials.getBackendUpdater().getBackendDatabase();
String uri = backendDatabase.getUriForLocation(URI_SCHEME, atlasCanaryScope.getLocation());

Expand Down
32 changes: 0 additions & 32 deletions kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/security/AtlasCredentials.java
View file
Open in desktop

This file was deleted.

63 changes: 0 additions & 63 deletions ...-atlas/src/main/java/com/netflix/kayenta/atlas/security/AtlasNamedAccountCredentials.java
View file
Open in desktop

This file was deleted.

127 changes: 56 additions & 71 deletions kayenta-aws/src/main/java/com/netflix/kayenta/aws/config/AwsConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,8 @@
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.netflix.kayenta.aws.security.AwsCredentials;
import com.netflix.kayenta.aws.security.AwsNamedAccountCredentials;
import com.netflix.kayenta.security.AccountCredentials;
import com.netflix.kayenta.security.AccountCredentialsRepository;
import java.io.IOException;
import java.util.List;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
Expand All @@ -57,70 +54,14 @@ AwsConfigurationProperties awsConfigurationProperties() {
@Bean
boolean registerAwsCredentials(
AwsConfigurationProperties awsConfigurationProperties,
AccountCredentialsRepository accountCredentialsRepository)
throws IOException {
AccountCredentialsRepository accountCredentialsRepository) {
for (AwsManagedAccount awsManagedAccount : awsConfigurationProperties.getAccounts()) {
String name = awsManagedAccount.getName();
List<AccountCredentials.Type> supportedTypes = awsManagedAccount.getSupportedTypes();

log.info("Registering AWS account {} with supported types {}.", name, supportedTypes);

ClientConfiguration clientConfiguration = new ClientConfiguration();

if (awsManagedAccount.getProxyProtocol() != null) {
if (awsManagedAccount.getProxyProtocol().equalsIgnoreCase("HTTPS")) {
clientConfiguration.setProtocol(Protocol.HTTPS);
} else {
clientConfiguration.setProtocol(Protocol.HTTP);
}
Optional.ofNullable(awsManagedAccount.getProxyHost())
.ifPresent(clientConfiguration::setProxyHost);
Optional.ofNullable(awsManagedAccount.getProxyPort())
.map(Integer::parseInt)
.ifPresent(clientConfiguration::setProxyPort);
}

AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard();
String profileName = awsManagedAccount.getProfileName();

if (!StringUtils.isEmpty(profileName)) {
amazonS3ClientBuilder.withCredentials(new ProfileCredentialsProvider(profileName));
}

AwsManagedAccount.ExplicitAwsCredentials explicitCredentials =
awsManagedAccount.getExplicitCredentials();
if (explicitCredentials != null) {
String sessionToken = explicitCredentials.getSessionToken();
AWSCredentials awsCreds =
(sessionToken == null)
? new BasicAWSCredentials(
explicitCredentials.getAccessKey(), explicitCredentials.getSecretKey())
: new BasicSessionCredentials(
explicitCredentials.getAccessKey(),
explicitCredentials.getSecretKey(),
sessionToken);
amazonS3ClientBuilder.withCredentials(new AWSStaticCredentialsProvider(awsCreds));
}

String endpoint = awsManagedAccount.getEndpoint();

if (!StringUtils.isEmpty(endpoint)) {
amazonS3ClientBuilder.setEndpointConfiguration(
new AwsClientBuilder.EndpointConfiguration(endpoint, null));
amazonS3ClientBuilder.setPathStyleAccessEnabled(true);
} else {
Optional.ofNullable(awsManagedAccount.getRegion())
.ifPresent(amazonS3ClientBuilder::setRegion);
}

AmazonS3 amazonS3 = amazonS3ClientBuilder.build();

try {
AwsCredentials awsCredentials = new AwsCredentials();
AwsNamedAccountCredentials.AwsNamedAccountCredentialsBuilder
awsNamedAccountCredentialsBuilder =
AwsNamedAccountCredentials.builder().name(name).credentials(awsCredentials);

if (!CollectionUtils.isEmpty(supportedTypes)) {
if (supportedTypes.contains(AccountCredentials.Type.OBJECT_STORE)) {
String bucket = awsManagedAccount.getBucket();
Expand All @@ -135,24 +76,68 @@ boolean registerAwsCredentials(
throw new IllegalArgumentException(
"AWS/S3 account " + name + " is required to specify a rootFolder.");
}

awsNamedAccountCredentialsBuilder.bucket(bucket);
awsNamedAccountCredentialsBuilder.region(awsManagedAccount.getRegion());
awsNamedAccountCredentialsBuilder.rootFolder(rootFolder);
awsNamedAccountCredentialsBuilder.amazonS3(amazonS3);
AmazonS3 amazonS3 = getAmazonS3(awsManagedAccount);
awsManagedAccount.setAmazonS3(amazonS3);
}

awsNamedAccountCredentialsBuilder.supportedTypes(supportedTypes);
}

AwsNamedAccountCredentials awsNamedAccountCredentials =
awsNamedAccountCredentialsBuilder.build();
accountCredentialsRepository.save(name, awsNamedAccountCredentials);
accountCredentialsRepository.save(awsManagedAccount);
} catch (Throwable t) {
log.error("Could not load AWS account " + name + ".", t);
}
}

return true;
}

// Good candidate for CredentialsParser type logic here...
private static AmazonS3 getAmazonS3(AwsManagedAccount awsManagedAccount) {
AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard();
if (awsManagedAccount.getProxyProtocol() != null) {
ClientConfiguration clientConfiguration = new ClientConfiguration();
if (awsManagedAccount.getProxyProtocol().equalsIgnoreCase("HTTPS")) {
clientConfiguration.setProtocol(Protocol.HTTPS);
} else {
clientConfiguration.setProtocol(Protocol.HTTP);
}
Optional.ofNullable(awsManagedAccount.getProxyHost())
.ifPresent(clientConfiguration::setProxyHost);
Optional.ofNullable(awsManagedAccount.getProxyPort())
.map(Integer::parseInt)
.ifPresent(clientConfiguration::setProxyPort);

amazonS3ClientBuilder.setClientConfiguration(clientConfiguration);
}

String profileName = awsManagedAccount.getProfileName();
if (StringUtils.hasText(profileName)) {
amazonS3ClientBuilder.withCredentials(new ProfileCredentialsProvider(profileName));
}

AwsManagedAccount.ExplicitAwsCredentials explicitCredentials =
awsManagedAccount.getExplicitCredentials();
if (explicitCredentials != null) {
String sessionToken = explicitCredentials.getSessionToken();
AWSCredentials awsCreds =
(sessionToken == null)
? new BasicAWSCredentials(
explicitCredentials.getAccessKey(), explicitCredentials.getSecretKey())
: new BasicSessionCredentials(
explicitCredentials.getAccessKey(),
explicitCredentials.getSecretKey(),
sessionToken);
amazonS3ClientBuilder.withCredentials(new AWSStaticCredentialsProvider(awsCreds));
}

String endpoint = awsManagedAccount.getEndpoint();
if (StringUtils.hasText(endpoint)) {
amazonS3ClientBuilder.setEndpointConfiguration(
new AwsClientBuilder.EndpointConfiguration(endpoint, null));
amazonS3ClientBuilder.setPathStyleAccessEnabled(true);
} else {
Optional.ofNullable(awsManagedAccount.getRegion())
.ifPresent(amazonS3ClientBuilder::setRegion);
}

return amazonS3ClientBuilder.build();
}
}
Loading