From 6b1c440480e4c10ea05987c09c0f9af038f9de10 Mon Sep 17 00:00:00 2001 From: Jason McIntosh Date: Tue, 28 Feb 2023 14:36:06 -0600 Subject: [PATCH 1/4] chore(cleanup): refactoring down to single accounts across all services --- .../atlas/config/AtlasConfiguration.java | 30 +---- .../atlas/config/AtlasManagedAccount.java | 26 +++- .../atlas/metrics/AtlasMetricsService.java | 6 +- .../atlas/security/AtlasCredentials.java | 32 ----- .../AtlasNamedAccountCredentials.java | 63 --------- .../kayenta/aws/config/AwsConfiguration.java | 127 ++++++++---------- .../kayenta/aws/config/AwsManagedAccount.java | 26 ++-- .../kayenta/aws/security/AwsCredentials.java | 30 ----- .../security/AwsNamedAccountCredentials.java | 44 ------ .../azure/config/AzureConfiguration.java | 55 +++++--- .../azure/config/AzureManagedAccount.java | 24 +++- .../azure/security/AzureCredentials.java | 69 ---------- .../AzureNamedAccountCredentials.java | 44 ------ .../blobs/config/BlobsConfiguration.java | 4 +- .../blobs/storage/BlobsStorageService.java | 32 ++--- .../TestableBlobsStorageServiceTest.java | 28 ++-- .../kayenta/security/AccountCredentials.java | 8 +- .../AccountCredentialsRepository.java | 5 + .../datadog/config/DatadogConfiguration.java | 23 +--- .../datadog/config/DatadogManagedAccount.java | 34 +++-- .../metrics/DatadogMetricsService.java | 96 +++++++------ .../datadog/security/DatadogCredentials.java | 34 ----- .../DatadogNamedAccountCredentials.java | 45 ------- .../kayenta/gcs/config/GcsConfiguration.java | 4 +- .../gcs/storage/GcsStorageService.java | 28 ++-- .../google/config/GoogleConfiguration.java | 22 +-- .../google/config/GoogleManagedAccount.java | 17 ++- .../GoogleNamedAccountCredentials.java | 53 -------- .../config/PrometheusManagedAccount.java | 13 +- .../kayenta/s3/config/S3Configuration.java | 10 +- .../kayenta/s3/storage/S3StorageService.java | 27 ++-- .../config/SignalFxConfiguration.java | 53 ++------ .../config/SignalFxManagedAccount.java | 31 ++++- .../metrics/SignalFxMetricsService.java | 10 +- .../security/SignalFxCredentials.java | 28 ---- .../SignalFxNamedAccountCredentials.java | 49 ------- .../config/StackdriverConfiguration.java | 4 +- .../metrics/StackdriverMetricsService.java | 76 ++++++----- 38 files changed, 402 insertions(+), 908 deletions(-) delete mode 100644 kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/security/AtlasCredentials.java delete mode 100644 kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/security/AtlasNamedAccountCredentials.java delete mode 100644 kayenta-aws/src/main/java/com/netflix/kayenta/aws/security/AwsCredentials.java delete mode 100644 kayenta-aws/src/main/java/com/netflix/kayenta/aws/security/AwsNamedAccountCredentials.java delete mode 100644 kayenta-azure/src/main/java/com/netflix/kayenta/azure/security/AzureCredentials.java delete mode 100644 kayenta-azure/src/main/java/com/netflix/kayenta/azure/security/AzureNamedAccountCredentials.java delete mode 100644 kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/security/DatadogCredentials.java delete mode 100644 kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/security/DatadogNamedAccountCredentials.java delete mode 100644 kayenta-google/src/main/java/com/netflix/kayenta/google/security/GoogleNamedAccountCredentials.java delete mode 100644 kayenta-signalfx/src/main/java/com/netflix/kayenta/signalfx/security/SignalFxCredentials.java delete mode 100644 kayenta-signalfx/src/main/java/com/netflix/kayenta/signalfx/security/SignalFxNamedAccountCredentials.java diff --git a/kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/config/AtlasConfiguration.java b/kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/config/AtlasConfiguration.java index 045bbc502..e5e262ba9 100644 --- a/kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/config/AtlasConfiguration.java +++ b/kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/config/AtlasConfiguration.java @@ -21,8 +21,6 @@ import com.netflix.kayenta.atlas.backends.BackendUpdater; import com.netflix.kayenta.atlas.backends.BackendUpdaterService; import com.netflix.kayenta.atlas.metrics.AtlasMetricsService; -import com.netflix.kayenta.atlas.security.AtlasCredentials; -import com.netflix.kayenta.atlas.security.AtlasNamedAccountCredentials; import com.netflix.kayenta.metrics.MetricsService; import com.netflix.kayenta.security.AccountCredentials; import com.netflix.kayenta.security.AccountCredentialsRepository; @@ -34,7 +32,6 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; -import org.springframework.util.CollectionUtils; @Configuration @ConditionalOnProperty("kayenta.atlas.enabled") @@ -63,9 +60,9 @@ AtlasConfigurationProperties atlasConfigurationProperties() { MetricsService atlasMetricsService( AtlasConfigurationProperties atlasConfigurationProperties, AccountCredentialsRepository accountCredentialsRepository) { + AtlasMetricsService.AtlasMetricsServiceBuilder atlasMetricsServiceBuilder = AtlasMetricsService.builder(); - for (AtlasManagedAccount atlasManagedAccount : atlasConfigurationProperties.getAccounts()) { String name = atlasManagedAccount.getName(); List supportedTypes = atlasManagedAccount.getSupportedTypes(); @@ -73,32 +70,17 @@ MetricsService atlasMetricsService( log.info("Registering Atlas account {} with supported types {}.", name, supportedTypes); - AtlasCredentials atlasCredentials = AtlasCredentials.builder().build(); - BackendUpdater backendUpdater = BackendUpdater.builder().uri(backendsJsonUriPrefix).build(); AtlasStorageUpdater atlasStorageUpdater = AtlasStorageUpdater.builder().uri(backendsJsonUriPrefix).build(); - AtlasNamedAccountCredentials.AtlasNamedAccountCredentialsBuilder - atlasNamedAccountCredentialsBuilder = - AtlasNamedAccountCredentials.builder() - .name(name) - .credentials(atlasCredentials) - .fetchId(atlasManagedAccount.getFetchId()) - .recommendedLocations(atlasManagedAccount.getRecommendedLocations()) - .atlasStorageUpdater(atlasStorageUpdater) - .backendUpdater(backendUpdater); - if (!CollectionUtils.isEmpty(supportedTypes)) { - atlasNamedAccountCredentialsBuilder.supportedTypes(supportedTypes); - } + atlasManagedAccount.setAtlasStorageUpdater(atlasStorageUpdater); + atlasManagedAccount.setBackendUpdater(backendUpdater); + accountCredentialsRepository.save(atlasManagedAccount); - AtlasNamedAccountCredentials atlasNamedAccountCredentials = - atlasNamedAccountCredentialsBuilder.build(); - accountCredentialsRepository.save(name, atlasNamedAccountCredentials); + backendUpdaterService.add(backendUpdater); + atlasStorageUpdaterService.add(atlasStorageUpdater); atlasMetricsServiceBuilder.accountName(name); - - backendUpdaterService.add(atlasNamedAccountCredentials.getBackendUpdater()); - atlasStorageUpdaterService.add(atlasNamedAccountCredentials.getAtlasStorageUpdater()); } AtlasMetricsService atlasMetricsService = atlasMetricsServiceBuilder.build(); diff --git a/kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/config/AtlasManagedAccount.java b/kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/config/AtlasManagedAccount.java index 7b5eba7aa..75ba5abd9 100644 --- a/kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/config/AtlasManagedAccount.java +++ b/kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/config/AtlasManagedAccount.java @@ -16,21 +16,35 @@ package com.netflix.kayenta.atlas.config; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.netflix.kayenta.atlas.backends.AtlasStorageUpdater; +import com.netflix.kayenta.atlas.backends.BackendUpdater; import com.netflix.kayenta.security.AccountCredentials; +import java.util.ArrayList; import java.util.List; import javax.validation.constraints.NotNull; import lombok.Data; @Data -public class AtlasManagedAccount { - - @NotNull private String name; - - private List supportedTypes; +public class AtlasManagedAccount extends AccountCredentials { @NotNull private String backendsJsonBaseUrl; private String fetchId; - private List recommendedLocations; + private List recommendedLocations = new ArrayList<>(); + + @Override + public List getLocations() { + return getBackendUpdater().getBackendDatabase().getLocations(); + } + + @JsonIgnore private transient BackendUpdater backendUpdater; + + @JsonIgnore private transient AtlasStorageUpdater atlasStorageUpdater; + + @Override + public String getType() { + return "atlas"; + } } diff --git a/kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/metrics/AtlasMetricsService.java b/kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/metrics/AtlasMetricsService.java index 2e3a0c2db..3864e2c93 100644 --- a/kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/metrics/AtlasMetricsService.java +++ b/kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/metrics/AtlasMetricsService.java @@ -22,11 +22,11 @@ import com.netflix.kayenta.atlas.backends.AtlasStorageDatabase; import com.netflix.kayenta.atlas.backends.BackendDatabase; import com.netflix.kayenta.atlas.canary.AtlasCanaryScope; +import com.netflix.kayenta.atlas.config.AtlasManagedAccount; import com.netflix.kayenta.atlas.config.AtlasSSEConverter; import com.netflix.kayenta.atlas.model.AtlasResults; import com.netflix.kayenta.atlas.model.AtlasResultsHelper; import com.netflix.kayenta.atlas.model.Backend; -import com.netflix.kayenta.atlas.security.AtlasNamedAccountCredentials; import com.netflix.kayenta.atlas.service.AtlasRemoteService; import com.netflix.kayenta.canary.CanaryConfig; import com.netflix.kayenta.canary.CanaryMetricConfig; @@ -83,7 +83,7 @@ public boolean servicesAccount(String accountName) { return accountNames.contains(accountName); } - private AtlasNamedAccountCredentials getCredentials(String accountName) { + private AtlasManagedAccount getCredentials(String accountName) { return accountCredentialsRepository.getRequiredOne(accountName); } @@ -107,7 +107,7 @@ public List queryMetrics( } AtlasCanaryScope atlasCanaryScope = (AtlasCanaryScope) canaryScope; - AtlasNamedAccountCredentials credentials = getCredentials(accountName); + AtlasManagedAccount credentials = getCredentials(accountName); BackendDatabase backendDatabase = credentials.getBackendUpdater().getBackendDatabase(); String uri = backendDatabase.getUriForLocation(URI_SCHEME, atlasCanaryScope.getLocation()); diff --git a/kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/security/AtlasCredentials.java b/kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/security/AtlasCredentials.java deleted file mode 100644 index 914b349ab..000000000 --- a/kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/security/AtlasCredentials.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2017 Google, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.kayenta.atlas.security; - -import java.util.Optional; -import lombok.Builder; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; - -@Builder -@Data -@Slf4j -public class AtlasCredentials { - - private static String applicationVersion = - Optional.ofNullable(AtlasCredentials.class.getPackage().getImplementationVersion()) - .orElse("Unknown"); -} diff --git a/kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/security/AtlasNamedAccountCredentials.java b/kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/security/AtlasNamedAccountCredentials.java deleted file mode 100644 index 0b9abb5e8..000000000 --- a/kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/security/AtlasNamedAccountCredentials.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2017 Google, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.kayenta.atlas.security; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.netflix.kayenta.atlas.backends.AtlasStorageUpdater; -import com.netflix.kayenta.atlas.backends.BackendUpdater; -import com.netflix.kayenta.security.AccountCredentials; -import java.util.Collections; -import java.util.List; -import javax.validation.constraints.NotNull; -import lombok.Getter; -import lombok.Setter; -import lombok.experimental.SuperBuilder; - -@SuperBuilder -@Setter -@Getter -public class AtlasNamedAccountCredentials extends AccountCredentials { - - @Override - public String getType() { - return "atlas"; - } - - @NotNull private AtlasCredentials credentials; - - private String fetchId; - - private List recommendedLocations; - - @JsonIgnore private BackendUpdater backendUpdater; - - @JsonIgnore private AtlasStorageUpdater atlasStorageUpdater; - - @Override - public List getLocations() { - return getBackendUpdater().getBackendDatabase().getLocations(); - } - - @Override - public List getRecommendedLocations() { - if (recommendedLocations == null) { - return Collections.emptyList(); - } else { - return recommendedLocations; - } - } -} diff --git a/kayenta-aws/src/main/java/com/netflix/kayenta/aws/config/AwsConfiguration.java b/kayenta-aws/src/main/java/com/netflix/kayenta/aws/config/AwsConfiguration.java index ec8b54c61..9bdd74e2f 100644 --- a/kayenta-aws/src/main/java/com/netflix/kayenta/aws/config/AwsConfiguration.java +++ b/kayenta-aws/src/main/java/com/netflix/kayenta/aws/config/AwsConfiguration.java @@ -26,11 +26,8 @@ import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.netflix.kayenta.aws.security.AwsCredentials; -import com.netflix.kayenta.aws.security.AwsNamedAccountCredentials; import com.netflix.kayenta.security.AccountCredentials; import com.netflix.kayenta.security.AccountCredentialsRepository; -import java.io.IOException; import java.util.List; import java.util.Optional; import lombok.extern.slf4j.Slf4j; @@ -57,70 +54,14 @@ AwsConfigurationProperties awsConfigurationProperties() { @Bean boolean registerAwsCredentials( AwsConfigurationProperties awsConfigurationProperties, - AccountCredentialsRepository accountCredentialsRepository) - throws IOException { + AccountCredentialsRepository accountCredentialsRepository) { for (AwsManagedAccount awsManagedAccount : awsConfigurationProperties.getAccounts()) { String name = awsManagedAccount.getName(); List supportedTypes = awsManagedAccount.getSupportedTypes(); log.info("Registering AWS account {} with supported types {}.", name, supportedTypes); - ClientConfiguration clientConfiguration = new ClientConfiguration(); - - if (awsManagedAccount.getProxyProtocol() != null) { - if (awsManagedAccount.getProxyProtocol().equalsIgnoreCase("HTTPS")) { - clientConfiguration.setProtocol(Protocol.HTTPS); - } else { - clientConfiguration.setProtocol(Protocol.HTTP); - } - Optional.ofNullable(awsManagedAccount.getProxyHost()) - .ifPresent(clientConfiguration::setProxyHost); - Optional.ofNullable(awsManagedAccount.getProxyPort()) - .map(Integer::parseInt) - .ifPresent(clientConfiguration::setProxyPort); - } - - AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard(); - String profileName = awsManagedAccount.getProfileName(); - - if (!StringUtils.isEmpty(profileName)) { - amazonS3ClientBuilder.withCredentials(new ProfileCredentialsProvider(profileName)); - } - - AwsManagedAccount.ExplicitAwsCredentials explicitCredentials = - awsManagedAccount.getExplicitCredentials(); - if (explicitCredentials != null) { - String sessionToken = explicitCredentials.getSessionToken(); - AWSCredentials awsCreds = - (sessionToken == null) - ? new BasicAWSCredentials( - explicitCredentials.getAccessKey(), explicitCredentials.getSecretKey()) - : new BasicSessionCredentials( - explicitCredentials.getAccessKey(), - explicitCredentials.getSecretKey(), - sessionToken); - amazonS3ClientBuilder.withCredentials(new AWSStaticCredentialsProvider(awsCreds)); - } - - String endpoint = awsManagedAccount.getEndpoint(); - - if (!StringUtils.isEmpty(endpoint)) { - amazonS3ClientBuilder.setEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration(endpoint, null)); - amazonS3ClientBuilder.setPathStyleAccessEnabled(true); - } else { - Optional.ofNullable(awsManagedAccount.getRegion()) - .ifPresent(amazonS3ClientBuilder::setRegion); - } - - AmazonS3 amazonS3 = amazonS3ClientBuilder.build(); - try { - AwsCredentials awsCredentials = new AwsCredentials(); - AwsNamedAccountCredentials.AwsNamedAccountCredentialsBuilder - awsNamedAccountCredentialsBuilder = - AwsNamedAccountCredentials.builder().name(name).credentials(awsCredentials); - if (!CollectionUtils.isEmpty(supportedTypes)) { if (supportedTypes.contains(AccountCredentials.Type.OBJECT_STORE)) { String bucket = awsManagedAccount.getBucket(); @@ -135,24 +76,68 @@ boolean registerAwsCredentials( throw new IllegalArgumentException( "AWS/S3 account " + name + " is required to specify a rootFolder."); } - - awsNamedAccountCredentialsBuilder.bucket(bucket); - awsNamedAccountCredentialsBuilder.region(awsManagedAccount.getRegion()); - awsNamedAccountCredentialsBuilder.rootFolder(rootFolder); - awsNamedAccountCredentialsBuilder.amazonS3(amazonS3); + AmazonS3 amazonS3 = getAmazonS3(awsManagedAccount); + awsManagedAccount.setAmazonS3(amazonS3); } - - awsNamedAccountCredentialsBuilder.supportedTypes(supportedTypes); } - AwsNamedAccountCredentials awsNamedAccountCredentials = - awsNamedAccountCredentialsBuilder.build(); - accountCredentialsRepository.save(name, awsNamedAccountCredentials); + accountCredentialsRepository.save(awsManagedAccount); } catch (Throwable t) { log.error("Could not load AWS account " + name + ".", t); } } - return true; } + + // Good candidate for CredentialsParser type logic here... + private static AmazonS3 getAmazonS3(AwsManagedAccount awsManagedAccount) { + AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard(); + if (awsManagedAccount.getProxyProtocol() != null) { + ClientConfiguration clientConfiguration = new ClientConfiguration(); + if (awsManagedAccount.getProxyProtocol().equalsIgnoreCase("HTTPS")) { + clientConfiguration.setProtocol(Protocol.HTTPS); + } else { + clientConfiguration.setProtocol(Protocol.HTTP); + } + Optional.ofNullable(awsManagedAccount.getProxyHost()) + .ifPresent(clientConfiguration::setProxyHost); + Optional.ofNullable(awsManagedAccount.getProxyPort()) + .map(Integer::parseInt) + .ifPresent(clientConfiguration::setProxyPort); + + amazonS3ClientBuilder.setClientConfiguration(clientConfiguration); + } + + String profileName = awsManagedAccount.getProfileName(); + if (StringUtils.hasText(profileName)) { + amazonS3ClientBuilder.withCredentials(new ProfileCredentialsProvider(profileName)); + } + + AwsManagedAccount.ExplicitAwsCredentials explicitCredentials = + awsManagedAccount.getExplicitCredentials(); + if (explicitCredentials != null) { + String sessionToken = explicitCredentials.getSessionToken(); + AWSCredentials awsCreds = + (sessionToken == null) + ? new BasicAWSCredentials( + explicitCredentials.getAccessKey(), explicitCredentials.getSecretKey()) + : new BasicSessionCredentials( + explicitCredentials.getAccessKey(), + explicitCredentials.getSecretKey(), + sessionToken); + amazonS3ClientBuilder.withCredentials(new AWSStaticCredentialsProvider(awsCreds)); + } + + String endpoint = awsManagedAccount.getEndpoint(); + if (StringUtils.hasText(endpoint)) { + amazonS3ClientBuilder.setEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration(endpoint, null)); + amazonS3ClientBuilder.setPathStyleAccessEnabled(true); + } else { + Optional.ofNullable(awsManagedAccount.getRegion()) + .ifPresent(amazonS3ClientBuilder::setRegion); + } + + return amazonS3ClientBuilder.build(); + } } diff --git a/kayenta-aws/src/main/java/com/netflix/kayenta/aws/config/AwsManagedAccount.java b/kayenta-aws/src/main/java/com/netflix/kayenta/aws/config/AwsManagedAccount.java index 1ee4bcb00..24a189bff 100644 --- a/kayenta-aws/src/main/java/com/netflix/kayenta/aws/config/AwsManagedAccount.java +++ b/kayenta-aws/src/main/java/com/netflix/kayenta/aws/config/AwsManagedAccount.java @@ -16,15 +16,20 @@ package com.netflix.kayenta.aws.config; +import com.amazonaws.services.s3.AmazonS3; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.netflix.kayenta.security.AccountCredentials; -import java.util.List; -import javax.validation.constraints.NotNull; import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.SuperBuilder; -@Data -public class AwsManagedAccount { - - @NotNull private String name; +@Getter +@Setter +@SuperBuilder +@NoArgsConstructor +public class AwsManagedAccount extends AccountCredentials { private String bucket; private String region; @@ -36,8 +41,6 @@ public class AwsManagedAccount { private String proxyProtocol; private ExplicitAwsCredentials explicitCredentials; - private List supportedTypes; - @Data public static class ExplicitAwsCredentials { @@ -45,4 +48,11 @@ public static class ExplicitAwsCredentials { String secretKey; String sessionToken; } + + @Override + public String getType() { + return "aws"; + } + + @JsonIgnore private transient AmazonS3 amazonS3; } diff --git a/kayenta-aws/src/main/java/com/netflix/kayenta/aws/security/AwsCredentials.java b/kayenta-aws/src/main/java/com/netflix/kayenta/aws/security/AwsCredentials.java deleted file mode 100644 index 961ec90e9..000000000 --- a/kayenta-aws/src/main/java/com/netflix/kayenta/aws/security/AwsCredentials.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2017 Google, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.kayenta.aws.security; - -import java.util.Optional; -import lombok.ToString; -import lombok.extern.slf4j.Slf4j; - -@ToString -@Slf4j -public class AwsCredentials { - - private static String applicationVersion = - Optional.ofNullable(AwsCredentials.class.getPackage().getImplementationVersion()) - .orElse("Unknown"); -} diff --git a/kayenta-aws/src/main/java/com/netflix/kayenta/aws/security/AwsNamedAccountCredentials.java b/kayenta-aws/src/main/java/com/netflix/kayenta/aws/security/AwsNamedAccountCredentials.java deleted file mode 100644 index 7b5475b29..000000000 --- a/kayenta-aws/src/main/java/com/netflix/kayenta/aws/security/AwsNamedAccountCredentials.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2017 Google, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.kayenta.aws.security; - -import com.amazonaws.services.s3.AmazonS3; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.netflix.kayenta.security.AccountCredentials; -import javax.validation.constraints.NotNull; -import lombok.Getter; -import lombok.Setter; -import lombok.experimental.SuperBuilder; - -@SuperBuilder -@Setter -@Getter -public class AwsNamedAccountCredentials extends AccountCredentials { - - @NotNull private AwsCredentials credentials; - - private String bucket; - private String region; - private String rootFolder; - - @Override - public String getType() { - return "aws"; - } - - @JsonIgnore private AmazonS3 amazonS3; -} diff --git a/kayenta-azure/src/main/java/com/netflix/kayenta/azure/config/AzureConfiguration.java b/kayenta-azure/src/main/java/com/netflix/kayenta/azure/config/AzureConfiguration.java index 9412f3c2d..9e333306f 100644 --- a/kayenta-azure/src/main/java/com/netflix/kayenta/azure/config/AzureConfiguration.java +++ b/kayenta-azure/src/main/java/com/netflix/kayenta/azure/config/AzureConfiguration.java @@ -16,8 +16,9 @@ package com.netflix.kayenta.azure.config; -import com.netflix.kayenta.azure.security.AzureCredentials; -import com.netflix.kayenta.azure.security.AzureNamedAccountCredentials; +import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.blob.CloudBlobClient; +import com.microsoft.azure.storage.blob.CloudBlobContainer; import com.netflix.kayenta.security.AccountCredentials; import com.netflix.kayenta.security.AccountCredentialsRepository; import java.util.List; @@ -53,21 +54,11 @@ boolean registerAzureCredentials( for (AzureManagedAccount azureManagedAccount : azureAccounts) { String name = azureManagedAccount.getName(); - String storageAccountName = azureManagedAccount.getStorageAccountName(); List supportedTypes = azureManagedAccount.getSupportedTypes(); log.info("Registering Azure account {} with supported types {}.", name, supportedTypes); try { - String accountAccessKey = azureManagedAccount.getAccountAccessKey(); - String endpointSuffix = azureManagedAccount.getEndpointSuffix(); - AzureCredentials azureCredentials = - new AzureCredentials(storageAccountName, accountAccessKey, endpointSuffix); - - AzureNamedAccountCredentials.AzureNamedAccountCredentialsBuilder - azureNamedAccountCredentialsBuilder = - AzureNamedAccountCredentials.builder().name(name).credentials(azureCredentials); - if (!CollectionUtils.isEmpty(supportedTypes)) { if (supportedTypes.contains(AccountCredentials.Type.OBJECT_STORE)) { String container = azureManagedAccount.getContainer(); @@ -82,18 +73,10 @@ boolean registerAzureCredentials( throw new IllegalArgumentException( "Azure/Blobs account " + name + " is required to specify a rootFolder."); } - - azureNamedAccountCredentialsBuilder.rootFolder(rootFolder); - azureNamedAccountCredentialsBuilder.azureContainer( - azureCredentials.getAzureContainer(container)); + azureManagedAccount.setAzureContainer(getAzureContainer(azureManagedAccount)); } - - azureNamedAccountCredentialsBuilder.supportedTypes(supportedTypes); } - - AzureNamedAccountCredentials azureNamedAccountCredentials = - azureNamedAccountCredentialsBuilder.build(); - accountCredentialsRepository.save(name, azureNamedAccountCredentials); + accountCredentialsRepository.save(azureManagedAccount); } catch (Throwable t) { log.error("Could not load Azure account " + name + ".", t); } @@ -101,4 +84,32 @@ boolean registerAzureCredentials( return true; } + + public static CloudBlobContainer getAzureContainer(AzureManagedAccount account) throws Exception { + final String storageConnectionString = + "DefaultEndpointsProtocol=https;" + + "AccountName=" + + account.getStorageAccountName() + + ";" + + "AccountKey=" + + account.getAccountAccessKey() + + ";" + + "EndpointSuffix=" + + account.getEndpointSuffix(); + // Retrieve storage account from connection-string. + CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString); + + // Create the blob client. + CloudBlobClient blobClient = storageAccount.createCloudBlobClient(); + + // Get a reference to a container. + // The container name must be lower case + + CloudBlobContainer container = blobClient.getContainerReference(account.getContainer()); + + // Create the container if it does not exist. + container.createIfNotExists(); + + return container; + } } diff --git a/kayenta-azure/src/main/java/com/netflix/kayenta/azure/config/AzureManagedAccount.java b/kayenta-azure/src/main/java/com/netflix/kayenta/azure/config/AzureManagedAccount.java index d25d5028a..9594c05b6 100644 --- a/kayenta-azure/src/main/java/com/netflix/kayenta/azure/config/AzureManagedAccount.java +++ b/kayenta-azure/src/main/java/com/netflix/kayenta/azure/config/AzureManagedAccount.java @@ -16,15 +16,20 @@ package com.netflix.kayenta.azure.config; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.microsoft.azure.storage.blob.CloudBlobContainer; import com.netflix.kayenta.security.AccountCredentials; -import java.util.List; import javax.validation.constraints.NotNull; -import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.SuperBuilder; -@Data -public class AzureManagedAccount { - - @NotNull private String name; +@Getter +@Setter +@SuperBuilder +@NoArgsConstructor +public class AzureManagedAccount extends AccountCredentials { @NotNull private String storageAccountName; @@ -36,5 +41,10 @@ public class AzureManagedAccount { private String rootFolder; - private List supportedTypes; + @JsonIgnore private transient CloudBlobContainer azureContainer; + + @Override + public String getType() { + return "azure"; + } } diff --git a/kayenta-azure/src/main/java/com/netflix/kayenta/azure/security/AzureCredentials.java b/kayenta-azure/src/main/java/com/netflix/kayenta/azure/security/AzureCredentials.java deleted file mode 100644 index 88675af32..000000000 --- a/kayenta-azure/src/main/java/com/netflix/kayenta/azure/security/AzureCredentials.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright 2019 Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.kayenta.azure.security; - -import com.microsoft.azure.storage.*; -import com.microsoft.azure.storage.blob.*; -import lombok.Getter; -import lombok.ToString; -import lombok.extern.slf4j.Slf4j; - -@ToString -@Slf4j -public class AzureCredentials { - - @Getter private String storageAccountName; - - @Getter private String accountAccessKey; - - @Getter private String endpointSuffix; - - public AzureCredentials( - String storageAccountName, String accountAccessKey, String endpointSuffix) { - this.storageAccountName = storageAccountName; - this.accountAccessKey = accountAccessKey; - this.endpointSuffix = endpointSuffix; - } - - public CloudBlobContainer getAzureContainer(String containerName) throws Exception { - final String storageConnectionString = - "DefaultEndpointsProtocol=https;" - + "AccountName=" - + this.storageAccountName - + ";" - + "AccountKey=" - + this.accountAccessKey - + ";" - + "EndpointSuffix=" - + this.endpointSuffix; - // Retrieve storage account from connection-string. - CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString); - - // Create the blob client. - CloudBlobClient blobClient = storageAccount.createCloudBlobClient(); - - // Get a reference to a container. - // The container name must be lower case - - CloudBlobContainer container = blobClient.getContainerReference(containerName); - - // Create the container if it does not exist. - container.createIfNotExists(); - - return container; - } -} diff --git a/kayenta-azure/src/main/java/com/netflix/kayenta/azure/security/AzureNamedAccountCredentials.java b/kayenta-azure/src/main/java/com/netflix/kayenta/azure/security/AzureNamedAccountCredentials.java deleted file mode 100644 index 8e17eb3fc..000000000 --- a/kayenta-azure/src/main/java/com/netflix/kayenta/azure/security/AzureNamedAccountCredentials.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2019 Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.kayenta.azure.security; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.microsoft.azure.storage.blob.CloudBlobContainer; -import com.netflix.kayenta.security.AccountCredentials; -import javax.validation.constraints.NotNull; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.experimental.SuperBuilder; - -@SuperBuilder -@Getter -@Setter -@NoArgsConstructor -public class AzureNamedAccountCredentials extends AccountCredentials { - - @NotNull private AzureCredentials credentials; - - private String rootFolder; - - @Override - public String getType() { - return "azure"; - } - - @JsonIgnore private CloudBlobContainer azureContainer; -} diff --git a/kayenta-blobs/src/main/java/com/netflix/kayenta/blobs/config/BlobsConfiguration.java b/kayenta-blobs/src/main/java/com/netflix/kayenta/blobs/config/BlobsConfiguration.java index 81d6c70c8..2f6ba853a 100644 --- a/kayenta-blobs/src/main/java/com/netflix/kayenta/blobs/config/BlobsConfiguration.java +++ b/kayenta-blobs/src/main/java/com/netflix/kayenta/blobs/config/BlobsConfiguration.java @@ -16,7 +16,7 @@ package com.netflix.kayenta.blobs.config; -import com.netflix.kayenta.azure.security.AzureNamedAccountCredentials; +import com.netflix.kayenta.azure.config.AzureManagedAccount; import com.netflix.kayenta.blobs.storage.BlobsStorageService; import com.netflix.kayenta.security.AccountCredentials; import com.netflix.kayenta.security.AccountCredentialsRepository; @@ -43,7 +43,7 @@ public BlobsStorageService blobsStorageService( BlobsStorageService.builder(); accountCredentialsRepository.getAll().stream() - .filter(c -> c instanceof AzureNamedAccountCredentials) + .filter(c -> c instanceof AzureManagedAccount) .filter(c -> c.getSupportedTypes().contains(AccountCredentials.Type.OBJECT_STORE)) .map(c -> c.getName()) .forEach(blobsStorageServiceBuilder::accountName); diff --git a/kayenta-blobs/src/main/java/com/netflix/kayenta/blobs/storage/BlobsStorageService.java b/kayenta-blobs/src/main/java/com/netflix/kayenta/blobs/storage/BlobsStorageService.java index b37098e07..e6118d504 100644 --- a/kayenta-blobs/src/main/java/com/netflix/kayenta/blobs/storage/BlobsStorageService.java +++ b/kayenta-blobs/src/main/java/com/netflix/kayenta/blobs/storage/BlobsStorageService.java @@ -21,9 +21,12 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import com.microsoft.azure.storage.*; -import com.microsoft.azure.storage.blob.*; -import com.netflix.kayenta.azure.security.AzureNamedAccountCredentials; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.BlobProperties; +import com.microsoft.azure.storage.blob.CloudBlobContainer; +import com.microsoft.azure.storage.blob.CloudBlockBlob; +import com.microsoft.azure.storage.blob.ListBlobItem; +import com.netflix.kayenta.azure.config.AzureManagedAccount; import com.netflix.kayenta.canary.CanaryConfig; import com.netflix.kayenta.index.CanaryConfigIndex; import com.netflix.kayenta.index.config.CanaryConfigIndexAction; @@ -63,8 +66,7 @@ public boolean servicesAccount(String accountName) { @Override public T loadObject(String accountName, ObjectType objectType, String objectKey) throws IllegalArgumentException, NotFoundException { - AzureNamedAccountCredentials credentials = - accountCredentialsRepository.getRequiredOne(accountName); + AzureManagedAccount credentials = accountCredentialsRepository.getRequiredOne(accountName); CloudBlobContainer azureContainer = credentials.getAzureContainer(); CloudBlockBlob blobItem; try { @@ -83,7 +85,7 @@ public T loadObject(String accountName, ObjectType objectType, String object private CloudBlockBlob resolveSingularBlob( ObjectType objectType, String objectKey, - AzureNamedAccountCredentials credentials, + AzureManagedAccount credentials, CloudBlobContainer azureContainer) { String rootFolder = daoRoot(credentials, objectType.getGroup()) + "/" + objectKey; @@ -135,8 +137,7 @@ public void storeObject( T obj, String filename, boolean isAnUpdate) { - AzureNamedAccountCredentials credentials = - accountCredentialsRepository.getRequiredOne(accountName); + AzureManagedAccount credentials = accountCredentialsRepository.getRequiredOne(accountName); CloudBlobContainer azureContainer = credentials.getAzureContainer(); String path = keyToPath(credentials, objectType, objectKey, filename); @@ -226,7 +227,7 @@ public void storeObject( } private void checkForDuplicateCanaryConfig( - CanaryConfig canaryConfig, String canaryConfigId, AzureNamedAccountCredentials credentials) { + CanaryConfig canaryConfig, String canaryConfigId, AzureManagedAccount credentials) { String canaryConfigName = canaryConfig.getName(); List applications = canaryConfig.getApplications(); String existingCanaryConfigId = @@ -247,8 +248,7 @@ private void checkForDuplicateCanaryConfig( @Override public void deleteObject(String accountName, ObjectType objectType, String objectKey) { - AzureNamedAccountCredentials credentials = - accountCredentialsRepository.getRequiredOne(accountName); + AzureManagedAccount credentials = accountCredentialsRepository.getRequiredOne(accountName); CloudBlobContainer azureContainer = credentials.getAzureContainer(); CloudBlockBlob item = resolveSingularBlob(objectType, objectKey, credentials, azureContainer); @@ -316,8 +316,7 @@ public void deleteObject(String accountName, ObjectType objectType, String objec @Override public List> listObjectKeys( String accountName, ObjectType objectType, List applications, boolean skipIndex) { - AzureNamedAccountCredentials credentials = - accountCredentialsRepository.getRequiredOne(accountName); + AzureManagedAccount credentials = accountCredentialsRepository.getRequiredOne(accountName); if (!skipIndex && objectType == ObjectType.CANARY_CONFIG) { Set> canaryConfigSet = @@ -373,7 +372,7 @@ public List> listObjectKeys( } } - private String daoRoot(AzureNamedAccountCredentials credentials, String daoTypeName) { + private String daoRoot(AzureManagedAccount credentials, String daoTypeName) { return credentials.getRootFolder() + '/' + daoTypeName; } @@ -384,10 +383,7 @@ private T deserialize(CloudBlockBlob blob, TypeReference typeReference) } private String keyToPath( - AzureNamedAccountCredentials credentials, - ObjectType objectType, - String objectKey, - String filename) { + AzureManagedAccount credentials, ObjectType objectType, String objectKey, String filename) { if (filename == null) { filename = objectType.getDefaultFilename(); } diff --git a/kayenta-blobs/src/test/java/com/netflix/kayenta/blobs/storage/TestableBlobsStorageServiceTest.java b/kayenta-blobs/src/test/java/com/netflix/kayenta/blobs/storage/TestableBlobsStorageServiceTest.java index 8b76ca7da..633da6dc7 100644 --- a/kayenta-blobs/src/test/java/com/netflix/kayenta/blobs/storage/TestableBlobsStorageServiceTest.java +++ b/kayenta-blobs/src/test/java/com/netflix/kayenta/blobs/storage/TestableBlobsStorageServiceTest.java @@ -19,8 +19,7 @@ import static org.mockito.Mockito.*; import com.fasterxml.jackson.databind.ObjectMapper; -import com.netflix.kayenta.azure.security.AzureCredentials; -import com.netflix.kayenta.azure.security.AzureNamedAccountCredentials; +import com.netflix.kayenta.azure.config.AzureManagedAccount; import com.netflix.kayenta.canary.CanaryConfig; import com.netflix.kayenta.index.CanaryConfigIndex; import com.netflix.kayenta.security.AccountCredentials; @@ -28,14 +27,16 @@ import com.netflix.kayenta.security.MapBackedAccountCredentialsRepository; import com.netflix.kayenta.storage.ObjectType; import com.netflix.spinnaker.kork.web.exceptions.NotFoundException; -import com.tngtech.java.junit.dataprovider.*; +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; import java.util.*; import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.junit.runner.*; +import org.junit.runner.RunWith; import org.mockito.Mockito; @RunWith(DataProviderRunner.class) @@ -56,17 +57,18 @@ public void setUp() { String azureAccountName = "AzDev_Testing_Account_1"; String accountAccessKey = "testAccessKey"; - AzureNamedAccountCredentials.AzureNamedAccountCredentialsBuilder credentialsBuilder = - AzureNamedAccountCredentials.builder(); - credentialsBuilder.name(kayenataAccountName); - credentialsBuilder.credentials( - new AzureCredentials(azureAccountName, accountAccessKey, "core.windows.net")); - credentialsBuilder.rootFolder(rootFolder); - credentialsBuilder.azureContainer(null); - accountCredentials = credentialsBuilder.build(); + accountCredentials = + AzureManagedAccount.builder() + .name(kayenataAccountName) + .storageAccountName(azureAccountName) + .accountAccessKey(accountAccessKey) + .rootFolder(rootFolder) + .azureContainer(null) + .endpointSuffix("core.windows.net") + .build(); credentialsRepository = new MapBackedAccountCredentialsRepository(); - credentialsRepository.save(kayenataAccountName, accountCredentials); + credentialsRepository.save(accountCredentials); ObjectMapper kayentaObjectMapper = new ObjectMapper(); this.mockedCanaryConfigIndex = mock(CanaryConfigIndex.class); diff --git a/kayenta-core/src/main/java/com/netflix/kayenta/security/AccountCredentials.java b/kayenta-core/src/main/java/com/netflix/kayenta/security/AccountCredentials.java index d6b1255a0..aac11cc5a 100644 --- a/kayenta-core/src/main/java/com/netflix/kayenta/security/AccountCredentials.java +++ b/kayenta-core/src/main/java/com/netflix/kayenta/security/AccountCredentials.java @@ -31,7 +31,8 @@ @EqualsAndHashCode(of = "name") public abstract class AccountCredentials { private String name; - @Builder.Default private List supportedTypes = new ArrayList<>(); + /** Account types other than metric accounts are going away. */ + @Deprecated @Builder.Default private List supportedTypes = new ArrayList<>(); public abstract String getType(); @@ -52,7 +53,10 @@ public abstract class AccountCredentials { @Builder.Default private List recommendedLocations = Collections.emptyList(); @JsonIgnore - public abstract T getCredentials(); + public T getCredentials() { + return (T) this; + } + ; public enum Type { METRICS_STORE, diff --git a/kayenta-core/src/main/java/com/netflix/kayenta/security/AccountCredentialsRepository.java b/kayenta-core/src/main/java/com/netflix/kayenta/security/AccountCredentialsRepository.java index 076d9c6c0..e7e2835bd 100644 --- a/kayenta-core/src/main/java/com/netflix/kayenta/security/AccountCredentialsRepository.java +++ b/kayenta-core/src/main/java/com/netflix/kayenta/security/AccountCredentialsRepository.java @@ -55,5 +55,10 @@ default Set getAllOf(AccountCredentials.Type credentialsType Set getAll(); + @Deprecated AccountCredentials save(String name, AccountCredentials credentials); + + default AccountCredentials save(AccountCredentials credentials) { + return save(credentials.getName(), credentials); + } } diff --git a/kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/config/DatadogConfiguration.java b/kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/config/DatadogConfiguration.java index ec58253a8..12c7f78af 100644 --- a/kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/config/DatadogConfiguration.java +++ b/kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/config/DatadogConfiguration.java @@ -18,8 +18,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.netflix.kayenta.datadog.metrics.DatadogMetricsService; -import com.netflix.kayenta.datadog.security.DatadogCredentials; -import com.netflix.kayenta.datadog.security.DatadogNamedAccountCredentials; import com.netflix.kayenta.datadog.service.DatadogRemoteService; import com.netflix.kayenta.metrics.MetricsService; import com.netflix.kayenta.retrofit.config.RemoteService; @@ -69,33 +67,18 @@ MetricsService datadogMetricsService( DatadogMetricsService.builder(); for (DatadogManagedAccount account : datadogConfigurationProperties.getAccounts()) { - String name = account.getName(); List supportedTypes = account.getSupportedTypes(); - DatadogCredentials credentials = - DatadogCredentials.builder() - .apiKey(account.getApiKey()) - .applicationKey(account.getApplicationKey()) - .build(); - - DatadogNamedAccountCredentials.DatadogNamedAccountCredentialsBuilder - accountCredentialsBuilder = - DatadogNamedAccountCredentials.builder() - .name(name) - .endpoint(account.getEndpoint()) - .credentials(credentials); - if (!CollectionUtils.isEmpty(supportedTypes)) { if (supportedTypes.contains(AccountCredentials.Type.METRICS_STORE)) { - accountCredentialsBuilder.datadogRemoteService( + account.setDatadogRemoteService( createDatadogRemoteService( retrofitClientFactory, objectMapper, account.getEndpoint(), okHttpClient)); } - accountCredentialsBuilder.supportedTypes(supportedTypes); } - accountCredentialsRepository.save(name, accountCredentialsBuilder.build()); - metricsServiceBuilder.accountName(name); + accountCredentialsRepository.save(account); + metricsServiceBuilder.accountName(account.getName()); } log.info( diff --git a/kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/config/DatadogManagedAccount.java b/kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/config/DatadogManagedAccount.java index b7b2696aa..4e2548ddf 100644 --- a/kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/config/DatadogManagedAccount.java +++ b/kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/config/DatadogManagedAccount.java @@ -16,21 +16,39 @@ package com.netflix.kayenta.datadog.config; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.netflix.kayenta.datadog.service.DatadogRemoteService; import com.netflix.kayenta.retrofit.config.RemoteService; import com.netflix.kayenta.security.AccountCredentials; import java.util.Collections; import java.util.List; import javax.validation.constraints.NotNull; -import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.SuperBuilder; -@Data -public class DatadogManagedAccount { - @NotNull private String name; - private String apiKey; - private String applicationKey; +@Getter +@Setter +@SuperBuilder +@NoArgsConstructor +public class DatadogManagedAccount extends AccountCredentials { + private static final List SUPPORTED_TYPES = + Collections.singletonList(AccountCredentials.Type.METRICS_STORE); + @JsonIgnore private String apiKey; + @JsonIgnore private String applicationKey; @NotNull private RemoteService endpoint; - private List supportedTypes = - Collections.singletonList(AccountCredentials.Type.METRICS_STORE); + @Override + public String getType() { + return "datadog"; + } + + @Override + public List getSupportedTypes() { + return SUPPORTED_TYPES; + } + + @JsonIgnore private transient DatadogRemoteService datadogRemoteService; } diff --git a/kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/metrics/DatadogMetricsService.java b/kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/metrics/DatadogMetricsService.java index 1561b122d..874d8610c 100644 --- a/kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/metrics/DatadogMetricsService.java +++ b/kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/metrics/DatadogMetricsService.java @@ -21,15 +21,13 @@ import com.netflix.kayenta.canary.CanaryScope; import com.netflix.kayenta.canary.providers.metrics.DatadogCanaryMetricSetQueryConfig; import com.netflix.kayenta.canary.providers.metrics.QueryConfigUtils; -import com.netflix.kayenta.datadog.security.DatadogCredentials; -import com.netflix.kayenta.datadog.security.DatadogNamedAccountCredentials; +import com.netflix.kayenta.datadog.config.DatadogManagedAccount; import com.netflix.kayenta.datadog.service.DatadogRemoteService; import com.netflix.kayenta.datadog.service.DatadogTimeSeries; import com.netflix.kayenta.metrics.MetricSet; import com.netflix.kayenta.metrics.MetricsService; import com.netflix.kayenta.model.DatadogMetricDescriptor; import com.netflix.kayenta.model.DatadogMetricDescriptorsResponse; -import com.netflix.kayenta.security.AccountCredentials; import com.netflix.kayenta.security.AccountCredentialsRepository; import com.netflix.spectator.api.Registry; import java.io.IOException; @@ -38,7 +36,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; import javax.validation.constraints.NotNull; import lombok.Builder; @@ -101,10 +98,9 @@ public List queryMetrics( CanaryMetricConfig canaryMetricConfig, CanaryScope canaryScope) throws IOException { - DatadogNamedAccountCredentials accountCredentials = + DatadogManagedAccount accountCredentials = accountCredentialsRepository.getRequiredOne(accountName); - DatadogCredentials credentials = accountCredentials.getCredentials(); DatadogRemoteService remoteService = accountCredentials.getDatadogRemoteService(); if (StringUtils.isEmpty(canaryScope.getStart())) { @@ -118,8 +114,8 @@ public List queryMetrics( String query = buildQuery(accountName, canaryConfig, canaryMetricConfig, canaryScope); DatadogTimeSeries timeSeries = remoteService.getTimeSeries( - credentials.getApiKey(), - credentials.getApplicationKey(), + accountCredentials.getApiKey(), + accountCredentials.getApplicationKey(), (int) canaryScope.getStart().getEpochSecond(), (int) canaryScope.getEnd().getEpochSecond(), query); @@ -178,47 +174,47 @@ public List getMetadata(String metricsAccountName, String filter) { @Scheduled(fixedDelayString = "#{@datadogConfigurationProperties.metadataCachingIntervalMS}") public void updateMetricDescriptorsCache() { - Set accountCredentialsSet = - accountCredentialsRepository.getAllOf(AccountCredentials.Type.METRICS_STORE); - - for (AccountCredentials credentials : accountCredentialsSet) { - if (credentials instanceof DatadogNamedAccountCredentials) { - DatadogNamedAccountCredentials datadogCredentials = - (DatadogNamedAccountCredentials) credentials; - DatadogRemoteService datadogRemoteService = datadogCredentials.getDatadogRemoteService(); - DatadogCredentials ddCredentials = datadogCredentials.getCredentials(); - // Retrieve all metrics actively reporting in the last hour. - long from = Instant.now().getEpochSecond() - 60 * 60; - DatadogMetricDescriptorsResponse datadogMetricDescriptorsResponse = - datadogRemoteService.getMetrics( - ddCredentials.getApiKey(), ddCredentials.getApplicationKey(), from); - - if (datadogMetricDescriptorsResponse != null) { - List metrics = datadogMetricDescriptorsResponse.getMetrics(); - - if (!CollectionUtils.isEmpty(metrics)) { - // TODO(duftler): Should we instead be building the union across all accounts? This - // doesn't seem quite right yet. - metricDescriptorsCache = - metrics.stream() - .map(metricName -> new DatadogMetricDescriptor(metricName)) - .collect(Collectors.toList()); - - log.debug( - "Updated cache with {} metric descriptors via account {}.", - metricDescriptorsCache.size(), - datadogCredentials.getName()); - } else { - log.debug( - "While updating cache, found no metric descriptors via account {}.", - datadogCredentials.getName()); - } - } else { - log.debug( - "While updating cache, found no metric descriptors via account {}.", - datadogCredentials.getName()); - } - } - } + accountCredentialsRepository.getAll().stream() + // .filter(each -> + // each.getSupportedTypes().contains(AccountCredentials.Type.METRICS_STORE)) // ONLY metrics + // stores for datadog accounts... + .filter(DatadogManagedAccount.class::isInstance) + .map(DatadogManagedAccount.class::cast) + .forEach( + credentials -> { + DatadogRemoteService datadogRemoteService = credentials.getDatadogRemoteService(); + // Retrieve all metrics actively reporting in the last hour. + long from = Instant.now().getEpochSecond() - 60 * 60; + DatadogMetricDescriptorsResponse datadogMetricDescriptorsResponse = + datadogRemoteService.getMetrics( + credentials.getApiKey(), credentials.getApplicationKey(), from); + + if (datadogMetricDescriptorsResponse != null) { + List metrics = datadogMetricDescriptorsResponse.getMetrics(); + + if (!CollectionUtils.isEmpty(metrics)) { + // TODO(duftler): Should we instead be building the union across all accounts? + // This + // doesn't seem quite right yet. + metricDescriptorsCache = + metrics.stream() + .map(metricName -> new DatadogMetricDescriptor(metricName)) + .collect(Collectors.toList()); + + log.debug( + "Updated cache with {} metric descriptors via account {}.", + metricDescriptorsCache.size(), + credentials.getName()); + } else { + log.debug( + "While updating cache, found no metric descriptors via account {}.", + credentials.getName()); + } + } else { + log.debug( + "While updating cache, found no metric descriptors via account {}.", + credentials.getName()); + } + }); } } diff --git a/kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/security/DatadogCredentials.java b/kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/security/DatadogCredentials.java deleted file mode 100644 index d44eeb6c4..000000000 --- a/kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/security/DatadogCredentials.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2018 Armory, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.kayenta.datadog.security; - -import java.util.Optional; -import lombok.Builder; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; - -@Builder -@Data -@Slf4j -public class DatadogCredentials { - private static String applicationVersion = - Optional.ofNullable(DatadogCredentials.class.getPackage().getImplementationVersion()) - .orElse("Unknown"); - - private String apiKey; - private String applicationKey; -} diff --git a/kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/security/DatadogNamedAccountCredentials.java b/kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/security/DatadogNamedAccountCredentials.java deleted file mode 100644 index 449e745f1..000000000 --- a/kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/security/DatadogNamedAccountCredentials.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2018 Armory, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.kayenta.datadog.security; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.netflix.kayenta.datadog.service.DatadogRemoteService; -import com.netflix.kayenta.retrofit.config.RemoteService; -import com.netflix.kayenta.security.AccountCredentials; -import javax.validation.constraints.NotNull; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.experimental.SuperBuilder; - -@SuperBuilder -@Getter -@Setter -@NoArgsConstructor -public class DatadogNamedAccountCredentials extends AccountCredentials { - - @NotNull private DatadogCredentials credentials; - - @NotNull private RemoteService endpoint; - - @Override - public String getType() { - return "datadog"; - } - - @JsonIgnore DatadogRemoteService datadogRemoteService; -} diff --git a/kayenta-gcs/src/main/java/com/netflix/kayenta/gcs/config/GcsConfiguration.java b/kayenta-gcs/src/main/java/com/netflix/kayenta/gcs/config/GcsConfiguration.java index 6f1ed8e7e..ad1c15858 100644 --- a/kayenta-gcs/src/main/java/com/netflix/kayenta/gcs/config/GcsConfiguration.java +++ b/kayenta-gcs/src/main/java/com/netflix/kayenta/gcs/config/GcsConfiguration.java @@ -17,7 +17,7 @@ package com.netflix.kayenta.gcs.config; import com.netflix.kayenta.gcs.storage.GcsStorageService; -import com.netflix.kayenta.google.security.GoogleNamedAccountCredentials; +import com.netflix.kayenta.google.config.GoogleManagedAccount; import com.netflix.kayenta.security.AccountCredentials; import com.netflix.kayenta.security.AccountCredentialsRepository; import lombok.extern.slf4j.Slf4j; @@ -41,7 +41,7 @@ public GcsStorageService gcsStorageService( GcsStorageService.builder(); accountCredentialsRepository.getAll().stream() - .filter(c -> c instanceof GoogleNamedAccountCredentials) + .filter(GoogleManagedAccount.class::isInstance) .filter(c -> c.getSupportedTypes().contains(AccountCredentials.Type.OBJECT_STORE)) .map(c -> c.getName()) .forEach(gcsStorageServiceBuilder::accountName); diff --git a/kayenta-gcs/src/main/java/com/netflix/kayenta/gcs/storage/GcsStorageService.java b/kayenta-gcs/src/main/java/com/netflix/kayenta/gcs/storage/GcsStorageService.java index 7445c9ff4..342f7ab09 100644 --- a/kayenta-gcs/src/main/java/com/netflix/kayenta/gcs/storage/GcsStorageService.java +++ b/kayenta-gcs/src/main/java/com/netflix/kayenta/gcs/storage/GcsStorageService.java @@ -28,7 +28,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.netflix.kayenta.canary.CanaryConfig; -import com.netflix.kayenta.google.security.GoogleNamedAccountCredentials; +import com.netflix.kayenta.google.config.GoogleManagedAccount; import com.netflix.kayenta.index.CanaryConfigIndex; import com.netflix.kayenta.index.config.CanaryConfigIndexAction; import com.netflix.kayenta.security.AccountCredentialsRepository; @@ -66,8 +66,7 @@ public boolean servicesAccount(String accountName) { /** Check to see if the bucket exists, creating it if it is not there. */ public void ensureBucketExists(String accountName) { - GoogleNamedAccountCredentials credentials = - accountCredentialsRepository.getRequiredOne(accountName); + GoogleManagedAccount credentials = accountCredentialsRepository.getRequiredOne(accountName); Storage storage = credentials.getStorage(); String projectName = credentials.getProject(); String bucketName = credentials.getBucket(); @@ -113,8 +112,7 @@ public void ensureBucketExists(String accountName) { @Override public T loadObject(String accountName, ObjectType objectType, String objectKey) throws IllegalArgumentException, NotFoundException { - GoogleNamedAccountCredentials credentials = - accountCredentialsRepository.getRequiredOne(accountName); + GoogleManagedAccount credentials = accountCredentialsRepository.getRequiredOne(accountName); Storage storage = credentials.getStorage(); String bucketName = credentials.getBucket(); StorageObject item; @@ -148,7 +146,7 @@ public T loadObject(String accountName, ObjectType objectType, String object private StorageObject resolveSingularItem( ObjectType objectType, String objectKey, - GoogleNamedAccountCredentials credentials, + GoogleManagedAccount credentials, Storage storage, String bucketName) { String rootFolder = daoRoot(credentials, objectType.getGroup()) + "/" + objectKey; @@ -194,8 +192,7 @@ public void storeObject( T obj, String filename, boolean isAnUpdate) { - GoogleNamedAccountCredentials credentials = - accountCredentialsRepository.getRequiredOne(accountName); + GoogleManagedAccount credentials = accountCredentialsRepository.getRequiredOne(accountName); Storage storage = credentials.getStorage(); String bucketName = credentials.getBucket(); String path = keyToPath(credentials, objectType, objectKey, filename); @@ -279,7 +276,7 @@ public void storeObject( } private void checkForDuplicateCanaryConfig( - CanaryConfig canaryConfig, String canaryConfigId, GoogleNamedAccountCredentials credentials) { + CanaryConfig canaryConfig, String canaryConfigId, GoogleManagedAccount credentials) { String canaryConfigName = canaryConfig.getName(); List applications = canaryConfig.getApplications(); String existingCanaryConfigId = @@ -300,8 +297,7 @@ private void checkForDuplicateCanaryConfig( @Override public void deleteObject(String accountName, ObjectType objectType, String objectKey) { - GoogleNamedAccountCredentials credentials = - accountCredentialsRepository.getRequiredOne(accountName); + GoogleManagedAccount credentials = accountCredentialsRepository.getRequiredOne(accountName); Storage storage = credentials.getStorage(); String bucketName = credentials.getBucket(); StorageObject item = @@ -394,8 +390,7 @@ public void deleteObject(String accountName, ObjectType objectType, String objec @Override public List> listObjectKeys( String accountName, ObjectType objectType, List applications, boolean skipIndex) { - GoogleNamedAccountCredentials credentials = - accountCredentialsRepository.getRequiredOne(accountName); + GoogleManagedAccount credentials = accountCredentialsRepository.getRequiredOne(accountName); if (!skipIndex && objectType == ObjectType.CANARY_CONFIG) { Set> canaryConfigSet = @@ -458,15 +453,12 @@ public List> listObjectKeys( } } - private String daoRoot(GoogleNamedAccountCredentials credentials, String daoTypeName) { + private String daoRoot(GoogleManagedAccount credentials, String daoTypeName) { return credentials.getRootFolder() + '/' + daoTypeName; } private String keyToPath( - GoogleNamedAccountCredentials credentials, - ObjectType objectType, - String objectKey, - String filename) { + GoogleManagedAccount credentials, ObjectType objectType, String objectKey, String filename) { if (filename == null) { filename = objectType.getDefaultFilename(); } diff --git a/kayenta-google/src/main/java/com/netflix/kayenta/google/config/GoogleConfiguration.java b/kayenta-google/src/main/java/com/netflix/kayenta/google/config/GoogleConfiguration.java index 43e8e45b1..601143f15 100644 --- a/kayenta-google/src/main/java/com/netflix/kayenta/google/config/GoogleConfiguration.java +++ b/kayenta-google/src/main/java/com/netflix/kayenta/google/config/GoogleConfiguration.java @@ -18,7 +18,6 @@ import com.netflix.kayenta.google.security.GoogleClientFactory; import com.netflix.kayenta.google.security.GoogleJsonClientFactory; -import com.netflix.kayenta.google.security.GoogleNamedAccountCredentials; import com.netflix.kayenta.security.AccountCredentials; import com.netflix.kayenta.security.AccountCredentialsRepository; import java.io.IOException; @@ -67,16 +66,9 @@ boolean registerGoogleCredentials( ? new GoogleJsonClientFactory(project, jsonKey) : new GoogleClientFactory(project); - GoogleNamedAccountCredentials.GoogleNamedAccountCredentialsBuilder - googleNamedAccountCredentialsBuilder = - GoogleNamedAccountCredentials.builder() - .name(name) - .project(project) - .credentials(googleClientFactory); - if (!CollectionUtils.isEmpty(supportedTypes)) { if (supportedTypes.contains(AccountCredentials.Type.METRICS_STORE)) { - googleNamedAccountCredentialsBuilder.monitoring(googleClientFactory.getMonitoring()); + googleManagedAccount.setMonitoring(googleClientFactory.getMonitoring()); } if (supportedTypes.contains(AccountCredentials.Type.OBJECT_STORE)) { @@ -93,19 +85,11 @@ boolean registerGoogleCredentials( "Google/GCS account " + name + " is required to specify a rootFolder."); } - googleNamedAccountCredentialsBuilder.bucket(bucket); - googleNamedAccountCredentialsBuilder.bucketLocation( - googleManagedAccount.getBucketLocation()); - googleNamedAccountCredentialsBuilder.rootFolder(rootFolder); - googleNamedAccountCredentialsBuilder.storage(googleClientFactory.getStorage()); + googleManagedAccount.setStorage(googleClientFactory.getStorage()); } - - googleNamedAccountCredentialsBuilder.supportedTypes(supportedTypes); } - GoogleNamedAccountCredentials googleNamedAccountCredentials = - googleNamedAccountCredentialsBuilder.build(); - accountCredentialsRepository.save(name, googleNamedAccountCredentials); + accountCredentialsRepository.save(googleManagedAccount); } catch (Throwable t) { log.error("Could not load Google account " + name + ".", t); } diff --git a/kayenta-google/src/main/java/com/netflix/kayenta/google/config/GoogleManagedAccount.java b/kayenta-google/src/main/java/com/netflix/kayenta/google/config/GoogleManagedAccount.java index c62421159..237d9f7ff 100644 --- a/kayenta-google/src/main/java/com/netflix/kayenta/google/config/GoogleManagedAccount.java +++ b/kayenta-google/src/main/java/com/netflix/kayenta/google/config/GoogleManagedAccount.java @@ -16,19 +16,19 @@ package com.netflix.kayenta.google.config; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.api.services.monitoring.v3.Monitoring; +import com.google.api.services.storage.Storage; import com.google.common.base.Charsets; import com.google.common.io.CharStreams; import com.netflix.kayenta.security.AccountCredentials; import java.io.*; -import java.util.List; import javax.validation.constraints.NotNull; import lombok.Data; import org.springframework.util.StringUtils; @Data -public class GoogleManagedAccount { - - @NotNull private String name; +public class GoogleManagedAccount extends AccountCredentials { @NotNull private String project; private String jsonPath; @@ -37,7 +37,10 @@ public class GoogleManagedAccount { private String bucketLocation; private String rootFolder = "kayenta"; - private List supportedTypes; + @Override + public String getType() { + return "google"; + } private InputStream getInputStream() throws FileNotFoundException { if (StringUtils.hasLength(jsonPath)) { @@ -58,4 +61,8 @@ public String getJsonKey() throws IOException { ? CharStreams.toString(new InputStreamReader(inputStream, Charsets.UTF_8)) : null; } + + @JsonIgnore private transient Monitoring monitoring; + + @JsonIgnore private transient Storage storage; } diff --git a/kayenta-google/src/main/java/com/netflix/kayenta/google/security/GoogleNamedAccountCredentials.java b/kayenta-google/src/main/java/com/netflix/kayenta/google/security/GoogleNamedAccountCredentials.java deleted file mode 100644 index 896e3999d..000000000 --- a/kayenta-google/src/main/java/com/netflix/kayenta/google/security/GoogleNamedAccountCredentials.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2017 Google, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.kayenta.google.security; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.api.services.monitoring.v3.Monitoring; -import com.google.api.services.storage.Storage; -import com.netflix.kayenta.security.AccountCredentials; -import javax.validation.constraints.NotNull; -import lombok.*; -import lombok.experimental.SuperBuilder; - -@Getter -@Setter -@SuperBuilder -@NoArgsConstructor -public class GoogleNamedAccountCredentials extends AccountCredentials { - - @NotNull private GoogleClientFactory credentials; - - @NotNull private String project; - - private String bucket; - private String bucketLocation; - private String rootFolder; - - @Override - public String getType() { - return "google"; - } - - public String getMetricsStoreType() { - return getSupportedTypes().contains(Type.METRICS_STORE) ? "stackdriver" : null; - } - - @JsonIgnore private Monitoring monitoring; - - @JsonIgnore private Storage storage; -} diff --git a/kayenta-prometheus/src/main/java/com/netflix/kayenta/prometheus/config/PrometheusManagedAccount.java b/kayenta-prometheus/src/main/java/com/netflix/kayenta/prometheus/config/PrometheusManagedAccount.java index daf5ce7f4..d04285360 100644 --- a/kayenta-prometheus/src/main/java/com/netflix/kayenta/prometheus/config/PrometheusManagedAccount.java +++ b/kayenta-prometheus/src/main/java/com/netflix/kayenta/prometheus/config/PrometheusManagedAccount.java @@ -31,7 +31,8 @@ @SuperBuilder @NoArgsConstructor public class PrometheusManagedAccount extends AccountCredentials { - + private static final List SUPPORTED_TYPES = + Collections.singletonList(AccountCredentials.Type.METRICS_STORE); // Location of prometheus server. @NotNull private RemoteService endpoint; @@ -44,8 +45,9 @@ public class PrometheusManagedAccount extends AccountCredentials getSupportedTypes() { - return Collections.singletonList(AccountCredentials.Type.METRICS_STORE); + @Override + public List getSupportedTypes() { + return SUPPORTED_TYPES; } // Optional parameter for use when protecting prometheus with bearer token. @@ -56,9 +58,4 @@ public List getSupportedTypes() { public String getType() { return "prometheus"; } - - @Override - public PrometheusManagedAccount getCredentials() { - return this; - } } diff --git a/kayenta-s3/src/main/java/com/netflix/kayenta/s3/config/S3Configuration.java b/kayenta-s3/src/main/java/com/netflix/kayenta/s3/config/S3Configuration.java index 8d1bf879b..47e145a19 100644 --- a/kayenta-s3/src/main/java/com/netflix/kayenta/s3/config/S3Configuration.java +++ b/kayenta-s3/src/main/java/com/netflix/kayenta/s3/config/S3Configuration.java @@ -17,7 +17,8 @@ package com.netflix.kayenta.s3.config; import com.fasterxml.jackson.databind.ObjectMapper; -import com.netflix.kayenta.aws.security.AwsNamedAccountCredentials; +import com.netflix.kayenta.aws.config.AwsManagedAccount; +import com.netflix.kayenta.index.CanaryConfigIndex; import com.netflix.kayenta.s3.storage.S3StorageService; import com.netflix.kayenta.security.AccountCredentials; import com.netflix.kayenta.security.AccountCredentialsRepository; @@ -40,15 +41,18 @@ public class S3Configuration { @Bean @DependsOn({"registerAwsCredentials"}) public S3StorageService s3StorageService( - AccountCredentialsRepository accountCredentialsRepository) { + AccountCredentialsRepository accountCredentialsRepository, CanaryConfigIndex configIndex) { S3StorageService.S3StorageServiceBuilder s3StorageServiceBuilder = S3StorageService.builder(); accountCredentialsRepository.getAll().stream() - .filter(c -> c instanceof AwsNamedAccountCredentials) + .filter(AwsManagedAccount.class::isInstance) .filter(c -> c.getSupportedTypes().contains(AccountCredentials.Type.OBJECT_STORE)) .map(c -> c.getName()) .forEach(s3StorageServiceBuilder::accountName); + s3StorageServiceBuilder.accountCredentialsRepository(accountCredentialsRepository); + s3StorageServiceBuilder.canaryConfigIndex(configIndex); + S3StorageService s3StorageService = s3StorageServiceBuilder.objectMapper(kayentaObjectMapper).build(); diff --git a/kayenta-s3/src/main/java/com/netflix/kayenta/s3/storage/S3StorageService.java b/kayenta-s3/src/main/java/com/netflix/kayenta/s3/storage/S3StorageService.java index eb36d8000..f4ef42f9c 100644 --- a/kayenta-s3/src/main/java/com/netflix/kayenta/s3/storage/S3StorageService.java +++ b/kayenta-s3/src/main/java/com/netflix/kayenta/s3/storage/S3StorageService.java @@ -24,7 +24,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import com.netflix.kayenta.aws.security.AwsNamedAccountCredentials; +import com.netflix.kayenta.aws.config.AwsManagedAccount; import com.netflix.kayenta.canary.CanaryConfig; import com.netflix.kayenta.index.CanaryConfigIndex; import com.netflix.kayenta.index.config.CanaryConfigIndexAction; @@ -70,8 +70,7 @@ public boolean servicesAccount(String accountName) { /** Check to see if the bucket exists, creating it if it is not there. */ public void ensureBucketExists(String accountName) { - AwsNamedAccountCredentials credentials = - accountCredentialsRepository.getRequiredOne(accountName); + AwsManagedAccount credentials = accountCredentialsRepository.getRequiredOne(accountName); AmazonS3 amazonS3 = credentials.getAmazonS3(); String bucket = credentials.getBucket(); @@ -100,8 +99,7 @@ public void ensureBucketExists(String accountName) { @Override public T loadObject(String accountName, ObjectType objectType, String objectKey) throws IllegalArgumentException, NotFoundException { - AwsNamedAccountCredentials credentials = - accountCredentialsRepository.getRequiredOne(accountName); + AwsManagedAccount credentials = accountCredentialsRepository.getRequiredOne(accountName); AmazonS3 amazonS3 = credentials.getAmazonS3(); String bucket = credentials.getBucket(); String path; @@ -130,7 +128,7 @@ public T loadObject(String accountName, ObjectType objectType, String object private String resolveSingularPath( ObjectType objectType, String objectKey, - AwsNamedAccountCredentials credentials, + AwsManagedAccount credentials, AmazonS3 amazonS3, String bucket) { String rootFolder = daoRoot(credentials, objectType.getGroup()) + "/" + objectKey; @@ -164,8 +162,7 @@ public void storeObject( T obj, String filename, boolean isAnUpdate) { - AwsNamedAccountCredentials credentials = - accountCredentialsRepository.getRequiredOne(accountName); + AwsManagedAccount credentials = accountCredentialsRepository.getRequiredOne(accountName); AmazonS3 amazonS3 = credentials.getAmazonS3(); String bucket = credentials.getBucket(); String group = objectType.getGroup(); @@ -260,7 +257,7 @@ public void storeObject( } private void checkForDuplicateCanaryConfig( - CanaryConfig canaryConfig, String canaryConfigId, AwsNamedAccountCredentials credentials) { + CanaryConfig canaryConfig, String canaryConfigId, AwsManagedAccount credentials) { String canaryConfigName = canaryConfig.getName(); List applications = canaryConfig.getApplications(); String existingCanaryConfigId = @@ -281,8 +278,7 @@ private void checkForDuplicateCanaryConfig( @Override public void deleteObject(String accountName, ObjectType objectType, String objectKey) { - AwsNamedAccountCredentials credentials = - accountCredentialsRepository.getRequiredOne(accountName); + AwsManagedAccount credentials = accountCredentialsRepository.getRequiredOne(accountName); AmazonS3 amazonS3 = credentials.getAmazonS3(); String bucket = credentials.getBucket(); String path = resolveSingularPath(objectType, objectKey, credentials, amazonS3, bucket); @@ -354,8 +350,7 @@ public void deleteObject(String accountName, ObjectType objectType, String objec @Override public List> listObjectKeys( String accountName, ObjectType objectType, List applications, boolean skipIndex) { - AwsNamedAccountCredentials credentials = - accountCredentialsRepository.getRequiredOne(accountName); + AwsManagedAccount credentials = accountCredentialsRepository.getRequiredOne(accountName); if (!skipIndex && objectType == ObjectType.CANARY_CONFIG) { Set> canaryConfigSet = @@ -415,12 +410,12 @@ public List> listObjectKeys( } } - private String daoRoot(AwsNamedAccountCredentials credentials, String daoTypeName) { + private String daoRoot(AwsManagedAccount credentials, String daoTypeName) { return credentials.getRootFolder() + '/' + daoTypeName; } private String buildS3Key( - AwsNamedAccountCredentials credentials, + AwsManagedAccount credentials, ObjectType objectType, String group, String objectKey, @@ -437,7 +432,7 @@ private String buildS3Key( .replace("//", "/"); } - private String buildTypedFolder(AwsNamedAccountCredentials credentials, String type) { + private String buildTypedFolder(AwsManagedAccount credentials, String type) { return daoRoot(credentials, type).replaceAll("//", "/"); } } diff --git a/kayenta-signalfx/src/main/java/com/netflix/kayenta/signalfx/config/SignalFxConfiguration.java b/kayenta-signalfx/src/main/java/com/netflix/kayenta/signalfx/config/SignalFxConfiguration.java index 7142986ca..a34809d2b 100644 --- a/kayenta-signalfx/src/main/java/com/netflix/kayenta/signalfx/config/SignalFxConfiguration.java +++ b/kayenta-signalfx/src/main/java/com/netflix/kayenta/signalfx/config/SignalFxConfiguration.java @@ -18,19 +18,13 @@ package com.netflix.kayenta.signalfx.config; import com.netflix.kayenta.metrics.MetricsService; -import com.netflix.kayenta.retrofit.config.RemoteService; import com.netflix.kayenta.retrofit.config.RetrofitClientFactory; -import com.netflix.kayenta.security.AccountCredentials; import com.netflix.kayenta.security.AccountCredentialsRepository; import com.netflix.kayenta.signalfx.metrics.SignalFxMetricsService; -import com.netflix.kayenta.signalfx.security.SignalFxCredentials; -import com.netflix.kayenta.signalfx.security.SignalFxNamedAccountCredentials; import com.netflix.kayenta.signalfx.service.SignalFxConverter; import com.netflix.kayenta.signalfx.service.SignalFxSignalFlowRemoteService; import com.squareup.okhttp.OkHttpClient; -import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -38,7 +32,6 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; -import org.springframework.util.CollectionUtils; @Configuration @ConditionalOnProperty("kayenta.signalfx.enabled") @@ -46,8 +39,6 @@ @Slf4j public class SignalFxConfiguration { - private static final String SIGNAL_FX_SIGNAL_FLOW_ENDPOINT_URI = "https://stream.signalfx.com"; - @Bean @ConfigurationProperties("kayenta.signalfx") SignalFxConfigurationProperties signalFxConfigurationProperties() { @@ -78,39 +69,19 @@ MetricsService signalFxMetricService( SignalFxMetricsService.SignalFxMetricsServiceBuilder metricsServiceBuilder = SignalFxMetricsService.builder(); - for (SignalFxManagedAccount signalFxManagedAccount : - signalFxConfigurationProperties.getAccounts()) { - String name = signalFxManagedAccount.getName(); - List supportedTypes = signalFxManagedAccount.getSupportedTypes(); - SignalFxCredentials signalFxCredentials = - new SignalFxCredentials(signalFxManagedAccount.getAccessToken()); - - final RemoteService signalFxSignalFlowEndpoint = - Optional.ofNullable(signalFxManagedAccount.getEndpoint()) - .orElse(new RemoteService().setBaseUrl(SIGNAL_FX_SIGNAL_FLOW_ENDPOINT_URI)); - - SignalFxNamedAccountCredentials.SignalFxNamedAccountCredentialsBuilder - accountCredentialsBuilder = - SignalFxNamedAccountCredentials.builder() - .name(name) - .endpoint(signalFxSignalFlowEndpoint) - .credentials(signalFxCredentials); - - if (!CollectionUtils.isEmpty(supportedTypes)) { - if (supportedTypes.contains(AccountCredentials.Type.METRICS_STORE)) { - accountCredentialsBuilder.signalFlowService( - retrofitClientFactory.createClient( - SignalFxSignalFlowRemoteService.class, - new SignalFxConverter(), - signalFxSignalFlowEndpoint, - okHttpClient)); - } - accountCredentialsBuilder.supportedTypes(supportedTypes); - } + signalFxConfigurationProperties.getAccounts().parallelStream() + .forEach( + signalFxManagedAccount -> { + signalFxManagedAccount.setSignalFlowService( + retrofitClientFactory.createClient( + SignalFxSignalFlowRemoteService.class, + new SignalFxConverter(), + signalFxManagedAccount.getEndpoint(), + okHttpClient)); - accountCredentialsRepository.save(name, accountCredentialsBuilder.build()); - metricsServiceBuilder.accountName(name); - } + accountCredentialsRepository.save(signalFxManagedAccount); + metricsServiceBuilder.accountName(signalFxManagedAccount.getName()); + }); log.info( "Configured the SignalFx Metrics Service with the following accounts: {}", diff --git a/kayenta-signalfx/src/main/java/com/netflix/kayenta/signalfx/config/SignalFxManagedAccount.java b/kayenta-signalfx/src/main/java/com/netflix/kayenta/signalfx/config/SignalFxManagedAccount.java index 95cfb8eb7..2a29b92a3 100644 --- a/kayenta-signalfx/src/main/java/com/netflix/kayenta/signalfx/config/SignalFxManagedAccount.java +++ b/kayenta-signalfx/src/main/java/com/netflix/kayenta/signalfx/config/SignalFxManagedAccount.java @@ -17,27 +17,48 @@ package com.netflix.kayenta.signalfx.config; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.netflix.kayenta.retrofit.config.RemoteService; import com.netflix.kayenta.security.AccountCredentials; +import com.netflix.kayenta.signalfx.service.SignalFxSignalFlowRemoteService; import java.util.Collections; import java.util.List; import javax.annotation.Nullable; import javax.validation.constraints.NotNull; -import lombok.Data; +import lombok.*; +import lombok.experimental.SuperBuilder; -@Data -public class SignalFxManagedAccount { +@Getter +@Setter +@SuperBuilder +@NoArgsConstructor +public class SignalFxManagedAccount extends AccountCredentials { + private static final String SIGNAL_FX_SIGNAL_FLOW_ENDPOINT_URI = "https://stream.signalfx.com"; @NotNull private String name; private String accessToken; - private List supportedTypes = + private static final List SUPPORTED_TYPES = Collections.singletonList(AccountCredentials.Type.METRICS_STORE); - @Nullable private RemoteService endpoint; + @Nullable @Builder.Default + private RemoteService endpoint = + new RemoteService().setBaseUrl(SIGNAL_FX_SIGNAL_FLOW_ENDPOINT_URI); @Nullable private String defaultScopeKey; @Nullable private String defaultLocationKey; + + @Override + public List getSupportedTypes() { + return SUPPORTED_TYPES; + } + + @Override + public String getType() { + return "signalfx"; + } + + @JsonIgnore private transient SignalFxSignalFlowRemoteService signalFlowService; } diff --git a/kayenta-signalfx/src/main/java/com/netflix/kayenta/signalfx/metrics/SignalFxMetricsService.java b/kayenta-signalfx/src/main/java/com/netflix/kayenta/signalfx/metrics/SignalFxMetricsService.java index 9898c93be..f092e8dd5 100644 --- a/kayenta-signalfx/src/main/java/com/netflix/kayenta/signalfx/metrics/SignalFxMetricsService.java +++ b/kayenta-signalfx/src/main/java/com/netflix/kayenta/signalfx/metrics/SignalFxMetricsService.java @@ -29,8 +29,8 @@ import com.netflix.kayenta.metrics.MetricsService; import com.netflix.kayenta.security.AccountCredentialsRepository; import com.netflix.kayenta.signalfx.canary.SignalFxCanaryScope; +import com.netflix.kayenta.signalfx.config.SignalFxManagedAccount; import com.netflix.kayenta.signalfx.config.SignalFxScopeConfiguration; -import com.netflix.kayenta.signalfx.security.SignalFxNamedAccountCredentials; import com.netflix.kayenta.signalfx.service.ErrorResponse; import com.netflix.kayenta.signalfx.service.SignalFlowExecutionResult; import com.netflix.kayenta.signalfx.service.SignalFxRequestError; @@ -38,11 +38,7 @@ import com.signalfx.signalflow.ChannelMessage; import java.time.Duration; import java.time.Instant; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Optional; +import java.util.*; import java.util.stream.Collectors; import javax.validation.constraints.NotNull; import lombok.Builder; @@ -122,7 +118,7 @@ public List queryMetrics( List queryPairs = Optional.ofNullable(queryConfig.getQueryPairs()).orElse(new LinkedList<>()); - SignalFxNamedAccountCredentials accountCredentials = + SignalFxManagedAccount accountCredentials = accountCredentialsRepository.getRequiredOne(metricsAccountName); String accessToken = accountCredentials.getCredentials().getAccessToken(); diff --git a/kayenta-signalfx/src/main/java/com/netflix/kayenta/signalfx/security/SignalFxCredentials.java b/kayenta-signalfx/src/main/java/com/netflix/kayenta/signalfx/security/SignalFxCredentials.java deleted file mode 100644 index 476c9cb9e..000000000 --- a/kayenta-signalfx/src/main/java/com/netflix/kayenta/signalfx/security/SignalFxCredentials.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2018 Nike, inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.netflix.kayenta.signalfx.security; - -import lombok.AllArgsConstructor; -import lombok.Data; - -@Data -@AllArgsConstructor -public class SignalFxCredentials { - - private String accessToken; -} diff --git a/kayenta-signalfx/src/main/java/com/netflix/kayenta/signalfx/security/SignalFxNamedAccountCredentials.java b/kayenta-signalfx/src/main/java/com/netflix/kayenta/signalfx/security/SignalFxNamedAccountCredentials.java deleted file mode 100644 index eeebd856e..000000000 --- a/kayenta-signalfx/src/main/java/com/netflix/kayenta/signalfx/security/SignalFxNamedAccountCredentials.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2018 Nike, inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.netflix.kayenta.signalfx.security; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.netflix.kayenta.retrofit.config.RemoteService; -import com.netflix.kayenta.security.AccountCredentials; -import com.netflix.kayenta.signalfx.service.SignalFxSignalFlowRemoteService; -import java.util.List; -import javax.validation.constraints.NotNull; -import lombok.*; -import lombok.experimental.SuperBuilder; - -@Getter -@Setter -@SuperBuilder -@NoArgsConstructor -public class SignalFxNamedAccountCredentials extends AccountCredentials { - - @NotNull private String name; - - @NotNull @Singular private List supportedTypes; - - @NotNull private SignalFxCredentials credentials; - - @NotNull private RemoteService endpoint; - - @Override - public String getType() { - return "signalfx"; - } - - @JsonIgnore SignalFxSignalFlowRemoteService signalFlowService; -} diff --git a/kayenta-stackdriver/src/main/java/com/netflix/kayenta/stackdriver/config/StackdriverConfiguration.java b/kayenta-stackdriver/src/main/java/com/netflix/kayenta/stackdriver/config/StackdriverConfiguration.java index b2cf80888..9d3004344 100644 --- a/kayenta-stackdriver/src/main/java/com/netflix/kayenta/stackdriver/config/StackdriverConfiguration.java +++ b/kayenta-stackdriver/src/main/java/com/netflix/kayenta/stackdriver/config/StackdriverConfiguration.java @@ -16,7 +16,7 @@ package com.netflix.kayenta.stackdriver.config; -import com.netflix.kayenta.google.security.GoogleNamedAccountCredentials; +import com.netflix.kayenta.google.config.GoogleManagedAccount; import com.netflix.kayenta.metrics.MetricsService; import com.netflix.kayenta.security.AccountCredentials; import com.netflix.kayenta.security.AccountCredentialsRepository; @@ -56,7 +56,7 @@ MetricsService stackdriverMetricsService( StackdriverMetricsService.builder(); accountCredentialsRepository.getAll().stream() - .filter(c -> c instanceof GoogleNamedAccountCredentials) + .filter(GoogleManagedAccount.class::isInstance) .filter(c -> c.getSupportedTypes().contains(AccountCredentials.Type.METRICS_STORE)) .map(c -> c.getName()) .forEach(stackdriverMetricsServiceBuilder::accountName); diff --git a/kayenta-stackdriver/src/main/java/com/netflix/kayenta/stackdriver/metrics/StackdriverMetricsService.java b/kayenta-stackdriver/src/main/java/com/netflix/kayenta/stackdriver/metrics/StackdriverMetricsService.java index f7254ff6a..c4b617c94 100644 --- a/kayenta-stackdriver/src/main/java/com/netflix/kayenta/stackdriver/metrics/StackdriverMetricsService.java +++ b/kayenta-stackdriver/src/main/java/com/netflix/kayenta/stackdriver/metrics/StackdriverMetricsService.java @@ -29,7 +29,7 @@ import com.netflix.kayenta.canary.CanaryScope; import com.netflix.kayenta.canary.providers.metrics.QueryConfigUtils; import com.netflix.kayenta.canary.providers.metrics.StackdriverCanaryMetricSetQueryConfig; -import com.netflix.kayenta.google.security.GoogleNamedAccountCredentials; +import com.netflix.kayenta.google.config.GoogleManagedAccount; import com.netflix.kayenta.metrics.MetricSet; import com.netflix.kayenta.metrics.MetricsService; import com.netflix.kayenta.security.AccountCredentials; @@ -46,7 +46,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import javax.validation.constraints.NotNull; @@ -276,7 +275,7 @@ public List queryMetrics( } StackdriverCanaryScope stackdriverCanaryScope = (StackdriverCanaryScope) canaryScope; - GoogleNamedAccountCredentials stackdriverCredentials = + GoogleManagedAccount stackdriverCredentials = accountCredentialsRepository.getRequiredOne(metricsAccountName); Monitoring monitoring = stackdriverCredentials.getMonitoring(); StackdriverCanaryMetricSetQueryConfig stackdriverMetricSetQuery = @@ -454,7 +453,7 @@ private String determineProjectId( String projectId = stackdriverCanaryScope.getProject(); if (StringUtils.isEmpty(projectId)) { - GoogleNamedAccountCredentials stackdriverCredentials = + GoogleManagedAccount stackdriverCredentials = accountCredentialsRepository.getRequiredOne(metricsAccountName); projectId = stackdriverCredentials.getProject(); @@ -480,38 +479,41 @@ public List getMetadata(String metricsAccountName, String filter) { @Scheduled(fixedDelayString = "#{@stackdriverConfigurationProperties.metadataCachingIntervalMS}") public void updateMetricDescriptorsCache() throws IOException { - Set accountCredentialsSet = - accountCredentialsRepository.getAllOf(AccountCredentials.Type.METRICS_STORE); - - for (AccountCredentials credentials : accountCredentialsSet) { - if (credentials instanceof GoogleNamedAccountCredentials) { - GoogleNamedAccountCredentials stackdriverCredentials = - (GoogleNamedAccountCredentials) credentials; - ListMetricDescriptorsResponse listMetricDescriptorsResponse = - stackdriverCredentials - .getMonitoring() - .projects() - .metricDescriptors() - .list("projects/" + stackdriverCredentials.getProject()) - .execute(); - List metricDescriptors = - listMetricDescriptorsResponse.getMetricDescriptors(); - - if (!CollectionUtils.isEmpty(metricDescriptors)) { - // TODO(duftler): Should we instead be building the union across all accounts? This - // doesn't seem quite right yet. - metricDescriptorsCache = metricDescriptors; - - log.debug( - "Updated cache with {} metric descriptors via account {}.", - metricDescriptors.size(), - stackdriverCredentials.getName()); - } else { - log.debug( - "While updating cache, found no metric descriptors via account {}.", - stackdriverCredentials.getName()); - } - } - } + accountCredentialsRepository.getAllOf(AccountCredentials.Type.METRICS_STORE).stream() + .filter(GoogleManagedAccount.class::isInstance) + .map(GoogleManagedAccount.class::cast) + .forEach( + stackdriverCredentials -> { + try { + ListMetricDescriptorsResponse listMetricDescriptorsResponse = + stackdriverCredentials + .getMonitoring() + .projects() + .metricDescriptors() + .list("projects/" + stackdriverCredentials.getProject()) + .execute(); + + List metricDescriptors = + listMetricDescriptorsResponse.getMetricDescriptors(); + + if (!CollectionUtils.isEmpty(metricDescriptors)) { + // TODO(duftler): Should we instead be building the union across all accounts? + // This + // doesn't seem quite right yet. + metricDescriptorsCache = metricDescriptors; + + log.debug( + "Updated cache with {} metric descriptors via account {}.", + metricDescriptors.size(), + stackdriverCredentials.getName()); + } else { + log.debug( + "While updating cache, found no metric descriptors via account {}.", + stackdriverCredentials.getName()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + }); } } From 92f996cc2dc92aaf61b672894a0643d484710127 Mon Sep 17 00:00:00 2001 From: Jason McIntosh Date: Tue, 28 Feb 2023 15:05:34 -0600 Subject: [PATCH 2/4] chore(refactor): Dedupe graphite --- .../config/GraphiteConfiguration.java | 45 ++++++------------- .../config/GraphiteManagedAccount.java | 24 +++++++--- .../metrics/GraphiteMetricsService.java | 14 ++---- .../security/GraphiteCredentials.java | 31 ------------- .../GraphiteNamedAccountCredentials.java | 43 ------------------ 5 files changed, 35 insertions(+), 122 deletions(-) delete mode 100644 kayenta-graphite/src/main/java/com/netflix/kayenta/graphite/security/GraphiteCredentials.java delete mode 100644 kayenta-graphite/src/main/java/com/netflix/kayenta/graphite/security/GraphiteNamedAccountCredentials.java diff --git a/kayenta-graphite/src/main/java/com/netflix/kayenta/graphite/config/GraphiteConfiguration.java b/kayenta-graphite/src/main/java/com/netflix/kayenta/graphite/config/GraphiteConfiguration.java index 5fbace763..836242950 100644 --- a/kayenta-graphite/src/main/java/com/netflix/kayenta/graphite/config/GraphiteConfiguration.java +++ b/kayenta-graphite/src/main/java/com/netflix/kayenta/graphite/config/GraphiteConfiguration.java @@ -18,8 +18,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.netflix.kayenta.graphite.metrics.GraphiteMetricsService; -import com.netflix.kayenta.graphite.security.GraphiteCredentials; -import com.netflix.kayenta.graphite.security.GraphiteNamedAccountCredentials; import com.netflix.kayenta.graphite.service.GraphiteRemoteService; import com.netflix.kayenta.metrics.MetricsService; import com.netflix.kayenta.retrofit.config.RetrofitClientFactory; @@ -27,14 +25,12 @@ import com.netflix.kayenta.security.AccountCredentialsRepository; import com.squareup.okhttp.OkHttpClient; import java.io.IOException; -import java.util.List; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; -import org.springframework.util.CollectionUtils; import retrofit.converter.JacksonConverter; @Configuration @@ -66,34 +62,19 @@ MetricsService graphiteMetricsService( GraphiteMetricsService.GraphiteMetricsServiceBuilder graphiteMetricsServiceBuilder = GraphiteMetricsService.builder(); - for (GraphiteManagedAccount account : graphiteConfigurationProperties.getAccounts()) { - String accountName = account.getName(); - List supportedTypes = account.getSupportedTypes(); - - GraphiteCredentials credentials = GraphiteCredentials.builder().build(); - - GraphiteNamedAccountCredentials.GraphiteNamedAccountCredentialsBuilder - accountCredentialsBuilder = - GraphiteNamedAccountCredentials.builder() - .name(accountName) - .endpoint(account.getEndpoint()) - .credentials(credentials); - if (!CollectionUtils.isEmpty(supportedTypes)) { - if (supportedTypes.contains(AccountCredentials.Type.METRICS_STORE)) { - accountCredentialsBuilder.graphiteRemoteService( - retrofitClientFactory.createClient( - GraphiteRemoteService.class, - new JacksonConverter(objectMapper), - account.getEndpoint(), - okHttpClient)); - } - - accountCredentialsBuilder.supportedTypes(supportedTypes); - } - - accountCredentialsRepository.save(accountName, accountCredentialsBuilder.build()); - graphiteMetricsServiceBuilder.accountName(accountName); - } + graphiteConfigurationProperties.getAccounts().stream() + .filter(each -> each.getSupportedTypes().contains(AccountCredentials.Type.METRICS_STORE)) + .forEach( + account -> { + account.setGraphiteRemoteService( + retrofitClientFactory.createClient( + GraphiteRemoteService.class, + new JacksonConverter(objectMapper), + account.getEndpoint(), + okHttpClient)); + accountCredentialsRepository.save(account); + graphiteMetricsServiceBuilder.accountName(account.getName()); + }); log.info( "Populated GraphiteMetricsService with {} Graphite accounts.", diff --git a/kayenta-graphite/src/main/java/com/netflix/kayenta/graphite/config/GraphiteManagedAccount.java b/kayenta-graphite/src/main/java/com/netflix/kayenta/graphite/config/GraphiteManagedAccount.java index 6cac192ee..67d18405b 100644 --- a/kayenta-graphite/src/main/java/com/netflix/kayenta/graphite/config/GraphiteManagedAccount.java +++ b/kayenta-graphite/src/main/java/com/netflix/kayenta/graphite/config/GraphiteManagedAccount.java @@ -16,17 +16,29 @@ package com.netflix.kayenta.graphite.config; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.netflix.kayenta.canary.providers.metrics.GraphiteCanaryMetricSetQueryConfig; +import com.netflix.kayenta.graphite.service.GraphiteRemoteService; import com.netflix.kayenta.retrofit.config.RemoteService; import com.netflix.kayenta.security.AccountCredentials; -import java.util.List; import javax.validation.constraints.NotNull; -import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.SuperBuilder; -@Data -public class GraphiteManagedAccount { - @NotNull private String name; +@Getter +@Setter +@SuperBuilder +@NoArgsConstructor +public class GraphiteManagedAccount extends AccountCredentials { @NotNull private RemoteService endpoint; - private List supportedTypes; + @Override + public String getType() { + return GraphiteCanaryMetricSetQueryConfig.SERVICE_TYPE; + } + + @JsonIgnore private transient GraphiteRemoteService graphiteRemoteService; } diff --git a/kayenta-graphite/src/main/java/com/netflix/kayenta/graphite/metrics/GraphiteMetricsService.java b/kayenta-graphite/src/main/java/com/netflix/kayenta/graphite/metrics/GraphiteMetricsService.java index fe1bb818e..0ba0f6e61 100644 --- a/kayenta-graphite/src/main/java/com/netflix/kayenta/graphite/metrics/GraphiteMetricsService.java +++ b/kayenta-graphite/src/main/java/com/netflix/kayenta/graphite/metrics/GraphiteMetricsService.java @@ -21,10 +21,10 @@ import com.netflix.kayenta.canary.CanaryMetricConfig; import com.netflix.kayenta.canary.CanaryScope; import com.netflix.kayenta.canary.providers.metrics.GraphiteCanaryMetricSetQueryConfig; +import com.netflix.kayenta.graphite.config.GraphiteManagedAccount; import com.netflix.kayenta.graphite.model.GraphiteMetricDescriptor; import com.netflix.kayenta.graphite.model.GraphiteMetricDescriptorsResponse; import com.netflix.kayenta.graphite.model.GraphiteResults; -import com.netflix.kayenta.graphite.security.GraphiteNamedAccountCredentials; import com.netflix.kayenta.graphite.service.GraphiteRemoteService; import com.netflix.kayenta.metrics.MetricSet; import com.netflix.kayenta.metrics.MetricsService; @@ -32,13 +32,7 @@ import com.netflix.spectator.api.Registry; import java.io.IOException; import java.time.Instant; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; import javax.validation.constraints.NotNull; import lombok.Builder; @@ -106,7 +100,7 @@ public List queryMetrics( CanaryMetricConfig canaryMetricConfig, CanaryScope canaryScope) throws IOException { - GraphiteNamedAccountCredentials accountCredentials = + GraphiteManagedAccount accountCredentials = accountCredentialsRepository.getRequiredOne(metricsAccountName); GraphiteRemoteService remoteService = accountCredentials.getGraphiteRemoteService(); @@ -155,7 +149,7 @@ public List getMetadata(String metricsAccountName, String filter) throws IO if (needSpecialDescriptors) { result.addAll(getSpecialMetricDescriptors(baseFilter)); } else { - GraphiteNamedAccountCredentials accountCredentials = + GraphiteManagedAccount accountCredentials = accountCredentialsRepository.getRequiredOne(metricsAccountName); GraphiteRemoteService remoteService = accountCredentials.getGraphiteRemoteService(); diff --git a/kayenta-graphite/src/main/java/com/netflix/kayenta/graphite/security/GraphiteCredentials.java b/kayenta-graphite/src/main/java/com/netflix/kayenta/graphite/security/GraphiteCredentials.java deleted file mode 100644 index 7dabb2098..000000000 --- a/kayenta-graphite/src/main/java/com/netflix/kayenta/graphite/security/GraphiteCredentials.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2018 Snap Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.kayenta.graphite.security; - -import java.util.Optional; -import lombok.Builder; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; - -@Builder -@Data -@Slf4j -public class GraphiteCredentials { - private static String applicationVersion = - Optional.ofNullable(GraphiteCredentials.class.getPackage().getImplementationVersion()) - .orElse("Unknown"); -} diff --git a/kayenta-graphite/src/main/java/com/netflix/kayenta/graphite/security/GraphiteNamedAccountCredentials.java b/kayenta-graphite/src/main/java/com/netflix/kayenta/graphite/security/GraphiteNamedAccountCredentials.java deleted file mode 100644 index 2ff974731..000000000 --- a/kayenta-graphite/src/main/java/com/netflix/kayenta/graphite/security/GraphiteNamedAccountCredentials.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2018 Snap Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.kayenta.graphite.security; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.netflix.kayenta.canary.providers.metrics.GraphiteCanaryMetricSetQueryConfig; -import com.netflix.kayenta.graphite.service.GraphiteRemoteService; -import com.netflix.kayenta.retrofit.config.RemoteService; -import com.netflix.kayenta.security.AccountCredentials; -import javax.validation.constraints.NotNull; -import lombok.*; -import lombok.experimental.SuperBuilder; - -@Getter -@Setter -@SuperBuilder -@NoArgsConstructor -public class GraphiteNamedAccountCredentials extends AccountCredentials { - @NotNull private GraphiteCredentials credentials; - - @NotNull private RemoteService endpoint; - - @Override - public String getType() { - return GraphiteCanaryMetricSetQueryConfig.SERVICE_TYPE; - } - - @JsonIgnore GraphiteRemoteService graphiteRemoteService; -} From a0e732ca08ae7b072e6c1e80115da3b3e2e36c7e Mon Sep 17 00:00:00 2001 From: Jason McIntosh Date: Tue, 28 Feb 2023 16:02:21 -0600 Subject: [PATCH 3/4] chore(refactor): Final set of accounts to cleanup --- .../kayenta/security/AccountCredentials.java | 3 +- .../config/InfluxDbConfiguration.java | 22 ++------ .../config/InfluxDbManagedAccount.java | 23 +++++--- .../metrics/InfluxDbMetricsService.java | 4 +- .../InfluxDbNamedAccountCredentials.java | 43 --------------- .../security/InfluxdbCredentials.java | 31 ----------- .../config/NewRelicConfiguration.java | 52 ++++--------------- .../config/NewRelicManagedAccount.java | 35 +++++++++---- .../metrics/NewRelicMetricsService.java | 7 ++- .../security/NewRelicCredentials.java | 35 ------------- .../NewRelicNamedAccountCredentials.java | 43 --------------- .../config/ConfigBinConfiguration.java | 26 +--------- .../config/ConfigBinManagedAccount.java | 24 ++++++--- .../security/ConfigBinAccountCredentials.java | 32 ------------ .../ConfigBinNamedAccountCredentials.java | 50 ------------------ .../storage/ConfigBinStorageService.java | 20 +++---- .../memory/config/MemoryConfiguration.java | 23 ++------ .../memory/config/MemoryManagedAccount.java | 23 ++++++-- .../security/MemoryAccountCredentials.java | 33 ------------ .../MemoryNamedAccountCredentials.java | 42 --------------- .../memory/storage/MemoryStorageService.java | 15 +++--- .../config/WavefrontConfiguration.java | 21 +------- .../config/WavefrontManagedAccount.java | 23 +++++--- .../metrics/WavefrontMetricsService.java | 8 ++- .../security/WavefrontCredentials.java | 38 -------------- .../WavefrontNamedAccountCredentials.java | 44 ---------------- 26 files changed, 139 insertions(+), 581 deletions(-) delete mode 100644 kayenta-influxdb/src/main/java/com/netflix/kayenta/influxdb/security/InfluxDbNamedAccountCredentials.java delete mode 100644 kayenta-influxdb/src/main/java/com/netflix/kayenta/influxdb/security/InfluxdbCredentials.java delete mode 100644 kayenta-newrelic-insights/src/main/java/com/netflix/kayenta/newrelic/security/NewRelicCredentials.java delete mode 100644 kayenta-newrelic-insights/src/main/java/com/netflix/kayenta/newrelic/security/NewRelicNamedAccountCredentials.java delete mode 100644 kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/security/ConfigBinAccountCredentials.java delete mode 100644 kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/security/ConfigBinNamedAccountCredentials.java delete mode 100644 kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/security/MemoryAccountCredentials.java delete mode 100644 kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/security/MemoryNamedAccountCredentials.java delete mode 100644 kayenta-wavefront/src/main/java/com/netflix/kayenta/wavefront/security/WavefrontCredentials.java delete mode 100644 kayenta-wavefront/src/main/java/com/netflix/kayenta/wavefront/security/WavefrontNamedAccountCredentials.java diff --git a/kayenta-core/src/main/java/com/netflix/kayenta/security/AccountCredentials.java b/kayenta-core/src/main/java/com/netflix/kayenta/security/AccountCredentials.java index aac11cc5a..1bdd2424c 100644 --- a/kayenta-core/src/main/java/com/netflix/kayenta/security/AccountCredentials.java +++ b/kayenta-core/src/main/java/com/netflix/kayenta/security/AccountCredentials.java @@ -56,8 +56,9 @@ public abstract class AccountCredentials { public T getCredentials() { return (T) this; } - ; + @Deprecated + /** Account types other than metric accounts are going away. No longer relevant post SQL. */ public enum Type { METRICS_STORE, OBJECT_STORE, diff --git a/kayenta-influxdb/src/main/java/com/netflix/kayenta/influxdb/config/InfluxDbConfiguration.java b/kayenta-influxdb/src/main/java/com/netflix/kayenta/influxdb/config/InfluxDbConfiguration.java index df2c38fd0..41cd68da1 100644 --- a/kayenta-influxdb/src/main/java/com/netflix/kayenta/influxdb/config/InfluxDbConfiguration.java +++ b/kayenta-influxdb/src/main/java/com/netflix/kayenta/influxdb/config/InfluxDbConfiguration.java @@ -16,10 +16,7 @@ package com.netflix.kayenta.influxdb.config; -import com.fasterxml.jackson.databind.ObjectMapper; import com.netflix.kayenta.influxdb.metrics.InfluxDbMetricsService; -import com.netflix.kayenta.influxdb.security.InfluxDbNamedAccountCredentials; -import com.netflix.kayenta.influxdb.security.InfluxdbCredentials; import com.netflix.kayenta.influxdb.service.InfluxDbRemoteService; import com.netflix.kayenta.metrics.MetricsService; import com.netflix.kayenta.retrofit.config.RetrofitClientFactory; @@ -59,7 +56,6 @@ MetricsService influxDbMetricsService( InfluxDbResponseConverter influxDbResponseConverter, InfluxDbConfigurationProperties influxDbConfigurationProperties, RetrofitClientFactory retrofitClientFactory, - ObjectMapper objectMapper, OkHttpClient okHttpClient, AccountCredentialsRepository accountCredentialsRepository) throws IOException { @@ -67,32 +63,20 @@ MetricsService influxDbMetricsService( InfluxDbMetricsService.builder(); for (InfluxDbManagedAccount account : influxDbConfigurationProperties.getAccounts()) { - String name = account.getName(); List supportedTypes = account.getSupportedTypes(); - - InfluxdbCredentials credentials = InfluxdbCredentials.builder().build(); - - InfluxDbNamedAccountCredentials.InfluxDbNamedAccountCredentialsBuilder - accountCredentialsBuilder = - InfluxDbNamedAccountCredentials.builder() - .name(name) - .endpoint(account.getEndpoint()) - .credentials(credentials); - if (!CollectionUtils.isEmpty(supportedTypes)) { if (supportedTypes.contains(AccountCredentials.Type.METRICS_STORE)) { - accountCredentialsBuilder.influxDbRemoteService( + account.setInfluxDbRemoteService( retrofitClientFactory.createClient( InfluxDbRemoteService.class, influxDbResponseConverter, account.getEndpoint(), okHttpClient)); } - accountCredentialsBuilder.supportedTypes(supportedTypes); } - accountCredentialsRepository.save(name, accountCredentialsBuilder.build()); - metricsServiceBuilder.accountName(name); + accountCredentialsRepository.save(account); + metricsServiceBuilder.accountName(account.getName()); } log.info( diff --git a/kayenta-influxdb/src/main/java/com/netflix/kayenta/influxdb/config/InfluxDbManagedAccount.java b/kayenta-influxdb/src/main/java/com/netflix/kayenta/influxdb/config/InfluxDbManagedAccount.java index 9887683de..de32f273e 100644 --- a/kayenta-influxdb/src/main/java/com/netflix/kayenta/influxdb/config/InfluxDbManagedAccount.java +++ b/kayenta-influxdb/src/main/java/com/netflix/kayenta/influxdb/config/InfluxDbManagedAccount.java @@ -16,19 +16,30 @@ package com.netflix.kayenta.influxdb.config; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.netflix.kayenta.canary.providers.metrics.InfluxdbCanaryMetricSetQueryConfig; +import com.netflix.kayenta.influxdb.service.InfluxDbRemoteService; import com.netflix.kayenta.retrofit.config.RemoteService; import com.netflix.kayenta.security.AccountCredentials; -import java.util.List; import javax.validation.constraints.NotNull; -import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.SuperBuilder; -@Data -public class InfluxDbManagedAccount { - @NotNull private String name; +@Getter +@Setter +@SuperBuilder +@NoArgsConstructor +public class InfluxDbManagedAccount extends AccountCredentials { private String apiKey; private String applicationKey; @NotNull private RemoteService endpoint; + @JsonIgnore private transient InfluxDbRemoteService influxDbRemoteService; - private List supportedTypes; + @Override + public String getType() { + return InfluxdbCanaryMetricSetQueryConfig.SERVICE_TYPE; + } } diff --git a/kayenta-influxdb/src/main/java/com/netflix/kayenta/influxdb/metrics/InfluxDbMetricsService.java b/kayenta-influxdb/src/main/java/com/netflix/kayenta/influxdb/metrics/InfluxDbMetricsService.java index 17c65ca7b..7ec413325 100644 --- a/kayenta-influxdb/src/main/java/com/netflix/kayenta/influxdb/metrics/InfluxDbMetricsService.java +++ b/kayenta-influxdb/src/main/java/com/netflix/kayenta/influxdb/metrics/InfluxDbMetricsService.java @@ -20,8 +20,8 @@ import com.netflix.kayenta.canary.CanaryMetricConfig; import com.netflix.kayenta.canary.CanaryScope; import com.netflix.kayenta.canary.providers.metrics.InfluxdbCanaryMetricSetQueryConfig; +import com.netflix.kayenta.influxdb.config.InfluxDbManagedAccount; import com.netflix.kayenta.influxdb.model.InfluxDbResult; -import com.netflix.kayenta.influxdb.security.InfluxDbNamedAccountCredentials; import com.netflix.kayenta.influxdb.service.InfluxDbRemoteService; import com.netflix.kayenta.metrics.MetricSet; import com.netflix.kayenta.metrics.MetricSet.MetricSetBuilder; @@ -70,7 +70,7 @@ public List queryMetrics( CanaryMetricConfig canaryMetricConfig, CanaryScope canaryScope) { - InfluxDbNamedAccountCredentials accountCredentials = + InfluxDbManagedAccount accountCredentials = accountCredentialsRepository.getRequiredOne(accountName); InfluxDbRemoteService remoteService = accountCredentials.getInfluxDbRemoteService(); diff --git a/kayenta-influxdb/src/main/java/com/netflix/kayenta/influxdb/security/InfluxDbNamedAccountCredentials.java b/kayenta-influxdb/src/main/java/com/netflix/kayenta/influxdb/security/InfluxDbNamedAccountCredentials.java deleted file mode 100644 index 001039889..000000000 --- a/kayenta-influxdb/src/main/java/com/netflix/kayenta/influxdb/security/InfluxDbNamedAccountCredentials.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2018 Joseph Motha - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.kayenta.influxdb.security; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.netflix.kayenta.canary.providers.metrics.InfluxdbCanaryMetricSetQueryConfig; -import com.netflix.kayenta.influxdb.service.InfluxDbRemoteService; -import com.netflix.kayenta.retrofit.config.RemoteService; -import com.netflix.kayenta.security.AccountCredentials; -import javax.validation.constraints.NotNull; -import lombok.*; -import lombok.experimental.SuperBuilder; - -@Getter -@Setter -@SuperBuilder -@NoArgsConstructor -public class InfluxDbNamedAccountCredentials extends AccountCredentials { - @NotNull private InfluxdbCredentials credentials; - - @NotNull private RemoteService endpoint; - - @Override - public String getType() { - return InfluxdbCanaryMetricSetQueryConfig.SERVICE_TYPE; - } - - @JsonIgnore InfluxDbRemoteService influxDbRemoteService; -} diff --git a/kayenta-influxdb/src/main/java/com/netflix/kayenta/influxdb/security/InfluxdbCredentials.java b/kayenta-influxdb/src/main/java/com/netflix/kayenta/influxdb/security/InfluxdbCredentials.java deleted file mode 100644 index 120963bee..000000000 --- a/kayenta-influxdb/src/main/java/com/netflix/kayenta/influxdb/security/InfluxdbCredentials.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2018 Joseph Motha - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.kayenta.influxdb.security; - -import java.util.Optional; -import lombok.Builder; -import lombok.Data; - -@Builder -@Data -public class InfluxdbCredentials { - private static String applicationVersion = - Optional.ofNullable(InfluxdbCredentials.class.getPackage().getImplementationVersion()) - .orElse("Unknown"); - - private String dbName; -} diff --git a/kayenta-newrelic-insights/src/main/java/com/netflix/kayenta/newrelic/config/NewRelicConfiguration.java b/kayenta-newrelic-insights/src/main/java/com/netflix/kayenta/newrelic/config/NewRelicConfiguration.java index 14df487b5..fb9f3c33c 100644 --- a/kayenta-newrelic-insights/src/main/java/com/netflix/kayenta/newrelic/config/NewRelicConfiguration.java +++ b/kayenta-newrelic-insights/src/main/java/com/netflix/kayenta/newrelic/config/NewRelicConfiguration.java @@ -19,15 +19,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.netflix.kayenta.metrics.MetricsService; import com.netflix.kayenta.newrelic.metrics.NewRelicMetricsService; -import com.netflix.kayenta.newrelic.security.NewRelicCredentials; -import com.netflix.kayenta.newrelic.security.NewRelicNamedAccountCredentials; import com.netflix.kayenta.newrelic.service.NewRelicRemoteService; -import com.netflix.kayenta.retrofit.config.RemoteService; import com.netflix.kayenta.retrofit.config.RetrofitClientFactory; -import com.netflix.kayenta.security.AccountCredentials; import com.netflix.kayenta.security.AccountCredentialsRepository; import com.squareup.okhttp.OkHttpClient; -import java.util.List; import java.util.Map; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -36,7 +31,6 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; -import org.springframework.util.CollectionUtils; import retrofit.converter.JacksonConverter; @Configuration @@ -92,44 +86,16 @@ MetricsService newrelicMetricsService( NewRelicMetricsService.NewRelicMetricsServiceBuilder metricsServiceBuilder = NewRelicMetricsService.builder(); - for (NewRelicManagedAccount account : newrelicConfigurationProperties.getAccounts()) { - String name = account.getName(); - List supportedTypes = account.getSupportedTypes(); - - NewRelicCredentials credentials = - NewRelicCredentials.builder() - .apiKey(account.getApiKey()) - .applicationKey(account.getApplicationKey()) - .build(); - - RemoteService endpoint = account.getEndpoint(); - - if (endpoint == null) { - endpoint = new RemoteService().setBaseUrl("https://insights-api.newrelic.com"); - } - - NewRelicNamedAccountCredentials.NewRelicNamedAccountCredentialsBuilder - accountCredentialsBuilder = - NewRelicNamedAccountCredentials.builder() - .name(name) - .endpoint(endpoint) - .credentials(credentials); - - if (!CollectionUtils.isEmpty(supportedTypes)) { - if (supportedTypes.contains(AccountCredentials.Type.METRICS_STORE)) { - accountCredentialsBuilder.newRelicRemoteService( - retrofitClientFactory.createClient( - NewRelicRemoteService.class, - new JacksonConverter(objectMapper), - endpoint, - okHttpClient)); - } - accountCredentialsBuilder.supportedTypes(supportedTypes); - } - - accountCredentialsRepository.save(name, accountCredentialsBuilder.build()); - metricsServiceBuilder.accountName(name); + account.setNewRelicRemoteService( + retrofitClientFactory.createClient( + NewRelicRemoteService.class, + new JacksonConverter(objectMapper), + account.getEndpoint(), + okHttpClient)); + + accountCredentialsRepository.save(account); + metricsServiceBuilder.accountName(account.getName()); } log.info( diff --git a/kayenta-newrelic-insights/src/main/java/com/netflix/kayenta/newrelic/config/NewRelicManagedAccount.java b/kayenta-newrelic-insights/src/main/java/com/netflix/kayenta/newrelic/config/NewRelicManagedAccount.java index b323cd0c8..023236930 100644 --- a/kayenta-newrelic-insights/src/main/java/com/netflix/kayenta/newrelic/config/NewRelicManagedAccount.java +++ b/kayenta-newrelic-insights/src/main/java/com/netflix/kayenta/newrelic/config/NewRelicManagedAccount.java @@ -16,27 +16,44 @@ package com.netflix.kayenta.newrelic.config; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.netflix.kayenta.newrelic.service.NewRelicRemoteService; import com.netflix.kayenta.retrofit.config.RemoteService; import com.netflix.kayenta.security.AccountCredentials; import java.util.Collections; import java.util.List; import javax.annotation.Nullable; import javax.validation.constraints.NotNull; -import lombok.Data; - -@Data -public class NewRelicManagedAccount { - - @NotNull private String name; +import lombok.*; +import lombok.experimental.SuperBuilder; + +@Getter +@Setter +@SuperBuilder +@NoArgsConstructor +public class NewRelicManagedAccount extends AccountCredentials { + private static final List SUPPORTED_TYPES = + Collections.singletonList(AccountCredentials.Type.METRICS_STORE); private String apiKey; private String applicationKey; - @NotNull private RemoteService endpoint; + @NotNull @Builder.Default + private RemoteService endpoint = + new RemoteService().setBaseUrl("https://insights-api.newrelic.com"); @Nullable private String defaultScopeKey; @Nullable private String defaultLocationKey; - private List supportedTypes = - Collections.singletonList(AccountCredentials.Type.METRICS_STORE); + @Override + public List getSupportedTypes() { + return SUPPORTED_TYPES; + } + + @Override + public String getType() { + return "newrelic"; + } + + @JsonIgnore private transient NewRelicRemoteService newRelicRemoteService; } diff --git a/kayenta-newrelic-insights/src/main/java/com/netflix/kayenta/newrelic/metrics/NewRelicMetricsService.java b/kayenta-newrelic-insights/src/main/java/com/netflix/kayenta/newrelic/metrics/NewRelicMetricsService.java index c96948624..6167fe38c 100644 --- a/kayenta-newrelic-insights/src/main/java/com/netflix/kayenta/newrelic/metrics/NewRelicMetricsService.java +++ b/kayenta-newrelic-insights/src/main/java/com/netflix/kayenta/newrelic/metrics/NewRelicMetricsService.java @@ -23,9 +23,8 @@ import com.netflix.kayenta.metrics.MetricSet; import com.netflix.kayenta.metrics.MetricsService; import com.netflix.kayenta.newrelic.canary.NewRelicCanaryScope; +import com.netflix.kayenta.newrelic.config.NewRelicManagedAccount; import com.netflix.kayenta.newrelic.config.NewRelicScopeConfiguration; -import com.netflix.kayenta.newrelic.security.NewRelicCredentials; -import com.netflix.kayenta.newrelic.security.NewRelicNamedAccountCredentials; import com.netflix.kayenta.newrelic.service.NewRelicRemoteService; import com.netflix.kayenta.newrelic.service.NewRelicTimeSeries; import com.netflix.kayenta.security.AccountCredentialsRepository; @@ -94,10 +93,10 @@ public List queryMetrics( CanaryMetricConfig canaryMetricConfig, CanaryScope canaryScope) throws IOException { - NewRelicNamedAccountCredentials accountCredentials = + NewRelicManagedAccount accountCredentials = accountCredentialsRepository.getRequiredOne(accountName); - NewRelicCredentials credentials = accountCredentials.getCredentials(); + NewRelicManagedAccount credentials = accountCredentials.getCredentials(); NewRelicRemoteService remoteService = accountCredentials.getNewRelicRemoteService(); String query = buildQuery(accountName, canaryConfig, canaryMetricConfig, canaryScope); diff --git a/kayenta-newrelic-insights/src/main/java/com/netflix/kayenta/newrelic/security/NewRelicCredentials.java b/kayenta-newrelic-insights/src/main/java/com/netflix/kayenta/newrelic/security/NewRelicCredentials.java deleted file mode 100644 index af5c0c5b5..000000000 --- a/kayenta-newrelic-insights/src/main/java/com/netflix/kayenta/newrelic/security/NewRelicCredentials.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2018 Adobe - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.kayenta.newrelic.security; - -import java.util.Optional; -import lombok.Builder; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; - -@Builder -@Data -@Slf4j -public class NewRelicCredentials { - - private static String applicationVersion = - Optional.ofNullable(NewRelicCredentials.class.getPackage().getImplementationVersion()) - .orElse("Unknown"); - - private String apiKey; - private String applicationKey; -} diff --git a/kayenta-newrelic-insights/src/main/java/com/netflix/kayenta/newrelic/security/NewRelicNamedAccountCredentials.java b/kayenta-newrelic-insights/src/main/java/com/netflix/kayenta/newrelic/security/NewRelicNamedAccountCredentials.java deleted file mode 100644 index 6b9324a2f..000000000 --- a/kayenta-newrelic-insights/src/main/java/com/netflix/kayenta/newrelic/security/NewRelicNamedAccountCredentials.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2018 Adobe - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.kayenta.newrelic.security; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.netflix.kayenta.newrelic.service.NewRelicRemoteService; -import com.netflix.kayenta.retrofit.config.RemoteService; -import com.netflix.kayenta.security.AccountCredentials; -import javax.validation.constraints.NotNull; -import lombok.*; -import lombok.experimental.SuperBuilder; - -@Getter -@Setter -@SuperBuilder -@NoArgsConstructor -public class NewRelicNamedAccountCredentials extends AccountCredentials { - - @NotNull private NewRelicCredentials credentials; - - @NotNull private RemoteService endpoint; - - @Override - public String getType() { - return "newrelic"; - } - - @JsonIgnore NewRelicRemoteService newRelicRemoteService; -} diff --git a/kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/config/ConfigBinConfiguration.java b/kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/config/ConfigBinConfiguration.java index 44210413b..9c5dda0ab 100644 --- a/kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/config/ConfigBinConfiguration.java +++ b/kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/config/ConfigBinConfiguration.java @@ -16,11 +16,8 @@ package com.netflix.kayenta.configbin.config; -import com.netflix.kayenta.configbin.security.ConfigBinAccountCredentials; -import com.netflix.kayenta.configbin.security.ConfigBinNamedAccountCredentials; import com.netflix.kayenta.configbin.service.ConfigBinRemoteService; import com.netflix.kayenta.configbin.storage.ConfigBinStorageService; -import com.netflix.kayenta.retrofit.config.RemoteService; import com.netflix.kayenta.retrofit.config.RetrofitClientFactory; import com.netflix.kayenta.security.AccountCredentials; import com.netflix.kayenta.security.AccountCredentialsRepository; @@ -60,24 +57,9 @@ ConfigBinStorageService configBinStorageService( for (ConfigBinManagedAccount configBinManagedAccount : configBinConfigurationProperties.getAccounts()) { String name = configBinManagedAccount.getName(); - String ownerApp = configBinManagedAccount.getOwnerApp(); - String configType = configBinManagedAccount.getConfigType(); - RemoteService endpoint = configBinManagedAccount.getEndpoint(); List supportedTypes = configBinManagedAccount.getSupportedTypes(); log.info("Registering ConfigBin account {} with supported types {}.", name, supportedTypes); - - ConfigBinAccountCredentials configbinAccountCredentials = - ConfigBinAccountCredentials.builder().build(); - ConfigBinNamedAccountCredentials.ConfigBinNamedAccountCredentialsBuilder - configBinNamedAccountCredentialsBuilder = - ConfigBinNamedAccountCredentials.builder() - .name(name) - .ownerApp(ownerApp) - .configType(configType) - .endpoint(endpoint) - .credentials(configbinAccountCredentials); - if (!CollectionUtils.isEmpty(supportedTypes)) { if (supportedTypes.contains(AccountCredentials.Type.CONFIGURATION_STORE)) { ConfigBinRemoteService configBinRemoteService = @@ -86,14 +68,10 @@ ConfigBinStorageService configBinStorageService( configBinConverter, configBinManagedAccount.getEndpoint(), okHttpClient); - configBinNamedAccountCredentialsBuilder.remoteService(configBinRemoteService); + configBinManagedAccount.setRemoteService(configBinRemoteService); } - configBinNamedAccountCredentialsBuilder.supportedTypes(supportedTypes); } - - ConfigBinNamedAccountCredentials configbinNamedAccountCredentials = - configBinNamedAccountCredentialsBuilder.build(); - accountCredentialsRepository.save(name, configbinNamedAccountCredentials); + accountCredentialsRepository.save(configBinManagedAccount); configbinStorageServiceBuilder.accountName(name); } diff --git a/kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/config/ConfigBinManagedAccount.java b/kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/config/ConfigBinManagedAccount.java index ea4524886..2b45de354 100644 --- a/kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/config/ConfigBinManagedAccount.java +++ b/kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/config/ConfigBinManagedAccount.java @@ -16,16 +16,21 @@ package com.netflix.kayenta.configbin.config; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.netflix.kayenta.configbin.service.ConfigBinRemoteService; import com.netflix.kayenta.retrofit.config.RemoteService; import com.netflix.kayenta.security.AccountCredentials; -import java.util.List; import javax.validation.constraints.NotNull; -import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.SuperBuilder; -@Data -public class ConfigBinManagedAccount { - - @NotNull private String name; +@Getter +@Setter +@SuperBuilder +@NoArgsConstructor +public class ConfigBinManagedAccount extends AccountCredentials { @NotNull private RemoteService endpoint; @@ -33,5 +38,10 @@ public class ConfigBinManagedAccount { @NotNull private String configType; - private List supportedTypes; + @Override + public String getType() { + return "configbin"; + } + + @JsonIgnore private transient ConfigBinRemoteService remoteService; } diff --git a/kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/security/ConfigBinAccountCredentials.java b/kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/security/ConfigBinAccountCredentials.java deleted file mode 100644 index a9647a46f..000000000 --- a/kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/security/ConfigBinAccountCredentials.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2017 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.kayenta.configbin.security; - -import java.util.Optional; -import lombok.Builder; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; - -@Builder -@Data -@Slf4j -public class ConfigBinAccountCredentials { - // ConfigBin does not currently require authentication, so we do not need anything here. - private static String applicationVersion = - Optional.ofNullable(ConfigBinAccountCredentials.class.getPackage().getImplementationVersion()) - .orElse("Unknown"); -} diff --git a/kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/security/ConfigBinNamedAccountCredentials.java b/kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/security/ConfigBinNamedAccountCredentials.java deleted file mode 100644 index 45e025e8b..000000000 --- a/kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/security/ConfigBinNamedAccountCredentials.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2017 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.kayenta.configbin.security; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.netflix.kayenta.configbin.service.ConfigBinRemoteService; -import com.netflix.kayenta.retrofit.config.RemoteService; -import com.netflix.kayenta.security.AccountCredentials; -import javax.validation.constraints.NotNull; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.experimental.SuperBuilder; - -@SuperBuilder -@Getter -@Setter -@NoArgsConstructor -public class ConfigBinNamedAccountCredentials - extends AccountCredentials { - - @NotNull private ConfigBinAccountCredentials credentials; - - @NotNull @Getter @Setter private RemoteService endpoint; - - @NotNull private String ownerApp; - - @NotNull private String configType; - - @Override - public String getType() { - return "configbin"; - } - - @JsonIgnore private ConfigBinRemoteService remoteService; -} diff --git a/kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/storage/ConfigBinStorageService.java b/kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/storage/ConfigBinStorageService.java index 81f4e38ff..205b0fab3 100644 --- a/kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/storage/ConfigBinStorageService.java +++ b/kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/storage/ConfigBinStorageService.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.netflix.kayenta.canary.CanaryConfig; -import com.netflix.kayenta.configbin.security.ConfigBinNamedAccountCredentials; +import com.netflix.kayenta.configbin.config.ConfigBinManagedAccount; import com.netflix.kayenta.configbin.service.ConfigBinRemoteService; import com.netflix.kayenta.index.CanaryConfigIndex; import com.netflix.kayenta.index.config.CanaryConfigIndexAction; @@ -72,8 +72,7 @@ public boolean servicesAccount(String accountName) { @Override public T loadObject(String accountName, ObjectType objectType, String objectKey) throws IllegalArgumentException, NotFoundException { - ConfigBinNamedAccountCredentials credentials = - accountCredentialsRepository.getRequiredOne(accountName); + ConfigBinManagedAccount credentials = accountCredentialsRepository.getRequiredOne(accountName); String ownerApp = credentials.getOwnerApp(); String configType = credentials.getConfigType(); ConfigBinRemoteService remoteService = credentials.getRemoteService(); @@ -107,8 +106,7 @@ public void storeObject( T obj, String filename, boolean isAnUpdate) { - ConfigBinNamedAccountCredentials credentials = - accountCredentialsRepository.getRequiredOne(accountName); + ConfigBinManagedAccount credentials = accountCredentialsRepository.getRequiredOne(accountName); String ownerApp = credentials.getOwnerApp(); String configType = credentials.getConfigType(); ConfigBinRemoteService remoteService = credentials.getRemoteService(); @@ -181,9 +179,7 @@ public void storeObject( } private void checkForDuplicateCanaryConfig( - CanaryConfig canaryConfig, - String canaryConfigId, - ConfigBinNamedAccountCredentials credentials) { + CanaryConfig canaryConfig, String canaryConfigId, ConfigBinManagedAccount credentials) { String canaryConfigName = canaryConfig.getName(); List applications = canaryConfig.getApplications(); String existingCanaryConfigId = @@ -204,8 +200,7 @@ private void checkForDuplicateCanaryConfig( @Override public void deleteObject(String accountName, ObjectType objectType, String objectKey) { - ConfigBinNamedAccountCredentials credentials = - accountCredentialsRepository.getRequiredOne(accountName); + ConfigBinManagedAccount credentials = accountCredentialsRepository.getRequiredOne(accountName); String ownerApp = credentials.getOwnerApp(); String configType = credentials.getConfigType(); @@ -273,8 +268,7 @@ public void deleteObject(String accountName, ObjectType objectType, String objec @Override public List> listObjectKeys( String accountName, ObjectType objectType, List applications, boolean skipIndex) { - ConfigBinNamedAccountCredentials credentials = - accountCredentialsRepository.getRequiredOne(accountName); + ConfigBinManagedAccount credentials = accountCredentialsRepository.getRequiredOne(accountName); if (!skipIndex && objectType == ObjectType.CANARY_CONFIG) { Set> canaryConfigSet = @@ -323,7 +317,7 @@ private static class ConfigBinPrefixResponse { public String configName; } - private Map metadataFor(ConfigBinNamedAccountCredentials credentials, String id) { + private Map metadataFor(ConfigBinManagedAccount credentials, String id) { // TODO: (mgraff) Should factor out to a common method, or just call .load() ConfigBinRemoteService remoteService = credentials.getRemoteService(); String ownerApp = credentials.getOwnerApp(); diff --git a/kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/config/MemoryConfiguration.java b/kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/config/MemoryConfiguration.java index aae4df136..cf1ee2d90 100644 --- a/kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/config/MemoryConfiguration.java +++ b/kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/config/MemoryConfiguration.java @@ -16,8 +16,6 @@ package com.netflix.kayenta.memory.config; -import com.netflix.kayenta.memory.security.MemoryAccountCredentials; -import com.netflix.kayenta.memory.security.MemoryNamedAccountCredentials; import com.netflix.kayenta.memory.storage.MemoryStorageService; import com.netflix.kayenta.security.AccountCredentials; import com.netflix.kayenta.security.AccountCredentialsRepository; @@ -32,7 +30,6 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; -import org.springframework.util.CollectionUtils; @Configuration @ConditionalOnProperty("kayenta.memory.enabled") @@ -59,27 +56,13 @@ StorageService storageService( log.info("Registering Memory account {} with supported types {}.", name, supportedTypes); - MemoryAccountCredentials memoryAccountCredentials = - MemoryAccountCredentials.builder().build(); - MemoryNamedAccountCredentials.MemoryNamedAccountCredentialsBuilder - memoryNamedAccountCredentialsBuilder = - MemoryNamedAccountCredentials.builder() - .name(name) - .credentials(memoryAccountCredentials); - - if (!CollectionUtils.isEmpty(supportedTypes)) { - memoryNamedAccountCredentialsBuilder.supportedTypes(supportedTypes); - } - // Set up the data maps for this in-memory storage account Map> objectStorage = new ConcurrentHashMap<>(); - memoryNamedAccountCredentialsBuilder.objects(objectStorage); + memoryManagedAccount.setObjects(objectStorage); Map>> metadataStorage = new ConcurrentHashMap<>(); - memoryNamedAccountCredentialsBuilder.metadata(metadataStorage); + memoryManagedAccount.setMetadata(metadataStorage); - MemoryNamedAccountCredentials memoryNamedAccountCredentials = - memoryNamedAccountCredentialsBuilder.build(); - accountCredentialsRepository.save(name, memoryNamedAccountCredentials); + accountCredentialsRepository.save(memoryManagedAccount); memoryStorageServiceBuilder.accountName(name); } diff --git a/kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/config/MemoryManagedAccount.java b/kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/config/MemoryManagedAccount.java index 7b3984a46..3f4c6c4fb 100644 --- a/kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/config/MemoryManagedAccount.java +++ b/kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/config/MemoryManagedAccount.java @@ -17,14 +17,29 @@ package com.netflix.kayenta.memory.config; import com.netflix.kayenta.security.AccountCredentials; +import com.netflix.kayenta.storage.ObjectType; import java.util.List; +import java.util.Map; import javax.validation.constraints.NotNull; -import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.SuperBuilder; -@Data -public class MemoryManagedAccount { +@Getter +@Setter +@SuperBuilder +@NoArgsConstructor +public class MemoryManagedAccount extends AccountCredentials { - @NotNull private String name; + @NotNull private Map> objects; + + @NotNull private Map>> metadata; private List supportedTypes; + + @Override + public String getType() { + return "memory"; + } } diff --git a/kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/security/MemoryAccountCredentials.java b/kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/security/MemoryAccountCredentials.java deleted file mode 100644 index a67e8756e..000000000 --- a/kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/security/MemoryAccountCredentials.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2017 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.kayenta.memory.security; - -import java.util.Optional; -import lombok.Builder; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; - -@Builder -@Data -@Slf4j -// TODO: Not sure what kind of credentials or configuration is really required here yet. -public class MemoryAccountCredentials { - - private static String applicationVersion = - Optional.ofNullable(MemoryAccountCredentials.class.getPackage().getImplementationVersion()) - .orElse("Unknown"); -} diff --git a/kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/security/MemoryNamedAccountCredentials.java b/kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/security/MemoryNamedAccountCredentials.java deleted file mode 100644 index 2da6bc53f..000000000 --- a/kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/security/MemoryNamedAccountCredentials.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2017 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.kayenta.memory.security; - -import com.netflix.kayenta.security.AccountCredentials; -import com.netflix.kayenta.storage.ObjectType; -import java.util.Map; -import javax.validation.constraints.NotNull; -import lombok.*; -import lombok.experimental.SuperBuilder; - -@Getter -@Setter -@SuperBuilder -@NoArgsConstructor -public class MemoryNamedAccountCredentials extends AccountCredentials { - - @NotNull private MemoryAccountCredentials credentials; - - @NotNull private Map> objects; - - @NotNull private Map>> metadata; - - @Override - public String getType() { - return "memory"; - } -} diff --git a/kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/storage/MemoryStorageService.java b/kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/storage/MemoryStorageService.java index c76f5c6ec..f36144cfb 100644 --- a/kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/storage/MemoryStorageService.java +++ b/kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/storage/MemoryStorageService.java @@ -18,7 +18,7 @@ import com.netflix.kayenta.canary.CanaryConfig; import com.netflix.kayenta.index.CanaryConfigIndex; -import com.netflix.kayenta.memory.security.MemoryNamedAccountCredentials; +import com.netflix.kayenta.memory.config.MemoryManagedAccount; import com.netflix.kayenta.security.AccountCredentialsRepository; import com.netflix.kayenta.storage.ObjectType; import com.netflix.kayenta.storage.StorageService; @@ -46,9 +46,8 @@ public boolean servicesAccount(String accountName) { return accountNames.contains(accountName); } - private MemoryNamedAccountCredentials getCredentials(String accountName, ObjectType objectType) { - MemoryNamedAccountCredentials credentials = - accountCredentialsRepository.getRequiredOne(accountName); + private MemoryManagedAccount getCredentials(String accountName, ObjectType objectType) { + MemoryManagedAccount credentials = accountCredentialsRepository.getRequiredOne(accountName); credentials.getObjects().putIfAbsent(objectType, new ConcurrentHashMap<>()); credentials.getMetadata().putIfAbsent(objectType, new ConcurrentHashMap<>()); return credentials; @@ -57,7 +56,7 @@ private MemoryNamedAccountCredentials getCredentials(String accountName, ObjectT @Override public T loadObject(String accountName, ObjectType objectType, String objectKey) throws IllegalArgumentException { - MemoryNamedAccountCredentials credentials = getCredentials(accountName, objectType); + MemoryManagedAccount credentials = getCredentials(accountName, objectType); Object entry = credentials.getObjects().get(objectType).get(objectKey); if (entry == null) { @@ -75,7 +74,7 @@ public void storeObject( T obj, String filename, boolean isAnUpdate) { - MemoryNamedAccountCredentials credentials = getCredentials(accountName, objectType); + MemoryManagedAccount credentials = getCredentials(accountName, objectType); long currentTimestamp = System.currentTimeMillis(); Map objectMetadataMap = new HashMap<>(); @@ -123,7 +122,7 @@ private void checkForDuplicateCanaryConfig( @Override public void deleteObject(String accountName, ObjectType objectType, String objectKey) { - MemoryNamedAccountCredentials credentials = getCredentials(accountName, objectType); + MemoryManagedAccount credentials = getCredentials(accountName, objectType); Object oldValue = credentials.getObjects().get(objectType).remove(objectKey); credentials.getMetadata().get(objectType).remove(objectKey); @@ -136,7 +135,7 @@ public void deleteObject(String accountName, ObjectType objectType, String objec @Override public List> listObjectKeys( String accountName, ObjectType objectType, List applications, boolean skipIndex) { - MemoryNamedAccountCredentials credentials = getCredentials(accountName, objectType); + MemoryManagedAccount credentials = getCredentials(accountName, objectType); boolean filterOnApplications = applications != null && applications.size() > 0; List> result = new ArrayList<>(); diff --git a/kayenta-wavefront/src/main/java/com/netflix/kayenta/wavefront/config/WavefrontConfiguration.java b/kayenta-wavefront/src/main/java/com/netflix/kayenta/wavefront/config/WavefrontConfiguration.java index 673413ac7..065f71c72 100644 --- a/kayenta-wavefront/src/main/java/com/netflix/kayenta/wavefront/config/WavefrontConfiguration.java +++ b/kayenta-wavefront/src/main/java/com/netflix/kayenta/wavefront/config/WavefrontConfiguration.java @@ -21,8 +21,6 @@ import com.netflix.kayenta.security.AccountCredentials; import com.netflix.kayenta.security.AccountCredentialsRepository; import com.netflix.kayenta.wavefront.metrics.WavefrontMetricsService; -import com.netflix.kayenta.wavefront.security.WavefrontCredentials; -import com.netflix.kayenta.wavefront.security.WavefrontNamedAccountCredentials; import com.netflix.kayenta.wavefront.service.WavefrontRemoteService; import com.squareup.okhttp.OkHttpClient; import java.io.IOException; @@ -62,18 +60,7 @@ MetricsService WavefrontMetricsService( wavefrontConfigurationProperties.getAccounts()) { String name = wavefrontManagedAccount.getName(); List supportedTypes = wavefrontManagedAccount.getSupportedTypes(); - log.info("Registering Wavefront account {} with supported types {}.", name, supportedTypes); - - WavefrontCredentials wavefrontCredentials = - WavefrontCredentials.builder().apiToken(wavefrontManagedAccount.getApiToken()).build(); - WavefrontNamedAccountCredentials.WavefrontNamedAccountCredentialsBuilder - wavefrontNamedAccountCredentialsBuilder = - WavefrontNamedAccountCredentials.builder() - .name(name) - .endpoint(wavefrontManagedAccount.getEndpoint()) - .credentials(wavefrontCredentials); - if (!CollectionUtils.isEmpty(supportedTypes)) { if (supportedTypes.contains(AccountCredentials.Type.METRICS_STORE)) { WavefrontRemoteService wavefrontRemoteService = @@ -83,15 +70,11 @@ MetricsService WavefrontMetricsService( wavefrontManagedAccount.getEndpoint(), okHttpClient); - wavefrontNamedAccountCredentialsBuilder.wavefrontRemoteService(wavefrontRemoteService); + wavefrontManagedAccount.setWavefrontRemoteService(wavefrontRemoteService); } - - wavefrontNamedAccountCredentialsBuilder.supportedTypes(supportedTypes); } - WavefrontNamedAccountCredentials wavefrontNamedAccountCredentials = - wavefrontNamedAccountCredentialsBuilder.build(); - accountCredentialsRepository.save(name, wavefrontNamedAccountCredentials); + accountCredentialsRepository.save(wavefrontManagedAccount); wavefrontMetricsServiceBuilder.accountName(name); } diff --git a/kayenta-wavefront/src/main/java/com/netflix/kayenta/wavefront/config/WavefrontManagedAccount.java b/kayenta-wavefront/src/main/java/com/netflix/kayenta/wavefront/config/WavefrontManagedAccount.java index 988106fd9..2818a8bce 100644 --- a/kayenta-wavefront/src/main/java/com/netflix/kayenta/wavefront/config/WavefrontManagedAccount.java +++ b/kayenta-wavefront/src/main/java/com/netflix/kayenta/wavefront/config/WavefrontManagedAccount.java @@ -15,18 +15,29 @@ */ package com.netflix.kayenta.wavefront.config; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.netflix.kayenta.retrofit.config.RemoteService; import com.netflix.kayenta.security.AccountCredentials; -import java.util.List; +import com.netflix.kayenta.wavefront.service.WavefrontRemoteService; import javax.validation.constraints.NotNull; -import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.SuperBuilder; -@Data -public class WavefrontManagedAccount { - @NotNull private String name; +@Getter +@Setter +@SuperBuilder +@NoArgsConstructor +public class WavefrontManagedAccount extends AccountCredentials { private String apiToken; @NotNull private RemoteService endpoint; - private List supportedTypes; + @Override + public String getType() { + return "wavefront"; + } + + @JsonIgnore private transient WavefrontRemoteService wavefrontRemoteService; } diff --git a/kayenta-wavefront/src/main/java/com/netflix/kayenta/wavefront/metrics/WavefrontMetricsService.java b/kayenta-wavefront/src/main/java/com/netflix/kayenta/wavefront/metrics/WavefrontMetricsService.java index 2bfaa8d4f..24155539b 100644 --- a/kayenta-wavefront/src/main/java/com/netflix/kayenta/wavefront/metrics/WavefrontMetricsService.java +++ b/kayenta-wavefront/src/main/java/com/netflix/kayenta/wavefront/metrics/WavefrontMetricsService.java @@ -23,8 +23,7 @@ import com.netflix.kayenta.metrics.MetricsService; import com.netflix.kayenta.security.AccountCredentialsRepository; import com.netflix.kayenta.wavefront.canary.WavefrontCanaryScope; -import com.netflix.kayenta.wavefront.security.WavefrontCredentials; -import com.netflix.kayenta.wavefront.security.WavefrontNamedAccountCredentials; +import com.netflix.kayenta.wavefront.config.WavefrontManagedAccount; import com.netflix.kayenta.wavefront.service.WavefrontRemoteService; import com.netflix.kayenta.wavefront.service.WavefrontTimeSeries; import com.netflix.spectator.api.Registry; @@ -92,9 +91,8 @@ public List queryMetrics( CanaryScope canaryScope) throws IOException { WavefrontCanaryScope wavefrontCanaryScope = (WavefrontCanaryScope) canaryScope; - WavefrontNamedAccountCredentials accountCredentials = + WavefrontManagedAccount accountCredentials = accountCredentialsRepository.getRequiredOne(accountName); - WavefrontCredentials credentials = accountCredentials.getCredentials(); WavefrontRemoteService wavefrontRemoteService = accountCredentials.getWavefrontRemoteService(); WavefrontCanaryMetricSetQueryConfig queryConfig = @@ -103,7 +101,7 @@ public List queryMetrics( WavefrontTimeSeries timeSeries = wavefrontRemoteService.fetch( - credentials.getApiToken(), + "Bearer " + accountCredentials.getApiToken(), "Kayenta-Query", query, wavefrontCanaryScope.getStart().toEpochMilli(), diff --git a/kayenta-wavefront/src/main/java/com/netflix/kayenta/wavefront/security/WavefrontCredentials.java b/kayenta-wavefront/src/main/java/com/netflix/kayenta/wavefront/security/WavefrontCredentials.java deleted file mode 100644 index 15b0e09e7..000000000 --- a/kayenta-wavefront/src/main/java/com/netflix/kayenta/wavefront/security/WavefrontCredentials.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2019 Intuit, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.kayenta.wavefront.security; - -import java.util.Optional; -import lombok.Builder; -import lombok.Data; -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; - -@Builder -@Data -@Slf4j -public class WavefrontCredentials { - - private static String applicationVersion = - Optional.ofNullable(WavefrontCredentials.class.getPackage().getImplementationVersion()) - .orElse("Unknown"); - - @NonNull private String apiToken; - - public String getApiToken() { - return "Bearer " + apiToken; - } -} diff --git a/kayenta-wavefront/src/main/java/com/netflix/kayenta/wavefront/security/WavefrontNamedAccountCredentials.java b/kayenta-wavefront/src/main/java/com/netflix/kayenta/wavefront/security/WavefrontNamedAccountCredentials.java deleted file mode 100644 index d19250fa9..000000000 --- a/kayenta-wavefront/src/main/java/com/netflix/kayenta/wavefront/security/WavefrontNamedAccountCredentials.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2019 Intuit, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.kayenta.wavefront.security; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.netflix.kayenta.retrofit.config.RemoteService; -import com.netflix.kayenta.security.AccountCredentials; -import com.netflix.kayenta.wavefront.service.WavefrontRemoteService; -import javax.validation.constraints.NotNull; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.experimental.SuperBuilder; - -@Getter -@Setter -@SuperBuilder -@NoArgsConstructor -public class WavefrontNamedAccountCredentials extends AccountCredentials { - - @NotNull private WavefrontCredentials credentials; - - @NotNull private RemoteService endpoint; - - @Override - public String getType() { - return "wavefront"; - } - - @JsonIgnore WavefrontRemoteService wavefrontRemoteService; -} From 44cdc7ce1ad037b7dc54b5618c8dd8c9c829a60a Mon Sep 17 00:00:00 2001 From: Jason McIntosh Date: Tue, 28 Feb 2023 16:02:34 -0600 Subject: [PATCH 4/4] chore(refactor): Final set of accounts to cleanup --- .../java/com/netflix/kayenta/security/AccountCredentials.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kayenta-core/src/main/java/com/netflix/kayenta/security/AccountCredentials.java b/kayenta-core/src/main/java/com/netflix/kayenta/security/AccountCredentials.java index 1bdd2424c..fd2d45146 100644 --- a/kayenta-core/src/main/java/com/netflix/kayenta/security/AccountCredentials.java +++ b/kayenta-core/src/main/java/com/netflix/kayenta/security/AccountCredentials.java @@ -58,7 +58,7 @@ public T getCredentials() { } @Deprecated - /** Account types other than metric accounts are going away. No longer relevant post SQL. */ + /** Account types other than metric accounts are going away. No longer relevant post SQL. */ public enum Type { METRICS_STORE, OBJECT_STORE,