Skip to content

Commit

Permalink
Rename env vars for better readability. (airbytehq#8447)
Browse files Browse the repository at this point in the history 
* Rename GcsStorageBucket to GcsLogBucket.

* Update all references to GCP_STORAGE_BUCKET to GCS_LOG_BUCKET.

* Undo this for configuration files for older Airbyte versions.

* Clean up Job env vars. (airbytehq#8462)

* Rename MAX_SYNC_JOB_ATTEMPTS to SYNC_JOB_MAX_ATTEMPTS.

* Rename MAX_SYNC_TIMEOUT_DAYS to SYNC_JOB_MAX_TIMEOUT_DAYS.

* Rename WORKER_POD_TOLERATIONS to JOB_POD_TOLERATIONS.

* Rename WORKER_POD_NODE_SELECTORS to JOB_POD_NODE_SELECTORS.

* Rename JOB_IMAGE_PULL_POLICY to JOB_POD_MAIN_CONTAINER_IMAGE_PULL_POLICY.

* Rename JOBS_IMAGE_PULL_SECRET to JOB_POD_MAIN_CONTAINER_IMAGE_PULL_SECRET.

* Rename JOB_SOCAT_IMAGE to JOB_POD_SOCAT_IMAGE.

* Rename JOB_BUSYBOX_IMAGE to JOB_POD_BUSYBOX_IMAGE.

* Rename JOB_CURL_IMAGE to JOB_POD_CURL_IMAGE.

* Rename KUBE_NAMESPACE to JOB_POD_KUBE_NAMESPACE.

* Rename RESOURCE_CPU_REQUEST to JOB_POD_MAIN_CONTAINER_CPU_REQUEST.

* Rename RESOURCE_CPU_LIMIT to JOB_POD_MAIN_CONTAINER_CPU_LIMIT.

* Rename RESOURCE_MEMORY_REQUEST to JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST.

* Rename RESOURCE_MEMORY_LIMIT to JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT.

* Remove worker suffix from created pods to reduce confusion with actual worker pods.

* Use sync instead of worker to name job pods.
  • Loading branch information
@davinchia
davinchia authored Dec 3, 2021
1 parent 5393070 commit 341f505
Show file tree
Hide file tree
Showing 37 changed files with 294 additions and 293 deletions.
20 changes: 10 additions & 10 deletions .env
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,9 @@ SUBMITTER_NUM_THREADS=10

# Job container images
# Usually you should not need to set these, they have defaults already set
JOB_SOCAT_IMAGE=
JOB_BUSYBOX_IMAGE=
JOB_CURL_IMAGE=
JOB_POD_SOCAT_IMAGE=
JOB_POD_BUSYBOX_IMAGE=
JOB_POD_CURL_IMAGE=

# Miscellaneous
TRACKING_STRATEGY=segment
Expand All @@ -68,19 +68,19 @@ AWS_SECRET_ACCESS_KEY=
S3_MINIO_ENDPOINT=
S3_PATH_STYLE_ACCESS=

GCP_STORAGE_BUCKET=
GCS_LOG_BUCKET=

# Docker Resource Limits
RESOURCE_CPU_REQUEST=
RESOURCE_CPU_LIMIT=
RESOURCE_MEMORY_REQUEST=
RESOURCE_MEMORY_LIMIT=
JOB_POD_MAIN_CONTAINER_CPU_REQUEST=
JOB_POD_MAIN_CONTAINER_CPU_LIMIT=
JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST=
JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT=

# Max attempts per sync and max retries per attempt
MAX_SYNC_JOB_ATTEMPTS=3
SYNC_JOB_MAX_ATTEMPTS=3

# Time in days to reach a timeout to cancel the synchronization
MAX_SYNC_TIMEOUT_DAYS=3
SYNC_JOB_MAX_TIMEOUT_DAYS=3

# Set secret persistence store to use. Do not change this for existing installations!
SECRET_PERSISTENCE=NONE
4 changes: 2 additions & 2 deletions .env.dev
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,5 +20,5 @@ HACK_LOCAL_ROOT_PARENT=/tmp
WEBAPP_URL=http://localhost:8000/
API_URL=/api/v1/
INTERNAL_API_HOST=airbyte-server:8001
MAX_SYNC_JOB_ATTEMPTS=3
MAX_SYNC_TIMEOUT_DAYS=3
SYNC_JOB_MAX_ATTEMPTS=3
SYNC_JOB_MAX_TIMEOUT_DAYS=3
4 changes: 2 additions & 2 deletions airbyte-commons/src/main/resources/log4j2.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
<Property name="s3-minio-endpoint">$${env:S3_MINIO_ENDPOINT}</Property>
<Property name="s3-path-style-access">$${env:S3_PATH_STYLE_ACCESS}</Property>

<Property name="gcp-storage-bucket">$${env:GCP_STORAGE_BUCKET}</Property>
<Property name="gcs-log-bucket">$${env:GCS_LOG_BUCKET}</Property>
</Properties>

<Appenders>
Expand Down Expand Up @@ -60,7 +60,7 @@
s3Bucket="${s3-bucket}" s3Path="job-logging${ctx:cloud_job_log_path}" s3Region="${s3-region}"
s3AwsKey="${s3-aws-key}" s3AwsSecret="${s3-aws-secret}"
s3ServiceEndpoint="${s3-minio-endpoint}" s3PathStyleAccess="${s3-path-style-access}"
gcpStorageBucket="${gcp-storage-bucket}" gcpStorageBlobNamePrefix="job-logging${ctx:cloud_job_log_path}">
gcpStorageBucket="${gcs-log-bucket}" gcpStorageBlobNamePrefix="job-logging${ctx:cloud_job_log_path}">
<PatternLayout pattern="${cloud-worker-file-pattern}"/>
</Log4j2Appender>
</Route>
Expand Down
28 changes: 14 additions & 14 deletions airbyte-config/models/src/main/java/io/airbyte/config/Configs.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,33 +72,33 @@ public interface Configs {
String getWebappUrl();

// Jobs
int getMaxSyncJobAttempts();
int getSyncJobMaxAttempts();

int getMaxSyncTimeoutDays();
int getSyncJobMaxTimeoutDays();

List<WorkerPodToleration> getWorkerPodTolerations();
List<TolerationPOJO> getJobPodTolerations();

Map<String, String> getWorkerNodeSelectors();
Map<String, String> getJobPodNodeSelectors();

String getJobImagePullPolicy();
String getJobPodMainContainerImagePullPolicy();

String getJobsImagePullSecret();
String getJobPodMainContainerImagePullSecret();

String getJobSocatImage();
String getJobPodSocatImage();

String getJobBusyboxImage();
String getJobPodBusyboxImage();

String getJobCurlImage();
String getJobPodCurlImage();

String getKubeNamespace();
String getJobPodKubeNamespace();

String getCpuRequest();
String getJobPodMainContainerCpuRequest();

String getCpuLimit();
String getJobPodMainContainerCpuLimit();

String getMemoryRequest();
String getJobPodMainContainerMemoryRequest();

String getMemoryLimit();
String getJobPodMainContainerMemoryLimit();

// Logging/Monitoring/Tracking
LogConfigs getLogConfigs();
Expand Down
106 changes: 53 additions & 53 deletions airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,14 +51,14 @@ public class EnvConfigs implements Configs {
public static final String CONFIG_DATABASE_URL = "CONFIG_DATABASE_URL";
public static final String RUN_DATABASE_MIGRATION_ON_STARTUP = "RUN_DATABASE_MIGRATION_ON_STARTUP";
public static final String WEBAPP_URL = "WEBAPP_URL";
public static final String JOB_IMAGE_PULL_POLICY = "JOB_IMAGE_PULL_POLICY";
public static final String WORKER_POD_TOLERATIONS = "WORKER_POD_TOLERATIONS";
public static final String WORKER_POD_NODE_SELECTORS = "WORKER_POD_NODE_SELECTORS";
public static final String JOB_SOCAT_IMAGE = "JOB_SOCAT_IMAGE";
public static final String JOB_BUSYBOX_IMAGE = "JOB_BUSYBOX_IMAGE";
public static final String JOB_CURL_IMAGE = "JOB_CURL_IMAGE";
public static final String MAX_SYNC_JOB_ATTEMPTS = "MAX_SYNC_JOB_ATTEMPTS";
public static final String MAX_SYNC_TIMEOUT_DAYS = "MAX_SYNC_TIMEOUT_DAYS";
public static final String JOB_POD_MAIN_CONTAINER_IMAGE_PULL_POLICY = "JOB_POD_MAIN_CONTAINER_IMAGE_PULL_POLICY";
public static final String JOB_POD_TOLERATIONS = "JOB_POD_TOLERATIONS";
public static final String JOB_POD_NODE_SELECTORS = "JOB_POD_NODE_SELECTORS";
public static final String JOB_POD_SOCAT_IMAGE = "JOB_POD_SOCAT_IMAGE";
public static final String JOB_POD_BUSYBOX_IMAGE = "JOB_POD_BUSYBOX_IMAGE";
public static final String JOB_POD_CURL_IMAGE = "JOB_POD_CURL_IMAGE";
public static final String SYNC_JOB_MAX_ATTEMPTS = "SYNC_JOB_MAX_ATTEMPTS";
public static final String SYNC_JOB_MAX_TIMEOUT_DAYS = "SYNC_JOB_MAX_TIMEOUT_DAYS";
private static final String MINIMUM_WORKSPACE_RETENTION_DAYS = "MINIMUM_WORKSPACE_RETENTION_DAYS";
private static final String MAXIMUM_WORKSPACE_RETENTION_DAYS = "MAXIMUM_WORKSPACE_RETENTION_DAYS";
private static final String MAXIMUM_WORKSPACE_SIZE_MB = "MAXIMUM_WORKSPACE_SIZE_MB";
Expand All @@ -68,27 +68,27 @@ public class EnvConfigs implements Configs {
public static final String MAX_SYNC_WORKERS = "MAX_SYNC_WORKERS";
private static final String TEMPORAL_HOST = "TEMPORAL_HOST";
private static final String TEMPORAL_WORKER_PORTS = "TEMPORAL_WORKER_PORTS";
private static final String KUBE_NAMESPACE = "KUBE_NAMESPACE";
private static final String JOB_POD_KUBE_NAMESPACE = "JOB_POD_KUBE_NAMESPACE";
private static final String SUBMITTER_NUM_THREADS = "SUBMITTER_NUM_THREADS";
private static final String RESOURCE_CPU_REQUEST = "RESOURCE_CPU_REQUEST";
private static final String RESOURCE_CPU_LIMIT = "RESOURCE_CPU_LIMIT";
private static final String RESOURCE_MEMORY_REQUEST = "RESOURCE_MEMORY_REQUEST";
private static final String RESOURCE_MEMORY_LIMIT = "RESOURCE_MEMORY_LIMIT";
private static final String JOB_POD_MAIN_CONTAINER_CPU_REQUEST = "JOB_POD_MAIN_CONTAINER_CPU_REQUEST";
private static final String JOB_POD_MAIN_CONTAINER_CPU_LIMIT = "JOB_POD_MAIN_CONTAINER_CPU_LIMIT";
private static final String JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST = "JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST";
private static final String JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT = "JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT";
private static final String SECRET_PERSISTENCE = "SECRET_PERSISTENCE";
private static final String JOBS_IMAGE_PULL_SECRET = "JOBS_IMAGE_PULL_SECRET";
private static final String JOB_POD_MAIN_CONTAINER_IMAGE_PULL_SECRET = "JOB_POD_MAIN_CONTAINER_IMAGE_PULL_SECRET";
private static final String PUBLISH_METRICS = "PUBLISH_METRICS";

// defaults
private static final String DEFAULT_SPEC_CACHE_BUCKET = "io-airbyte-cloud-spec-cache";
private static final String DEFAULT_KUBE_NAMESPACE = "default";
private static final String DEFAULT_RESOURCE_REQUIREMENT_CPU = null;
private static final String DEFAULT_RESOURCE_REQUIREMENT_MEMORY = null;
private static final String DEFAULT_JOB_IMAGE_PULL_POLICY = "IfNotPresent";
private static final String DEFAULT_JOB_POD_KUBE_NAMESPACE = "default";
private static final String DEFAULT_JOB_POD_CPU_REQUIREMENT = null;
private static final String DEFAULT_JOB_POD_MEMORY_REQUIREMENT = null;
private static final String DEFAULT_JOB_POD_MAIN_CONTAINER_IMAGE_PULL_POLICY = "IfNotPresent";
private static final String SECRET_STORE_GCP_PROJECT_ID = "SECRET_STORE_GCP_PROJECT_ID";
private static final String SECRET_STORE_GCP_CREDENTIALS = "SECRET_STORE_GCP_CREDENTIALS";
private static final String DEFAULT_JOB_SOCAT_IMAGE = "alpine/socat:1.7.4.1-r1";
private static final String DEFAULT_JOB_BUSYBOX_IMAGE = "busybox:1.28";
private static final String DEFAULT_JOB_CURL_IMAGE = "curlimages/curl:7.77.0";
private static final String DEFAULT_JOB_POD_SOCAT_IMAGE = "alpine/socat:1.7.4.1-r1";
private static final String DEFAULT_JOB_POD_BUSYBOX_IMAGE = "busybox:1.28";
private static final String DEFAULT_JOB_POD_CURL_IMAGE = "curlimages/curl:7.77.0";
private static final long DEFAULT_MINIMUM_WORKSPACE_RETENTION_DAYS = 1;
private static final long DEFAULT_MAXIMUM_WORKSPACE_RETENTION_DAYS = 60;
private static final long DEFAULT_MAXIMUM_WORKSPACE_SIZE_MB = 5000;
Expand All @@ -115,7 +115,7 @@ public EnvConfigs() {
getEnvOrDefault(LogClientSingleton.AWS_ACCESS_KEY_ID, ""),
getEnvOrDefault(LogClientSingleton.AWS_SECRET_ACCESS_KEY, ""),
getEnvOrDefault(LogClientSingleton.S3_MINIO_ENDPOINT, ""),
getEnvOrDefault(LogClientSingleton.GCP_STORAGE_BUCKET, ""),
getEnvOrDefault(LogClientSingleton.GCS_LOG_BUCKET, ""),
getEnvOrDefault(LogClientSingleton.GOOGLE_APPLICATION_CREDENTIALS, ""));
}

Expand Down Expand Up @@ -268,13 +268,13 @@ public String getWebappUrl() {

// Jobs
@Override
public int getMaxSyncJobAttempts() {
return Integer.parseInt(getEnvOrDefault(MAX_SYNC_JOB_ATTEMPTS, "3"));
public int getSyncJobMaxAttempts() {
return Integer.parseInt(getEnvOrDefault(SYNC_JOB_MAX_ATTEMPTS, "3"));
}

@Override
public int getMaxSyncTimeoutDays() {
return Integer.parseInt(getEnvOrDefault(MAX_SYNC_TIMEOUT_DAYS, "3"));
public int getSyncJobMaxTimeoutDays() {
return Integer.parseInt(getEnvOrDefault(SYNC_JOB_MAX_TIMEOUT_DAYS, "3"));
}

/**
Expand All @@ -294,28 +294,28 @@ public int getMaxSyncTimeoutDays() {
* @return list of WorkerPodToleration parsed from env
*/
@Override
public List<WorkerPodToleration> getWorkerPodTolerations() {
final String tolerationsStr = getEnvOrDefault(WORKER_POD_TOLERATIONS, "");
public List<TolerationPOJO> getJobPodTolerations() {
final String tolerationsStr = getEnvOrDefault(JOB_POD_TOLERATIONS, "");

final Stream<String> tolerations = Strings.isNullOrEmpty(tolerationsStr) ? Stream.of()
: Splitter.on(";")
.splitToStream(tolerationsStr)
.filter(tolerationStr -> !Strings.isNullOrEmpty(tolerationStr));

return tolerations
.map(this::workerPodToleration)
.map(this::parseToleration)
.filter(Objects::nonNull)
.collect(Collectors.toList());
}

private WorkerPodToleration workerPodToleration(final String tolerationStr) {
private TolerationPOJO parseToleration(final String tolerationStr) {
final Map<String, String> tolerationMap = Splitter.on(",")
.splitToStream(tolerationStr)
.map(s -> s.split("="))
.collect(Collectors.toMap(s -> s[0], s -> s[1]));

if (tolerationMap.containsKey("key") && tolerationMap.containsKey("effect") && tolerationMap.containsKey("operator")) {
return new WorkerPodToleration(
return new TolerationPOJO(
tolerationMap.get("key"),
tolerationMap.get("effect"),
tolerationMap.get("value"),
Expand All @@ -339,17 +339,17 @@ private WorkerPodToleration workerPodToleration(final String tolerationStr) {
* @return map containing kv pairs of node selectors
*/
@Override
public Map<String, String> getWorkerNodeSelectors() {
public Map<String, String> getJobPodNodeSelectors() {
return Splitter.on(",")
.splitToStream(getEnvOrDefault(WORKER_POD_NODE_SELECTORS, ""))
.splitToStream(getEnvOrDefault(JOB_POD_NODE_SELECTORS, ""))
.filter(s -> !Strings.isNullOrEmpty(s) && s.contains("="))
.map(s -> s.split("="))
.collect(Collectors.toMap(s -> s[0], s -> s[1]));
}

@Override
public String getJobImagePullPolicy() {
return getEnvOrDefault(JOB_IMAGE_PULL_POLICY, DEFAULT_JOB_IMAGE_PULL_POLICY);
public String getJobPodMainContainerImagePullPolicy() {
return getEnvOrDefault(JOB_POD_MAIN_CONTAINER_IMAGE_PULL_POLICY, DEFAULT_JOB_POD_MAIN_CONTAINER_IMAGE_PULL_POLICY);
}

/**
Expand All @@ -358,48 +358,48 @@ public String getJobImagePullPolicy() {
* no-op value.
*/
@Override
public String getJobsImagePullSecret() {
return getEnvOrDefault(JOBS_IMAGE_PULL_SECRET, "");
public String getJobPodMainContainerImagePullSecret() {
return getEnvOrDefault(JOB_POD_MAIN_CONTAINER_IMAGE_PULL_SECRET, "");
}

@Override
public String getJobSocatImage() {
return getEnvOrDefault(JOB_SOCAT_IMAGE, DEFAULT_JOB_SOCAT_IMAGE);
public String getJobPodSocatImage() {
return getEnvOrDefault(JOB_POD_SOCAT_IMAGE, DEFAULT_JOB_POD_SOCAT_IMAGE);
}

@Override
public String getJobBusyboxImage() {
return getEnvOrDefault(JOB_BUSYBOX_IMAGE, DEFAULT_JOB_BUSYBOX_IMAGE);
public String getJobPodBusyboxImage() {
return getEnvOrDefault(JOB_POD_BUSYBOX_IMAGE, DEFAULT_JOB_POD_BUSYBOX_IMAGE);
}

@Override
public String getJobCurlImage() {
return getEnvOrDefault(JOB_CURL_IMAGE, DEFAULT_JOB_CURL_IMAGE);
public String getJobPodCurlImage() {
return getEnvOrDefault(JOB_POD_CURL_IMAGE, DEFAULT_JOB_POD_CURL_IMAGE);
}

@Override
public String getKubeNamespace() {
return getEnvOrDefault(KUBE_NAMESPACE, DEFAULT_KUBE_NAMESPACE);
public String getJobPodKubeNamespace() {
return getEnvOrDefault(JOB_POD_KUBE_NAMESPACE, DEFAULT_JOB_POD_KUBE_NAMESPACE);
}

@Override
public String getCpuRequest() {
return getEnvOrDefault(RESOURCE_CPU_REQUEST, DEFAULT_RESOURCE_REQUIREMENT_CPU);
public String getJobPodMainContainerCpuRequest() {
return getEnvOrDefault(JOB_POD_MAIN_CONTAINER_CPU_REQUEST, DEFAULT_JOB_POD_CPU_REQUIREMENT);
}

@Override
public String getCpuLimit() {
return getEnvOrDefault(RESOURCE_CPU_LIMIT, DEFAULT_RESOURCE_REQUIREMENT_CPU);
public String getJobPodMainContainerCpuLimit() {
return getEnvOrDefault(JOB_POD_MAIN_CONTAINER_CPU_LIMIT, DEFAULT_JOB_POD_CPU_REQUIREMENT);
}

@Override
public String getMemoryRequest() {
return getEnvOrDefault(RESOURCE_MEMORY_REQUEST, DEFAULT_RESOURCE_REQUIREMENT_MEMORY);
public String getJobPodMainContainerMemoryRequest() {
return getEnvOrDefault(JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST, DEFAULT_JOB_POD_MEMORY_REQUIREMENT);
}

@Override
public String getMemoryLimit() {
return getEnvOrDefault(RESOURCE_MEMORY_LIMIT, DEFAULT_RESOURCE_REQUIREMENT_MEMORY);
public String getJobPodMainContainerMemoryLimit() {
return getEnvOrDefault(JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT, DEFAULT_JOB_POD_MEMORY_REQUIREMENT);
}

// Logging/Monitoring/Tracking
Expand Down
6 changes: 3 additions & 3 deletions ...o/airbyte/config/WorkerPodToleration.java → ...ava/io/airbyte/config/TolerationPOJO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,14 @@
/**
* Represents a minimal io.fabric8.kubernetes.api.model.Toleration
*/
public class WorkerPodToleration {
public class TolerationPOJO {

private final String key;
private final String effect;
private final String value;
private final String operator;

public WorkerPodToleration(final String key, final String effect, final String value, final String operator) {
public TolerationPOJO(final String key, final String effect, final String value, final String operator) {
this.key = key;
this.effect = effect;
this.value = value;
Expand Down Expand Up @@ -47,7 +47,7 @@ public boolean equals(final Object o) {
if (o == null || getClass() != o.getClass()) {
return false;
}
final WorkerPodToleration that = (WorkerPodToleration) o;
final TolerationPOJO that = (TolerationPOJO) o;
return Objects.equals(key, that.key) && Objects.equals(effect, that.effect)
&& Objects.equals(value, that.value) && Objects.equals(operator,
that.operator);
Expand Down
2 changes: 1 addition & 1 deletion airbyte-config/models/src/main/java/io/airbyte/config/helpers/CloudLogs.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ private static boolean hasS3Configuration(final LogConfigs configs) {
}

private static boolean hasGcpConfiguration(final LogConfigs configs) {
return !configs.getGcpStorageBucket().isBlank() &&
return !configs.getGcsLogBucket().isBlank() &&
!configs.getGoogleApplicationCredentials().isBlank();
}

Expand Down
8 changes: 4 additions & 4 deletions airbyte-config/models/src/main/java/io/airbyte/config/helpers/GcsLogs.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ static File getFile(final LogConfigs configs, final String logPath, final int pa

LOGGER.debug("Start GCS list request.");
final Page<Blob> blobs = GCS.list(
configs.getGcpStorageBucket(),
configs.getGcsLogBucket(),
Storage.BlobListOption.prefix(logPath),
Storage.BlobListOption.pageSize(pageSize));

Expand All @@ -63,7 +63,7 @@ public List<String> tailCloudLog(final LogConfigs configs, final String logPath,

LOGGER.debug("Start GCS list request.");
final Page<Blob> blobs = GCS.list(
configs.getGcpStorageBucket(),
configs.getGcsLogBucket(),
Storage.BlobListOption.prefix(logPath));

final var ascendingTimestampBlobs = new ArrayList<Blob>();
Expand Down Expand Up @@ -102,7 +102,7 @@ public void deleteLogs(final LogConfigs configs, final String logPath) {
createGcsClientIfNotExists(configs);

LOGGER.debug("Start GCS list and delete request.");
final Page<Blob> blobs = GCS.list(configs.getGcpStorageBucket(), Storage.BlobListOption.prefix(logPath));
final Page<Blob> blobs = GCS.list(configs.getGcsLogBucket(), Storage.BlobListOption.prefix(logPath));
for (final Blob blob : blobs.iterateAll()) {
blob.delete(BlobSourceOption.generationMatch());
}
Expand All @@ -111,7 +111,7 @@ public void deleteLogs(final LogConfigs configs, final String logPath) {

private static void createGcsClientIfNotExists(final LogConfigs configs) {
if (GCS == null) {
Preconditions.checkNotNull(configs.getGcpStorageBucket());
Preconditions.checkNotNull(configs.getGcsLogBucket());
Preconditions.checkNotNull(configs.getGoogleApplicationCredentials());

GCS = StorageOptions.getDefaultInstance().getService();
Expand Down
Loading

0 comments on commit 341f505

Please sign in to comment.