diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..db8125b1 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,8 @@ +repos: +- repo: https://github.com/jguttman94/pre-commit-gradle + rev: efaf1704425cc50df91d63920e8353d9d081a622 + hooks: + - id: gradle-task + args: [-w, -o, spotlessApply] + verbose: true + diff --git a/README.md b/README.md index 2cddce91..fce56e27 100644 --- a/README.md +++ b/README.md @@ -621,3 +621,17 @@ Fluency fluency = builder.build(); ##### Other configurations Some of other usages are same as ingestion to Fluentd. See `Ingestion to Fluentd > Usage` above. + +## Development + +### Pre-commit hook + +This project uses [pre-commit](https://pre-commit.com/) to automate code format and so on as much as possible. If you're interested in the development of this project, please [install pre-commit](https://pre-commit.com/#installation) and the git hook script as follows. + +``` +$ ls -a .pre-commit-config.yaml +.pre-commit-config.yaml +$ pre-commit install +``` + +The code formatter is automatically executed when committing files. A commit will fail and be formatted by the formatter when any invalid code format is detected. Try to commit the change again. diff --git a/build.gradle.kts b/build.gradle.kts index 562d7fd9..4a833312 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -14,6 +14,7 @@ plugins { `maven-publish` id("com.github.kt3k.coveralls") version "2.12.2" id("com.github.johnrengelman.shadow") version "8.1.1" + id("com.diffplug.spotless") version "6.13.0" } subprojects { @@ -26,6 +27,7 @@ subprojects { apply(plugin = "maven-publish") apply(plugin = "jacoco") apply(plugin = "com.github.kt3k.coveralls") + apply(plugin = "com.diffplug.spotless") repositories { mavenCentral() @@ -187,5 +189,14 @@ subprojects { tasks.coveralls { dependsOn(jacocoRootReport) } + + spotless { + java { + target("src/*/java/**/*.java") + importOrder() + removeUnusedImports() + googleJavaFormat("1.7") + } + } } diff --git a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/FluencyBuilderForAwsS3.java b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/FluencyBuilderForAwsS3.java index e63d875d..e1705029 100644 --- a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/FluencyBuilderForAwsS3.java +++ b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/FluencyBuilderForAwsS3.java @@ -16,367 +16,339 @@ package org.komamitsu.fluency.aws.s3; +import java.time.ZoneId; +import java.util.List; import org.komamitsu.fluency.Fluency; +import org.komamitsu.fluency.aws.s3.ingester.AwsS3Ingester; import org.komamitsu.fluency.aws.s3.ingester.DefaultS3DestinationDecider; import org.komamitsu.fluency.aws.s3.ingester.S3DestinationDecider; +import org.komamitsu.fluency.aws.s3.ingester.sender.AwsS3Sender; import org.komamitsu.fluency.aws.s3.recordformat.AwsS3RecordFormatter; import org.komamitsu.fluency.aws.s3.recordformat.CsvRecordFormatter; import org.komamitsu.fluency.aws.s3.recordformat.JsonlRecordFormatter; import org.komamitsu.fluency.aws.s3.recordformat.MessagePackRecordFormatter; -import org.komamitsu.fluency.aws.s3.ingester.AwsS3Ingester; -import org.komamitsu.fluency.aws.s3.ingester.sender.AwsS3Sender; import org.msgpack.core.annotations.VisibleForTesting; import software.amazon.awssdk.services.s3.S3Client; -import java.time.ZoneId; -import java.util.List; - -public class FluencyBuilderForAwsS3 - extends org.komamitsu.fluency.FluencyBuilder -{ - private FormatType formatType; - private List formatCsvColumnNames; - private String awsEndpoint; - private String awsRegion; - private String awsAccessKeyId; - private String awsSecretAccessKey; - private Integer senderRetryMax; - private Integer senderRetryIntervalMillis; - private Integer senderMaxRetryIntervalMillis; - private Float senderRetryFactor; - private Integer senderWorkBufSize; - private boolean compressionEnabled = true; - private String s3KeyPrefix; - private String s3KeySuffix; - private ZoneId s3KeyTimeZoneId; - private S3DestinationDecider customS3DestinationDecider; - - public enum FormatType { - MESSAGE_PACK, - JSONL, - CSV - } - - public FluencyBuilderForAwsS3() - { - setBufferChunkRetentionTimeMillis(30 * 1000); - setBufferChunkInitialSize(4 * 1024 * 1024); - setBufferChunkRetentionSize(64 * 1024 * 1024); - } - - public FormatType getFormatType() - { - return formatType; - } - - public void setFormatType(FormatType formatType) - { - this.formatType = formatType; - } - - public List getFormatCsvColumnNames() - { - return formatCsvColumnNames; - } - - public void setFormatCsvColumnNames(List formatCsvColumnNames) - { - this.formatCsvColumnNames = formatCsvColumnNames; - } - - public String getAwsEndpoint() - { - return awsEndpoint; - } - - public void setAwsEndpoint(String awsEndpoint) - { - this.awsEndpoint = awsEndpoint; - } - - public String getAwsRegion() - { - return awsRegion; - } - - public void setAwsRegion(String awsRegion) - { - this.awsRegion = awsRegion; - } - - public String getAwsAccessKeyId() - { - return awsAccessKeyId; - } - - public void setAwsAccessKeyId(String senderAwsAccessKeyId) - { - this.awsAccessKeyId = senderAwsAccessKeyId; - } - - public String getAwsSecretAccessKey() - { - return awsSecretAccessKey; - } - - public void setAwsSecretAccessKey(String senderAwsSecretAccessKey) - { - this.awsSecretAccessKey = senderAwsSecretAccessKey; - } +public class FluencyBuilderForAwsS3 extends org.komamitsu.fluency.FluencyBuilder { + private FormatType formatType; + private List formatCsvColumnNames; + private String awsEndpoint; + private String awsRegion; + private String awsAccessKeyId; + private String awsSecretAccessKey; + private Integer senderRetryMax; + private Integer senderRetryIntervalMillis; + private Integer senderMaxRetryIntervalMillis; + private Float senderRetryFactor; + private Integer senderWorkBufSize; + private boolean compressionEnabled = true; + private String s3KeyPrefix; + private String s3KeySuffix; + private ZoneId s3KeyTimeZoneId; + private S3DestinationDecider customS3DestinationDecider; + + public enum FormatType { + MESSAGE_PACK, + JSONL, + CSV + } + + public FluencyBuilderForAwsS3() { + setBufferChunkRetentionTimeMillis(30 * 1000); + setBufferChunkInitialSize(4 * 1024 * 1024); + setBufferChunkRetentionSize(64 * 1024 * 1024); + } + + public FormatType getFormatType() { + return formatType; + } + + public void setFormatType(FormatType formatType) { + this.formatType = formatType; + } + + public List getFormatCsvColumnNames() { + return formatCsvColumnNames; + } + + public void setFormatCsvColumnNames(List formatCsvColumnNames) { + this.formatCsvColumnNames = formatCsvColumnNames; + } + + public String getAwsEndpoint() { + return awsEndpoint; + } + + public void setAwsEndpoint(String awsEndpoint) { + this.awsEndpoint = awsEndpoint; + } + + public String getAwsRegion() { + return awsRegion; + } + + public void setAwsRegion(String awsRegion) { + this.awsRegion = awsRegion; + } + + public String getAwsAccessKeyId() { + return awsAccessKeyId; + } + + public void setAwsAccessKeyId(String senderAwsAccessKeyId) { + this.awsAccessKeyId = senderAwsAccessKeyId; + } + + public String getAwsSecretAccessKey() { + return awsSecretAccessKey; + } + + public void setAwsSecretAccessKey(String senderAwsSecretAccessKey) { + this.awsSecretAccessKey = senderAwsSecretAccessKey; + } + + public Integer getSenderRetryMax() { + return senderRetryMax; + } + + public void setSenderRetryMax(Integer senderRetryMax) { + this.senderRetryMax = senderRetryMax; + } + + public Integer getSenderRetryIntervalMillis() { + return senderRetryIntervalMillis; + } + + public void setSenderRetryIntervalMillis(Integer senderRetryIntervalMillis) { + this.senderRetryIntervalMillis = senderRetryIntervalMillis; + } + + public Integer getSenderMaxRetryIntervalMillis() { + return senderMaxRetryIntervalMillis; + } + + public void setSenderMaxRetryIntervalMillis(Integer senderMaxRetryIntervalMillis) { + this.senderMaxRetryIntervalMillis = senderMaxRetryIntervalMillis; + } + + public Float getSenderRetryFactor() { + return senderRetryFactor; + } + + public void setSenderRetryFactor(Float senderRetryFactor) { + this.senderRetryFactor = senderRetryFactor; + } + + public Integer getSenderWorkBufSize() { + return senderWorkBufSize; + } + + public void setSenderWorkBufSize(Integer senderWorkBufSize) { + this.senderWorkBufSize = senderWorkBufSize; + } + + public boolean isCompressionEnabled() { + return compressionEnabled; + } + + public void setCompressionEnabled(boolean compressionEnabled) { + this.compressionEnabled = compressionEnabled; + } + + public String getS3KeyPrefix() { + return s3KeyPrefix; + } + + public void setS3KeyPrefix(String s3KeyPrefix) { + this.s3KeyPrefix = s3KeyPrefix; + } + + public String getS3KeySuffix() { + return s3KeySuffix; + } + + public void setS3KeySuffix(String s3KeySuffix) { + this.s3KeySuffix = s3KeySuffix; + } - public Integer getSenderRetryMax() - { - return senderRetryMax; - } + public ZoneId getS3KeyTimeZoneId() { + return s3KeyTimeZoneId; + } - public void setSenderRetryMax(Integer senderRetryMax) - { - this.senderRetryMax = senderRetryMax; - } + public void setS3KeyTimeZoneId(ZoneId s3KeyTimeZoneId) { + this.s3KeyTimeZoneId = s3KeyTimeZoneId; + } - public Integer getSenderRetryIntervalMillis() - { - return senderRetryIntervalMillis; - } + public S3DestinationDecider getCustomS3DestinationDecider() { + return customS3DestinationDecider; + } - public void setSenderRetryIntervalMillis(Integer senderRetryIntervalMillis) - { - this.senderRetryIntervalMillis = senderRetryIntervalMillis; - } + public void setCustomS3DestinationDecider(S3DestinationDecider customS3DestinationDecider) { + this.customS3DestinationDecider = customS3DestinationDecider; + } - public Integer getSenderMaxRetryIntervalMillis() - { - return senderMaxRetryIntervalMillis; - } + private String defaultKeyPrefix(AwsS3RecordFormatter recordFormatter) { + return "." + recordFormatter.formatName(); + } - public void setSenderMaxRetryIntervalMillis(Integer senderMaxRetryIntervalMillis) - { - this.senderMaxRetryIntervalMillis = senderMaxRetryIntervalMillis; - } + public Fluency build(AwsS3RecordFormatter recordFormatter, AwsS3Sender.Config senderConfig) { + return buildFromIngester(recordFormatter, createIngester(recordFormatter, senderConfig)); + } - public Float getSenderRetryFactor() - { - return senderRetryFactor; - } + public Fluency build(AwsS3RecordFormatter recordFormatter) { + AwsS3Sender.Config senderConfig = createSenderConfig(); - public void setSenderRetryFactor(Float senderRetryFactor) - { - this.senderRetryFactor = senderRetryFactor; - } + return build(recordFormatter, senderConfig); + } - public Integer getSenderWorkBufSize() - { - return senderWorkBufSize; - } + public Fluency build() { + return build(createRecordFormatter()); + } - public void setSenderWorkBufSize(Integer senderWorkBufSize) - { - this.senderWorkBufSize = senderWorkBufSize; - } + @VisibleForTesting + AwsS3Sender createSender(AwsS3Sender.Config senderConfig) { + return new AwsS3Sender(S3Client.builder(), senderConfig); + } - public boolean isCompressionEnabled() - { - return compressionEnabled; - } + private AwsS3Ingester createIngester( + AwsS3RecordFormatter recordFormatter, AwsS3Sender.Config senderConfig) { + S3DestinationDecider s3DestinationDecider; + if (getCustomS3DestinationDecider() == null) { + DefaultS3DestinationDecider.Config config = new DefaultS3DestinationDecider.Config(); - public void setCompressionEnabled(boolean compressionEnabled) - { - this.compressionEnabled = compressionEnabled; - } + if (getS3KeyPrefix() != null) { + config.setKeyPrefix(getS3KeyPrefix()); + } - public String getS3KeyPrefix() - { - return s3KeyPrefix; - } + if (getS3KeySuffix() != null) { + config.setKeySuffix(getS3KeySuffix()); + } else { + config.setKeySuffix( + defaultKeyPrefix(recordFormatter) + (isCompressionEnabled() ? ".gz" : "")); + } - public void setS3KeyPrefix(String s3KeyPrefix) - { - this.s3KeyPrefix = s3KeyPrefix; + if (getS3KeyTimeZoneId() != null) { + config.setZoneId(getS3KeyTimeZoneId()); + } + s3DestinationDecider = new DefaultS3DestinationDecider(config); + } else { + s3DestinationDecider = getCustomS3DestinationDecider(); } - public String getS3KeySuffix() - { - return s3KeySuffix; - } + AwsS3Sender sender = createSender(senderConfig); + return new AwsS3Ingester(sender, s3DestinationDecider); + } - public void setS3KeySuffix(String s3KeySuffix) - { - this.s3KeySuffix = s3KeySuffix; + private AwsS3RecordFormatter createRecordFormatter() { + if (formatType == null) { + throw new IllegalArgumentException("format type must be set"); } - public ZoneId getS3KeyTimeZoneId() - { - return s3KeyTimeZoneId; + AwsS3RecordFormatter recordFormatter; + switch (getFormatType()) { + case MESSAGE_PACK: + recordFormatter = new MessagePackRecordFormatter(); + break; + case JSONL: + recordFormatter = new JsonlRecordFormatter(); + break; + case CSV: + CsvRecordFormatter.Config config = new CsvRecordFormatter.Config(); + config.setColumnNames(getFormatCsvColumnNames()); + recordFormatter = new CsvRecordFormatter(config); + break; + default: + throw new IllegalArgumentException("Unexpected format type: " + getFormatType()); } - public void setS3KeyTimeZoneId(ZoneId s3KeyTimeZoneId) - { - this.s3KeyTimeZoneId = s3KeyTimeZoneId; - } + return recordFormatter; + } - public S3DestinationDecider getCustomS3DestinationDecider() - { - return customS3DestinationDecider; - } + private AwsS3Sender.Config createSenderConfig() { + AwsS3Sender.Config config = new AwsS3Sender.Config(); - public void setCustomS3DestinationDecider(S3DestinationDecider customS3DestinationDecider) - { - this.customS3DestinationDecider = customS3DestinationDecider; + if (getAwsEndpoint() != null) { + config.setEndpoint(getAwsEndpoint()); } - private String defaultKeyPrefix(AwsS3RecordFormatter recordFormatter) - { - return "." + recordFormatter.formatName(); + if (getAwsRegion() != null) { + config.setRegion(getAwsRegion()); } - public Fluency build(AwsS3RecordFormatter recordFormatter, AwsS3Sender.Config senderConfig) - { - return buildFromIngester(recordFormatter, createIngester(recordFormatter, senderConfig)); + if (getAwsAccessKeyId() != null) { + config.setAwsAccessKeyId(getAwsAccessKeyId()); } - public Fluency build(AwsS3RecordFormatter recordFormatter) - { - AwsS3Sender.Config senderConfig = createSenderConfig(); - - - return build(recordFormatter, senderConfig); + if (getAwsSecretAccessKey() != null) { + config.setAwsSecretAccessKey(getAwsSecretAccessKey()); } - public Fluency build() - { - return build(createRecordFormatter()); + if (getSenderRetryMax() != null) { + config.setRetryMax(getSenderRetryMax()); } - @VisibleForTesting - AwsS3Sender createSender(AwsS3Sender.Config senderConfig) - { - return new AwsS3Sender(S3Client.builder(), senderConfig); + if (getSenderRetryIntervalMillis() != null) { + config.setRetryIntervalMs(getSenderRetryIntervalMillis()); } - private AwsS3Ingester createIngester(AwsS3RecordFormatter recordFormatter, AwsS3Sender.Config senderConfig) - { - S3DestinationDecider s3DestinationDecider; - if (getCustomS3DestinationDecider() == null) { - DefaultS3DestinationDecider.Config config = new DefaultS3DestinationDecider.Config(); - - if (getS3KeyPrefix() != null) { - config.setKeyPrefix(getS3KeyPrefix()); - } - - if (getS3KeySuffix() != null) { - config.setKeySuffix(getS3KeySuffix()); - } - else { - config.setKeySuffix( - defaultKeyPrefix(recordFormatter) + (isCompressionEnabled() ? ".gz" : "")); - } - - if (getS3KeyTimeZoneId() != null) { - config.setZoneId(getS3KeyTimeZoneId()); - } - s3DestinationDecider = new DefaultS3DestinationDecider(config); - } - else { - s3DestinationDecider = getCustomS3DestinationDecider(); - } - - AwsS3Sender sender = createSender(senderConfig); - return new AwsS3Ingester(sender, s3DestinationDecider); + if (getSenderMaxRetryIntervalMillis() != null) { + config.setMaxRetryIntervalMs(getSenderMaxRetryIntervalMillis()); } - private AwsS3RecordFormatter createRecordFormatter() - { - if (formatType == null) { - throw new IllegalArgumentException("format type must be set"); - } - - AwsS3RecordFormatter recordFormatter; - switch (getFormatType()) { - case MESSAGE_PACK: - recordFormatter = new MessagePackRecordFormatter(); - break; - case JSONL: - recordFormatter = new JsonlRecordFormatter(); - break; - case CSV: - CsvRecordFormatter.Config config = new CsvRecordFormatter.Config(); - config.setColumnNames(getFormatCsvColumnNames()); - recordFormatter = new CsvRecordFormatter(config); - break; - default: - throw new IllegalArgumentException("Unexpected format type: " + getFormatType()); - } - - return recordFormatter; + if (getSenderRetryFactor() != null) { + config.setRetryFactor(getSenderRetryFactor()); } - private AwsS3Sender.Config createSenderConfig() - { - AwsS3Sender.Config config = new AwsS3Sender.Config(); - - if (getAwsEndpoint() != null) { - config.setEndpoint(getAwsEndpoint()); - } - - if (getAwsRegion()!= null) { - config.setRegion(getAwsRegion()); - } - - if (getAwsAccessKeyId() != null) { - config.setAwsAccessKeyId(getAwsAccessKeyId()); - } - - if (getAwsSecretAccessKey() != null) { - config.setAwsSecretAccessKey(getAwsSecretAccessKey()); - } - - if (getSenderRetryMax() != null) { - config.setRetryMax(getSenderRetryMax()); - } - - if (getSenderRetryIntervalMillis() != null) { - config.setRetryIntervalMs(getSenderRetryIntervalMillis()); - } + if (getErrorHandler() != null) { + config.setErrorHandler(getErrorHandler()); + } + + if (getSenderWorkBufSize() != null) { + config.setWorkBufSize(getSenderWorkBufSize()); + } + + config.setCompressionEnabled(isCompressionEnabled()); + + return config; + } - if (getSenderMaxRetryIntervalMillis() != null) { - config.setMaxRetryIntervalMs(getSenderMaxRetryIntervalMillis()); - } - - if (getSenderRetryFactor() != null) { - config.setRetryFactor(getSenderRetryFactor()); - } - - if (getErrorHandler() != null) { - config.setErrorHandler(getErrorHandler()); - } - - if (getSenderWorkBufSize() != null) { - config.setWorkBufSize(getSenderWorkBufSize()); - } - - config.setCompressionEnabled(isCompressionEnabled()); - - return config; - } - - @Override - public String toString() - { - return "FluencyBuilderForAwsS3{" + - "formatType=" + formatType + - ", formatCsvColumnNames=" + formatCsvColumnNames + - ", awsEndpoint='" + awsEndpoint + '\'' + - ", awsRegion='" + awsRegion + '\'' + - ", senderRetryMax=" + senderRetryMax + - ", senderRetryIntervalMillis=" + senderRetryIntervalMillis + - ", senderMaxRetryIntervalMillis=" + senderMaxRetryIntervalMillis + - ", senderRetryFactor=" + senderRetryFactor + - ", senderWorkBufSize=" + senderWorkBufSize + - ", compressionEnabled=" + compressionEnabled + - ", s3KeyPrefix='" + s3KeyPrefix + '\'' + - ", s3KeySuffix='" + s3KeySuffix + '\'' + - ", s3KeyTimeZoneId=" + s3KeyTimeZoneId + - ", customS3DestinationDecider=" + customS3DestinationDecider + - "} " + super.toString(); - } + @Override + public String toString() { + return "FluencyBuilderForAwsS3{" + + "formatType=" + + formatType + + ", formatCsvColumnNames=" + + formatCsvColumnNames + + ", awsEndpoint='" + + awsEndpoint + + '\'' + + ", awsRegion='" + + awsRegion + + '\'' + + ", senderRetryMax=" + + senderRetryMax + + ", senderRetryIntervalMillis=" + + senderRetryIntervalMillis + + ", senderMaxRetryIntervalMillis=" + + senderMaxRetryIntervalMillis + + ", senderRetryFactor=" + + senderRetryFactor + + ", senderWorkBufSize=" + + senderWorkBufSize + + ", compressionEnabled=" + + compressionEnabled + + ", s3KeyPrefix='" + + s3KeyPrefix + + '\'' + + ", s3KeySuffix='" + + s3KeySuffix + + '\'' + + ", s3KeyTimeZoneId=" + + s3KeyTimeZoneId + + ", customS3DestinationDecider=" + + customS3DestinationDecider + + "} " + + super.toString(); + } } diff --git a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/AwsS3Ingester.java b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/AwsS3Ingester.java index fbd011d7..101ef2cc 100644 --- a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/AwsS3Ingester.java +++ b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/AwsS3Ingester.java @@ -16,55 +16,46 @@ package org.komamitsu.fluency.aws.s3.ingester; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.time.Instant; import org.komamitsu.fluency.aws.s3.ingester.sender.AwsS3Sender; import org.komamitsu.fluency.ingester.Ingester; import org.komamitsu.fluency.ingester.sender.Sender; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.time.Instant; - -public class AwsS3Ingester - implements Ingester -{ - private static final Logger LOG = LoggerFactory.getLogger(AwsS3Ingester.class); - private final AwsS3Sender sender; - private final S3DestinationDecider s3DestinationDecider; - - public AwsS3Ingester(AwsS3Sender sender, S3DestinationDecider s3DestinationDecider) - { - this.sender = sender; - this.s3DestinationDecider = s3DestinationDecider; - } - - @Override - public void ingest(String tag, ByteBuffer dataBuffer) - throws IOException - { - S3DestinationDecider.S3Destination s3Destination = - s3DestinationDecider.decide(tag, Instant.now()); - String bucket = s3Destination.getBucket(); - String key = s3Destination.getKey(); - - sender.send(bucket, key, dataBuffer); - } - - @Override - public Sender getSender() - { - return sender; - } - - public S3DestinationDecider getS3DestinationDecider() - { - return s3DestinationDecider; - } - - @Override - public void close() - { - sender.close(); - } +public class AwsS3Ingester implements Ingester { + private static final Logger LOG = LoggerFactory.getLogger(AwsS3Ingester.class); + private final AwsS3Sender sender; + private final S3DestinationDecider s3DestinationDecider; + + public AwsS3Ingester(AwsS3Sender sender, S3DestinationDecider s3DestinationDecider) { + this.sender = sender; + this.s3DestinationDecider = s3DestinationDecider; + } + + @Override + public void ingest(String tag, ByteBuffer dataBuffer) throws IOException { + S3DestinationDecider.S3Destination s3Destination = + s3DestinationDecider.decide(tag, Instant.now()); + String bucket = s3Destination.getBucket(); + String key = s3Destination.getKey(); + + sender.send(bucket, key, dataBuffer); + } + + @Override + public Sender getSender() { + return sender; + } + + public S3DestinationDecider getS3DestinationDecider() { + return s3DestinationDecider; + } + + @Override + public void close() { + sender.close(); + } } diff --git a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/DefaultS3DestinationDecider.java b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/DefaultS3DestinationDecider.java index cfe1157e..5313111f 100644 --- a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/DefaultS3DestinationDecider.java +++ b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/DefaultS3DestinationDecider.java @@ -16,111 +16,97 @@ package org.komamitsu.fluency.aws.s3.ingester; -import org.komamitsu.fluency.validation.Validatable; - import java.time.Instant; import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import org.komamitsu.fluency.validation.Validatable; + +public class DefaultS3DestinationDecider implements S3DestinationDecider { + private static final DateTimeFormatter FORMATTER = + DateTimeFormatter.ofPattern("yyyy/MM/dd/HH/mm-ss-SSSSSS"); + private final Config config; + + public DefaultS3DestinationDecider(Config config) { + config.validateValues(); + this.config = config; + } + + protected String getBucket(String tag) { + return tag; + } + + protected String getKeyBase(Instant time) { + return ZonedDateTime.ofInstant(time, getZoneId()).format(FORMATTER); + } + + @Override + public S3Destination decide(String tag, Instant time) { + String keyPrefix = getKeyPrefix() == null ? "" : getKeyPrefix() + "/"; + String keySuffix = getKeySuffix() == null ? "" : getKeySuffix(); + return new S3Destination(getBucket(tag), keyPrefix + getKeyBase(time) + keySuffix); + } -public class DefaultS3DestinationDecider - implements S3DestinationDecider -{ - private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy/MM/dd/HH/mm-ss-SSSSSS"); - private final Config config; + public String getKeyPrefix() { + return config.getKeyPrefix(); + } - public DefaultS3DestinationDecider(Config config) - { - config.validateValues(); - this.config = config; + public String getKeySuffix() { + return config.getKeySuffix(); + } + + public ZoneId getZoneId() { + return config.getZoneId(); + } + + public static class Config implements Validatable { + private String keyPrefix; + + private String keySuffix; + + private ZoneId zoneId = ZoneOffset.UTC; + + public String getKeyPrefix() { + return keyPrefix; } - protected String getBucket(String tag) - { - return tag; + public void setKeyPrefix(String keyPrefix) { + this.keyPrefix = keyPrefix; } - protected String getKeyBase(Instant time) - { - return ZonedDateTime.ofInstant(time, getZoneId()).format(FORMATTER); + public String getKeySuffix() { + return keySuffix; } - @Override - public S3Destination decide(String tag, Instant time) - { - String keyPrefix = getKeyPrefix() == null ? "" : getKeyPrefix() + "/"; - String keySuffix = getKeySuffix() == null ? "" : getKeySuffix(); - return new S3Destination(getBucket(tag), keyPrefix + getKeyBase(time) + keySuffix); + public void setKeySuffix(String keySuffix) { + this.keySuffix = keySuffix; } - public String getKeyPrefix() - { - return config.getKeyPrefix(); + public ZoneId getZoneId() { + return zoneId; } - public String getKeySuffix() - { - return config.getKeySuffix(); + public void setZoneId(ZoneId zoneId) { + this.zoneId = zoneId; } - public ZoneId getZoneId() - { - return config.getZoneId(); + void validateValues() { + validate(); } - public static class Config - implements Validatable - { - private String keyPrefix; - - private String keySuffix; - - private ZoneId zoneId = ZoneOffset.UTC; - - public String getKeyPrefix() - { - return keyPrefix; - } - - public void setKeyPrefix(String keyPrefix) - { - this.keyPrefix = keyPrefix; - } - - public String getKeySuffix() - { - return keySuffix; - } - - public void setKeySuffix(String keySuffix) - { - this.keySuffix = keySuffix; - } - - public ZoneId getZoneId() - { - return zoneId; - } - - public void setZoneId(ZoneId zoneId) - { - this.zoneId = zoneId; - } - - void validateValues() - { - validate(); - } - - @Override - public String toString() - { - return "Config{" + - "keyPrefix='" + keyPrefix + '\'' + - ", keySuffix='" + keySuffix + '\'' + - ", zoneId=" + zoneId + - '}'; - } + @Override + public String toString() { + return "Config{" + + "keyPrefix='" + + keyPrefix + + '\'' + + ", keySuffix='" + + keySuffix + + '\'' + + ", zoneId=" + + zoneId + + '}'; } + } } diff --git a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/S3DestinationDecider.java b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/S3DestinationDecider.java index f3bef752..c32552d6 100644 --- a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/S3DestinationDecider.java +++ b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/S3DestinationDecider.java @@ -18,29 +18,24 @@ import java.time.Instant; -public interface S3DestinationDecider -{ - class S3Destination - { - private final String bucket; - private final String key; +public interface S3DestinationDecider { + class S3Destination { + private final String bucket; + private final String key; - public S3Destination(String bucket, String key) - { - this.bucket = bucket; - this.key = key; - } + public S3Destination(String bucket, String key) { + this.bucket = bucket; + this.key = key; + } - public String getBucket() - { - return bucket; - } + public String getBucket() { + return bucket; + } - public String getKey() - { - return key; - } + public String getKey() { + return key; } + } - S3Destination decide(String tag, Instant time); + S3Destination decide(String tag, Instant time); } diff --git a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/sender/AwsS3Sender.java b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/sender/AwsS3Sender.java index da279696..5ff37f99 100644 --- a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/sender/AwsS3Sender.java +++ b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/sender/AwsS3Sender.java @@ -17,6 +17,18 @@ package org.komamitsu.fluency.aws.s3.ingester.sender; import com.fasterxml.jackson.databind.util.ByteBufferBackedInputStream; +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.time.temporal.ChronoUnit; +import java.util.zip.GZIPOutputStream; import net.jodah.failsafe.Failsafe; import net.jodah.failsafe.RetryPolicy; import org.komamitsu.fluency.NonRetryableException; @@ -37,317 +49,268 @@ import software.amazon.awssdk.services.s3.S3ClientBuilder; import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import java.io.Closeable; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.nio.file.StandardOpenOption; -import java.time.temporal.ChronoUnit; -import java.util.zip.GZIPOutputStream; +public class AwsS3Sender implements Closeable, Sender { + private static final Logger LOG = LoggerFactory.getLogger(AwsS3Sender.class); + private final Config config; + private final RetryPolicy retryPolicy; + private final S3Client client; + + public AwsS3Sender(S3ClientBuilder s3ClientBuilder) { + this(s3ClientBuilder, new Config()); + } + + public AwsS3Sender(S3ClientBuilder s3ClientBuilder, Config config) { + config.validateValues(); + this.config = config; + this.retryPolicy = + new RetryPolicy() + .handleIf( + ex -> { + if (ex == null) { + // Success. Shouldn't retry. + return false; + } + + ErrorHandler errorHandler = config.getErrorHandler(); + + if (errorHandler != null) { + errorHandler.handle(ex); + } + + if (ex instanceof InterruptedException || ex instanceof NonRetryableException) { + return false; + } + + return true; + }) + .withBackoff( + getRetryInternalMs(), getMaxRetryInternalMs(), ChronoUnit.MILLIS, getRetryFactor()) + .withMaxRetries(getRetryMax()); + + this.client = buildClient(s3ClientBuilder); + } + + @VisibleForTesting + protected S3Client buildClient(S3ClientBuilder s3ClientBuilder) { + if (config.getEndpoint() != null) { + try { + URI uri = new URI(config.getEndpoint()); + s3ClientBuilder.endpointOverride(uri); + } catch (URISyntaxException e) { + throw new NonRetryableException( + String.format("Invalid endpoint. %s", config.getEndpoint()), e); + } + } -public class AwsS3Sender - implements Closeable, Sender -{ - private static final Logger LOG = LoggerFactory.getLogger(AwsS3Sender.class); - private final Config config; - private final RetryPolicy retryPolicy; - private final S3Client client; - - public AwsS3Sender(S3ClientBuilder s3ClientBuilder) - { - this(s3ClientBuilder, new Config()); + if (config.getRegion() != null) { + s3ClientBuilder.region(Region.of(config.getRegion())); } - public AwsS3Sender(S3ClientBuilder s3ClientBuilder, Config config) - { - config.validateValues(); - this.config = config; - this.retryPolicy = - new RetryPolicy(). - handleIf(ex -> { - if (ex == null) { - // Success. Shouldn't retry. - return false; - } - - ErrorHandler errorHandler = config.getErrorHandler(); - - if (errorHandler != null) { - errorHandler.handle(ex); - } - - if (ex instanceof InterruptedException || ex instanceof NonRetryableException) { - return false; - } - - return true; - }). - withBackoff( - getRetryInternalMs(), - getMaxRetryInternalMs(), - ChronoUnit.MILLIS, - getRetryFactor()). - withMaxRetries(getRetryMax()); - - this.client = buildClient(s3ClientBuilder); + if (config.getAwsAccessKeyId() != null && config.getAwsSecretAccessKey() != null) { + AwsBasicCredentials credentials = + AwsBasicCredentials.create(config.getAwsAccessKeyId(), config.getAwsSecretAccessKey()); + s3ClientBuilder.credentialsProvider(StaticCredentialsProvider.create(credentials)); } - @VisibleForTesting - protected S3Client buildClient(S3ClientBuilder s3ClientBuilder) - { - if (config.getEndpoint() != null) { - try { - URI uri = new URI(config.getEndpoint()); - s3ClientBuilder.endpointOverride(uri); - } - catch (URISyntaxException e) { - throw new NonRetryableException( - String.format("Invalid endpoint. %s", config.getEndpoint()), e); - } - } - - if (config.getRegion() != null) { - s3ClientBuilder.region(Region.of(config.getRegion())); - } - - if (config.getAwsAccessKeyId() != null && config.getAwsSecretAccessKey() != null) { - AwsBasicCredentials credentials = - AwsBasicCredentials.create(config.getAwsAccessKeyId(), config.getAwsSecretAccessKey()); - s3ClientBuilder.credentialsProvider(StaticCredentialsProvider.create(credentials)); - } - - return s3ClientBuilder.build(); + return s3ClientBuilder.build(); + } + + public int getRetryInternalMs() { + return config.getRetryIntervalMs(); + } + + public int getMaxRetryInternalMs() { + return config.getMaxRetryIntervalMs(); + } + + public float getRetryFactor() { + return config.getRetryFactor(); + } + + public int getRetryMax() { + return config.getRetryMax(); + } + + public int getWorkBufSize() { + return config.getWorkBufSize(); + } + + private void copyStreams(InputStream in, OutputStream out) throws IOException { + byte[] buf = new byte[getWorkBufSize()]; + while (true) { + int readLen = in.read(buf); + if (readLen < 0) { + break; + } + out.write(buf, 0, readLen); + } + } + + private void uploadData(String bucket, String key, File file) { + LOG.debug("Upload data to S3: bucket={}, key={}, fileSize={}", bucket, key, file.length()); + + try { + PutObjectRequest.Builder builder = PutObjectRequest.builder().bucket(bucket).key(key); + client.putObject(builder.build(), RequestBody.fromFile(file)); + } catch (NonRetryableException e) { + throw e; + } catch (Throwable e) { + throw new RetryableException( + String.format("Failed to upload data. bucket=%s, key=%s", bucket, key), e); + } + } + + public void send(String bucket, String key, ByteBuffer dataBuffer) throws IOException { + File file = File.createTempFile("tmp-fluency-", ".tmp"); + try { + try (InputStream in = new ByteBufferBackedInputStream(dataBuffer); + OutputStream fout = Files.newOutputStream(file.toPath(), StandardOpenOption.WRITE); + OutputStream out = config.isCompressionEnabled() ? new GZIPOutputStream(fout) : fout) { + copyStreams(in, out); + } + + Failsafe.with(retryPolicy).run(() -> uploadData(bucket, key, file)); + } finally { + if (!file.delete()) { + LOG.warn("Failed to delete a temp file: {}", file.getAbsolutePath()); + } } + } + + @Override + public void close() { + client.close(); + } + + public static class Config extends Sender.Config implements Validatable { + private String endpoint; + private String region; + private String awsAccessKeyId; + private String awsSecretAccessKey; + private boolean isCompressionEnabled = true; + + @Min(10) + private int retryIntervalMs = 1000; - public int getRetryInternalMs() - { - return config.getRetryIntervalMs(); + @Min(10) + private int maxRetryIntervalMs = 30000; + + @DecimalMin("1.0") + private float retryFactor = 2; + + @Min(0) + private int retryMax = 10; + + @Min(1024) + private int workBufSize = 8192; + + public String getEndpoint() { + return endpoint; } - public int getMaxRetryInternalMs() - { - return config.getMaxRetryIntervalMs(); + public void setEndpoint(String endpoint) { + this.endpoint = endpoint; } - public float getRetryFactor() - { - return config.getRetryFactor(); + public String getRegion() { + return region; } - public int getRetryMax() - { - return config.getRetryMax(); + public void setRegion(String region) { + this.region = region; } - public int getWorkBufSize() - { - return config.getWorkBufSize(); + public String getAwsAccessKeyId() { + return awsAccessKeyId; } - private void copyStreams(InputStream in, OutputStream out) - throws IOException - { - byte[] buf = new byte[getWorkBufSize()]; - while (true) { - int readLen = in.read(buf); - if (readLen < 0) { - break; - } - out.write(buf, 0, readLen); - } + public void setAwsAccessKeyId(String awsAccessKeyId) { + this.awsAccessKeyId = awsAccessKeyId; } - private void uploadData(String bucket, String key, File file) - { - LOG.debug("Upload data to S3: bucket={}, key={}, fileSize={}", - bucket, key, file.length()); - - try { - PutObjectRequest.Builder builder = PutObjectRequest.builder() - .bucket(bucket) - .key(key); - client.putObject(builder.build(), RequestBody.fromFile(file)); - } - catch (NonRetryableException e) { - throw e; - } - catch (Throwable e) { - throw new RetryableException( - String.format("Failed to upload data. bucket=%s, key=%s", bucket, key), e); - } + public String getAwsSecretAccessKey() { + return awsSecretAccessKey; } - public void send(String bucket, String key, ByteBuffer dataBuffer) - throws IOException - { - File file = File.createTempFile("tmp-fluency-", ".tmp"); - try { - try (InputStream in = new ByteBufferBackedInputStream(dataBuffer); - OutputStream fout = Files.newOutputStream(file.toPath(), StandardOpenOption.WRITE); - OutputStream out = config.isCompressionEnabled() ? new GZIPOutputStream(fout) : fout) { - copyStreams(in, out); - } - - Failsafe.with(retryPolicy).run(() -> uploadData(bucket, key, file)); - } - finally { - if (!file.delete()) { - LOG.warn("Failed to delete a temp file: {}", file.getAbsolutePath()); - } - } + public void setAwsSecretAccessKey(String awsSecretAccessKey) { + this.awsSecretAccessKey = awsSecretAccessKey; } - @Override - public void close() - { - client.close(); + public boolean isCompressionEnabled() { + return isCompressionEnabled; + } + + public void setCompressionEnabled(boolean isCompressionEnabled) { + this.isCompressionEnabled = isCompressionEnabled; + } + + public int getRetryIntervalMs() { + return retryIntervalMs; + } + + public void setRetryIntervalMs(int retryIntervalMs) { + this.retryIntervalMs = retryIntervalMs; + } + + public int getMaxRetryIntervalMs() { + return maxRetryIntervalMs; + } + + public void setMaxRetryIntervalMs(int maxRetryIntervalMs) { + this.maxRetryIntervalMs = maxRetryIntervalMs; } - public static class Config - extends Sender.Config - implements Validatable - { - private String endpoint; - private String region; - private String awsAccessKeyId; - private String awsSecretAccessKey; - private boolean isCompressionEnabled = true; - - @Min(10) - private int retryIntervalMs = 1000; - @Min(10) - private int maxRetryIntervalMs = 30000; - @DecimalMin("1.0") - private float retryFactor = 2; - @Min(0) - private int retryMax = 10; - @Min(1024) - private int workBufSize = 8192; - - public String getEndpoint() - { - return endpoint; - } - - public void setEndpoint(String endpoint) - { - this.endpoint = endpoint; - } - - public String getRegion() - { - return region; - } - - public void setRegion(String region) - { - this.region = region; - } - - public String getAwsAccessKeyId() - { - return awsAccessKeyId; - } - - public void setAwsAccessKeyId(String awsAccessKeyId) - { - this.awsAccessKeyId = awsAccessKeyId; - } - - public String getAwsSecretAccessKey() - { - return awsSecretAccessKey; - } - - public void setAwsSecretAccessKey(String awsSecretAccessKey) - { - this.awsSecretAccessKey = awsSecretAccessKey; - } - - public boolean isCompressionEnabled() - { - return isCompressionEnabled; - } - - public void setCompressionEnabled(boolean isCompressionEnabled) - { - this.isCompressionEnabled = isCompressionEnabled; - } - - public int getRetryIntervalMs() - { - return retryIntervalMs; - } - - public void setRetryIntervalMs(int retryIntervalMs) - { - this.retryIntervalMs = retryIntervalMs; - } - - public int getMaxRetryIntervalMs() - { - return maxRetryIntervalMs; - } - - public void setMaxRetryIntervalMs(int maxRetryIntervalMs) - { - this.maxRetryIntervalMs = maxRetryIntervalMs; - } - - public float getRetryFactor() - { - return retryFactor; - } - - public void setRetryFactor(float retryFactor) - { - this.retryFactor = retryFactor; - } - - public int getRetryMax() - { - return retryMax; - } - - public void setRetryMax(int retryMax) - { - this.retryMax = retryMax; - } - - public int getWorkBufSize() - { - return workBufSize; - } - - public void setWorkBufSize(int workBufSize) - { - this.workBufSize = workBufSize; - } - - @Override - public String toString() - { - return "Config{" + - "endpoint='" + endpoint + '\'' + - ", region='" + region + '\'' + - ", isCompressionEnabled=" + isCompressionEnabled + - ", retryIntervalMs=" + retryIntervalMs + - ", maxRetryIntervalMs=" + maxRetryIntervalMs + - ", retryFactor=" + retryFactor + - ", retryMax=" + retryMax + - ", workBufSize=" + workBufSize + - "} " + super.toString(); - } - - void validateValues() - { - validate(); - } + public float getRetryFactor() { + return retryFactor; + } + + public void setRetryFactor(float retryFactor) { + this.retryFactor = retryFactor; + } + + public int getRetryMax() { + return retryMax; + } + + public void setRetryMax(int retryMax) { + this.retryMax = retryMax; + } + + public int getWorkBufSize() { + return workBufSize; + } + + public void setWorkBufSize(int workBufSize) { + this.workBufSize = workBufSize; + } + + @Override + public String toString() { + return "Config{" + + "endpoint='" + + endpoint + + '\'' + + ", region='" + + region + + '\'' + + ", isCompressionEnabled=" + + isCompressionEnabled + + ", retryIntervalMs=" + + retryIntervalMs + + ", maxRetryIntervalMs=" + + maxRetryIntervalMs + + ", retryFactor=" + + retryFactor + + ", retryMax=" + + retryMax + + ", workBufSize=" + + workBufSize + + "} " + + super.toString(); + } + void validateValues() { + validate(); } + } } diff --git a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/AwsS3RecordFormatter.java b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/AwsS3RecordFormatter.java index a4979677..94633edd 100644 --- a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/AwsS3RecordFormatter.java +++ b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/AwsS3RecordFormatter.java @@ -18,7 +18,4 @@ import org.komamitsu.fluency.recordformat.RecordFormatter; -public interface AwsS3RecordFormatter - extends RecordFormatter -{ -} +public interface AwsS3RecordFormatter extends RecordFormatter {} diff --git a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/CsvRecordFormatter.java b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/CsvRecordFormatter.java index 15569f98..e111165e 100644 --- a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/CsvRecordFormatter.java +++ b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/CsvRecordFormatter.java @@ -16,191 +16,169 @@ package org.komamitsu.fluency.aws.s3.recordformat; +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; import org.komamitsu.fluency.recordformat.AbstractRecordFormatter; +import org.komamitsu.fluency.recordformat.RecordFormatter; import org.komamitsu.fluency.recordformat.recordaccessor.MapRecordAccessor; import org.komamitsu.fluency.recordformat.recordaccessor.MessagePackRecordAccessor; import org.komamitsu.fluency.recordformat.recordaccessor.RecordAccessor; -import org.komamitsu.fluency.recordformat.RecordFormatter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.ByteArrayOutputStream; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.Map; +public class CsvRecordFormatter extends AbstractRecordFormatter implements AwsS3RecordFormatter { + private static final Logger LOG = LoggerFactory.getLogger(CsvRecordFormatter.class); + private final Config config; + private final byte[] delimiter; + private final byte[] quote; + private final byte[] lineBreak; + + public CsvRecordFormatter() { + this(new Config()); + } + + public CsvRecordFormatter(Config config) { + super(config); + config.validateValues(); + this.config = config; + + String delimiterInConfig = config.getDelimiter(); + if (delimiterInConfig == null) { + delimiter = null; + } else { + delimiter = delimiterInConfig.getBytes(StandardCharsets.UTF_8); + } -public class CsvRecordFormatter - extends AbstractRecordFormatter - implements AwsS3RecordFormatter -{ - private static final Logger LOG = LoggerFactory.getLogger(CsvRecordFormatter.class); - private final Config config; - private final byte[] delimiter; - private final byte[] quote; - private final byte[] lineBreak; - - public CsvRecordFormatter() - { - this(new Config()); + String quoteInConfig = config.getQuote(); + if (quoteInConfig == null) { + quote = null; + } else { + quote = quoteInConfig.getBytes(StandardCharsets.UTF_8); } - public CsvRecordFormatter(Config config) - { - super(config); - config.validateValues(); - this.config = config; + lineBreak = new byte[] {0x0A}; + } + + private byte[] formatInternal(RecordAccessor recordAccessor) { + boolean isFirst = true; + ByteArrayOutputStream output = new ByteArrayOutputStream(); + for (String columnName : config.getColumnNames()) { + if (delimiter != null) { + if (isFirst) { + isFirst = false; + } else { + output.write(delimiter, 0, delimiter.length); + } + } + + String value = recordAccessor.getAsString(columnName); + if (value == null) { + continue; + } + + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + + if (quote == null) { + output.write(bytes, 0, bytes.length); + } else { + output.write(quote, 0, quote.length); + output.write(bytes, 0, bytes.length); + output.write(quote, 0, quote.length); + } + } + output.write(lineBreak, 0, lineBreak.length); + + return output.toByteArray(); + } + + @Override + public byte[] format(String tag, Object timestamp, Map data) { + try { + RecordAccessor recordAccessor = new MapRecordAccessor(data); + recordAccessor.setTimestamp(getEpoch(timestamp)); + return formatInternal(recordAccessor); + } catch (Throwable e) { + LOG.error( + String.format( + "Failed to format a Map record: cause=%s, tag=%s, timestamp=%s, recordCount=%d", + e.getMessage(), tag, timestamp, data.size())); + throw e; + } + } + + @Override + public byte[] formatFromMessagePack( + String tag, Object timestamp, byte[] mapValue, int offset, int len) { + try { + RecordAccessor recordAccessor = + new MessagePackRecordAccessor(ByteBuffer.wrap(mapValue, offset, len)); + recordAccessor.setTimestamp(getEpoch(timestamp)); + return formatInternal(recordAccessor); + } catch (Throwable e) { + LOG.error( + String.format( + "Failed to format a MessagePack record: cause=%s, tag=%s, timestamp=%s, offset=%d, len=%d", + e.getMessage(), tag, timestamp, offset, len)); + throw e; + } + } + + @Override + public byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue) { + try { + RecordAccessor recordAccessor = new MessagePackRecordAccessor(mapValue); + recordAccessor.setTimestamp(getEpoch(timestamp)); + return formatInternal(recordAccessor); + } catch (Throwable e) { + LOG.error( + String.format( + "Failed to format a MessagePack record: cause=%s, tag=%s, timestamp=%s, bytebuf=%s", + e.getMessage(), tag, timestamp, mapValue)); + throw e; + } + } - String delimiterInConfig = config.getDelimiter(); - if (delimiterInConfig == null) { - delimiter = null; - } - else { - delimiter = delimiterInConfig.getBytes(StandardCharsets.UTF_8); - } + @Override + public String formatName() { + return "csv"; + } - String quoteInConfig = config.getQuote(); - if (quoteInConfig == null) { - quote = null; - } - else { - quote = quoteInConfig.getBytes(StandardCharsets.UTF_8); - } + public static class Config extends RecordFormatter.Config { + private List columnNames; + private String quoteString; + private String delimiter = ","; - lineBreak = new byte[] { 0x0A }; + public List getColumnNames() { + return columnNames; } - private byte[] formatInternal(RecordAccessor recordAccessor) - { - boolean isFirst = true; - ByteArrayOutputStream output = new ByteArrayOutputStream(); - for (String columnName : config.getColumnNames()) { - if (delimiter != null) { - if (isFirst) { - isFirst = false; - } - else { - output.write(delimiter, 0, delimiter.length); - } - } - - String value = recordAccessor.getAsString(columnName); - if (value == null) { - continue; - } - - byte[] bytes = value.getBytes(StandardCharsets.UTF_8); - - if (quote == null) { - output.write(bytes, 0, bytes.length); - } - else { - output.write(quote, 0, quote.length); - output.write(bytes, 0, bytes.length); - output.write(quote, 0, quote.length); - } - } - output.write(lineBreak, 0, lineBreak.length); - - return output.toByteArray(); + public void setColumnNames(List columnNames) { + this.columnNames = columnNames; } - @Override - public byte[] format(String tag, Object timestamp, Map data) - { - try { - RecordAccessor recordAccessor = new MapRecordAccessor(data); - recordAccessor.setTimestamp(getEpoch(timestamp)); - return formatInternal(recordAccessor); - } - catch (Throwable e) { - LOG.error(String.format( - "Failed to format a Map record: cause=%s, tag=%s, timestamp=%s, recordCount=%d", - e.getMessage(), tag, timestamp, data.size())); - throw e; - } + public String getQuote() { + return quoteString; } - @Override - public byte[] formatFromMessagePack(String tag, Object timestamp, byte[] mapValue, int offset, int len) - { - try { - RecordAccessor recordAccessor = new MessagePackRecordAccessor(ByteBuffer.wrap(mapValue, offset, len)); - recordAccessor.setTimestamp(getEpoch(timestamp)); - return formatInternal(recordAccessor); - } - catch (Throwable e) { - LOG.error(String.format( - "Failed to format a MessagePack record: cause=%s, tag=%s, timestamp=%s, offset=%d, len=%d", - e.getMessage(), tag, timestamp, offset, len)); - throw e; - } + public void setQuote(String quoteString) { + this.quoteString = quoteString; } - @Override - public byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue) - { - try { - RecordAccessor recordAccessor = new MessagePackRecordAccessor(mapValue); - recordAccessor.setTimestamp(getEpoch(timestamp)); - return formatInternal(recordAccessor); - } - catch (Throwable e) { - LOG.error(String.format( - "Failed to format a MessagePack record: cause=%s, tag=%s, timestamp=%s, bytebuf=%s", - e.getMessage(), tag, timestamp, mapValue)); - throw e; - } + public String getDelimiter() { + return delimiter; } - @Override - public String formatName() - { - return "csv"; + public void setDelimiter(String delimiter) { + this.delimiter = delimiter; } - public static class Config - extends RecordFormatter.Config - { - private List columnNames; - private String quoteString; - private String delimiter = ","; - - public List getColumnNames() - { - return columnNames; - } - - public void setColumnNames(List columnNames) - { - this.columnNames = columnNames; - } - - public String getQuote() - { - return quoteString; - } - - public void setQuote(String quoteString) - { - this.quoteString = quoteString; - } - - public String getDelimiter() - { - return delimiter; - } - - public void setDelimiter(String delimiter) - { - this.delimiter = delimiter; - } - - public void validateValues() - { - if (columnNames == null || columnNames.isEmpty()) { - throw new IllegalArgumentException("Column names should be empty"); - } - } + public void validateValues() { + if (columnNames == null || columnNames.isEmpty()) { + throw new IllegalArgumentException("Column names should be empty"); + } } + } } diff --git a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/JsonlRecordFormatter.java b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/JsonlRecordFormatter.java index a384a9d9..1025c664 100644 --- a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/JsonlRecordFormatter.java +++ b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/JsonlRecordFormatter.java @@ -17,95 +17,83 @@ package org.komamitsu.fluency.aws.s3.recordformat; import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Map; import org.komamitsu.fluency.recordformat.AbstractRecordFormatter; +import org.komamitsu.fluency.recordformat.RecordFormatter; import org.komamitsu.fluency.recordformat.recordaccessor.MapRecordAccessor; import org.komamitsu.fluency.recordformat.recordaccessor.MessagePackRecordAccessor; -import org.komamitsu.fluency.recordformat.RecordFormatter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.Map; - -public class JsonlRecordFormatter - extends AbstractRecordFormatter - implements AwsS3RecordFormatter -{ - private static final Logger LOG = LoggerFactory.getLogger(JsonlRecordFormatter.class); - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); +public class JsonlRecordFormatter extends AbstractRecordFormatter implements AwsS3RecordFormatter { + private static final Logger LOG = LoggerFactory.getLogger(JsonlRecordFormatter.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - public JsonlRecordFormatter() - { - this(new Config()); - } + public JsonlRecordFormatter() { + this(new Config()); + } - public JsonlRecordFormatter(Config config) - { - super(config); - } + public JsonlRecordFormatter(Config config) { + super(config); + } - private byte[] appendNewLine(String json) - { - return (json + "\n").getBytes(StandardCharsets.UTF_8); - } + private byte[] appendNewLine(String json) { + return (json + "\n").getBytes(StandardCharsets.UTF_8); + } - @Override - public byte[] format(String tag, Object timestamp, Map data) - { - try { - MapRecordAccessor recordAccessor = new MapRecordAccessor(data); - recordAccessor.setTimestamp(getEpoch(timestamp)); - return appendNewLine(recordAccessor.toJson(OBJECT_MAPPER)); - } - catch (Throwable e) { - LOG.error(String.format( - "Failed to format a Map record: cause=%s, tag=%s, timestamp=%s, recordCount=%d", - e.getMessage(), tag, timestamp, data.size())); - throw e; - } + @Override + public byte[] format(String tag, Object timestamp, Map data) { + try { + MapRecordAccessor recordAccessor = new MapRecordAccessor(data); + recordAccessor.setTimestamp(getEpoch(timestamp)); + return appendNewLine(recordAccessor.toJson(OBJECT_MAPPER)); + } catch (Throwable e) { + LOG.error( + String.format( + "Failed to format a Map record: cause=%s, tag=%s, timestamp=%s, recordCount=%d", + e.getMessage(), tag, timestamp, data.size())); + throw e; } + } - @Override - public byte[] formatFromMessagePack(String tag, Object timestamp, byte[] mapValue, int offset, int len) - { - try { - MessagePackRecordAccessor recordAccessor = new MessagePackRecordAccessor(ByteBuffer.wrap(mapValue, offset, len)); - recordAccessor.setTimestamp(getEpoch(timestamp)); - return appendNewLine(recordAccessor.toJson(OBJECT_MAPPER)); - } - catch (Throwable e) { - LOG.error(String.format( - "Failed to format a MessagePack record: cause=%s, tag=%s, timestamp=%s, offset=%d, len=%d", - e.getMessage(), tag, timestamp, offset, len)); - throw e; - } + @Override + public byte[] formatFromMessagePack( + String tag, Object timestamp, byte[] mapValue, int offset, int len) { + try { + MessagePackRecordAccessor recordAccessor = + new MessagePackRecordAccessor(ByteBuffer.wrap(mapValue, offset, len)); + recordAccessor.setTimestamp(getEpoch(timestamp)); + return appendNewLine(recordAccessor.toJson(OBJECT_MAPPER)); + } catch (Throwable e) { + LOG.error( + String.format( + "Failed to format a MessagePack record: cause=%s, tag=%s, timestamp=%s, offset=%d, len=%d", + e.getMessage(), tag, timestamp, offset, len)); + throw e; } + } - @Override - public byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue) - { - try { - MessagePackRecordAccessor recordAccessor = new MessagePackRecordAccessor(mapValue); - recordAccessor.setTimestamp(getEpoch(timestamp)); - return appendNewLine(recordAccessor.toJson(OBJECT_MAPPER)); - } - catch (Throwable e) { - LOG.error(String.format( - "Failed to format a MessagePack record: cause=%s, tag=%s, timestamp=%s, bytebuf=%s", - e.getMessage(), tag, timestamp, mapValue)); - throw e; - } + @Override + public byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue) { + try { + MessagePackRecordAccessor recordAccessor = new MessagePackRecordAccessor(mapValue); + recordAccessor.setTimestamp(getEpoch(timestamp)); + return appendNewLine(recordAccessor.toJson(OBJECT_MAPPER)); + } catch (Throwable e) { + LOG.error( + String.format( + "Failed to format a MessagePack record: cause=%s, tag=%s, timestamp=%s, bytebuf=%s", + e.getMessage(), tag, timestamp, mapValue)); + throw e; } + } - @Override - public String formatName() - { - return "jsonl"; - } + @Override + public String formatName() { + return "jsonl"; + } - public static class Config - extends RecordFormatter.Config - { - } + public static class Config extends RecordFormatter.Config {} } diff --git a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/MessagePackRecordFormatter.java b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/MessagePackRecordFormatter.java index 0e8e06b9..227ad1e2 100644 --- a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/MessagePackRecordFormatter.java +++ b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/MessagePackRecordFormatter.java @@ -16,23 +16,17 @@ package org.komamitsu.fluency.aws.s3.recordformat; - public class MessagePackRecordFormatter - extends org.komamitsu.fluency.recordformat.MessagePackRecordFormatter - implements AwsS3RecordFormatter -{ - public MessagePackRecordFormatter() - { - this(new Config()); - } + extends org.komamitsu.fluency.recordformat.MessagePackRecordFormatter + implements AwsS3RecordFormatter { + public MessagePackRecordFormatter() { + this(new Config()); + } - public MessagePackRecordFormatter(Config config) - { - super(config); - } + public MessagePackRecordFormatter(Config config) { + super(config); + } - public static class Config - extends org.komamitsu.fluency.recordformat.MessagePackRecordFormatter.Config - { - } + public static class Config + extends org.komamitsu.fluency.recordformat.MessagePackRecordFormatter.Config {} } diff --git a/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/FluencyBuilderForAwsS3Test.java b/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/FluencyBuilderForAwsS3Test.java index 54146bf6..d1dd779b 100644 --- a/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/FluencyBuilderForAwsS3Test.java +++ b/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/FluencyBuilderForAwsS3Test.java @@ -16,7 +16,23 @@ package org.komamitsu.fluency.aws.s3; +import static java.time.ZoneId.SHORT_IDS; +import static java.time.ZoneOffset.UTC; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + import com.google.common.collect.ImmutableList; +import java.time.ZoneId; +import java.time.ZoneOffset; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -36,198 +52,191 @@ import org.mockito.ArgumentCaptor; import software.amazon.awssdk.regions.Region; -import java.time.ZoneId; -import java.time.ZoneOffset; - -import static java.time.ZoneId.SHORT_IDS; -import static java.time.ZoneOffset.UTC; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; +class FluencyBuilderForAwsS3Test { + private FluencyBuilderForAwsS3 builderWithDefaultConfig; + private FluencyBuilderForAwsS3 builderWithCustomConfig; + private Fluency fluency; + + @BeforeEach + void setUp() { + fluency = mock(Fluency.class); + + builderWithDefaultConfig = spy(new FluencyBuilderForAwsS3()); + builderWithDefaultConfig.setFormatCsvColumnNames(ImmutableList.of("dummy")); + doReturn(fluency) + .when(builderWithDefaultConfig) + .createFluency( + any(RecordFormatter.class), + any(Ingester.class), + any(Buffer.Config.class), + any(Flusher.Config.class)); -class FluencyBuilderForAwsS3Test -{ - private FluencyBuilderForAwsS3 builderWithDefaultConfig; - private FluencyBuilderForAwsS3 builderWithCustomConfig; - private Fluency fluency; - - @BeforeEach - void setUp() { - fluency = mock(Fluency.class); - - builderWithDefaultConfig = spy(new FluencyBuilderForAwsS3()); - builderWithDefaultConfig.setFormatCsvColumnNames(ImmutableList.of("dummy")); - doReturn(fluency).when(builderWithDefaultConfig) - .createFluency( - any(RecordFormatter.class), - any(Ingester.class), - any(Buffer.Config.class), - any(Flusher.Config.class)); - - { - FluencyBuilderForAwsS3 builder = new FluencyBuilderForAwsS3(); - builder.setAwsEndpoint("https://foo.bar.org"); - builder.setAwsRegion("us-east-1"); - builder.setCompressionEnabled(false); - builder.setAwsAccessKeyId("ACCESSKEYID"); - builder.setAwsSecretAccessKey("SECRETACCESSKEY"); - builder.setBufferChunkInitialSize(2 * 1024 * 1024); - builder.setBufferChunkRetentionSize(9 * 1024 * 1024); - builder.setBufferChunkRetentionTimeMillis(1234); - builder.setMaxBufferSize(42 * 1024 * 1024L); - builder.setS3KeyPrefix("mydata"); - builder.setS3KeySuffix(".zzz"); - builder.setS3KeyTimeZoneId(ZoneOffset.of("JST", SHORT_IDS)); - builder.setSenderRetryMax(4); - builder.setSenderMaxRetryIntervalMillis(65432); - builder.setSenderRetryIntervalMillis(543); - builder.setSenderRetryFactor(1.234f); - builder.setSenderWorkBufSize(99 * 1024); - builderWithCustomConfig = spy(builder); - } - doReturn(fluency).when(builderWithCustomConfig) - .createFluency( - any(RecordFormatter.class), - any(Ingester.class), - any(Buffer.Config.class), - any(Flusher.Config.class)); + FluencyBuilderForAwsS3 builder = new FluencyBuilderForAwsS3(); + builder.setAwsEndpoint("https://foo.bar.org"); + builder.setAwsRegion("us-east-1"); + builder.setCompressionEnabled(false); + builder.setAwsAccessKeyId("ACCESSKEYID"); + builder.setAwsSecretAccessKey("SECRETACCESSKEY"); + builder.setBufferChunkInitialSize(2 * 1024 * 1024); + builder.setBufferChunkRetentionSize(9 * 1024 * 1024); + builder.setBufferChunkRetentionTimeMillis(1234); + builder.setMaxBufferSize(42 * 1024 * 1024L); + builder.setS3KeyPrefix("mydata"); + builder.setS3KeySuffix(".zzz"); + builder.setS3KeyTimeZoneId(ZoneOffset.of("JST", SHORT_IDS)); + builder.setSenderRetryMax(4); + builder.setSenderMaxRetryIntervalMillis(65432); + builder.setSenderRetryIntervalMillis(543); + builder.setSenderRetryFactor(1.234f); + builder.setSenderWorkBufSize(99 * 1024); + builderWithCustomConfig = spy(builder); } - - @ParameterizedTest - @EnumSource(FluencyBuilderForAwsS3.FormatType.class) - void buildWithDefaultConfig(FluencyBuilderForAwsS3.FormatType formatType) - { - FluencyBuilderForAwsS3 builder = builderWithDefaultConfig; - AwsS3Sender sender = mock(AwsS3Sender.class); - doReturn(sender).when(builder).createSender(any(AwsS3Sender.Config.class)); - - builder.setFormatType(formatType); - Fluency fluency = builder.build(); - assertEquals(fluency, this.fluency); - - ArgumentCaptor configArgumentCaptor = ArgumentCaptor.forClass(AwsS3Sender.Config.class); - verify(builder, times(1)).createSender(configArgumentCaptor.capture()); - AwsS3Sender.Config senderConfig = configArgumentCaptor.getValue(); - assertNull(senderConfig.getEndpoint()); - assertNull(senderConfig.getRegion()); - assertNull(senderConfig.getAwsAccessKeyId()); - assertNull(senderConfig.getAwsSecretAccessKey()); - assertEquals(10, senderConfig.getRetryMax()); - assertEquals(1000, senderConfig.getRetryIntervalMs()); - assertEquals(30000, senderConfig.getMaxRetryIntervalMs()); - assertEquals(2.0, senderConfig.getRetryFactor()); - assertEquals(8192, senderConfig.getWorkBufSize()); - assertTrue(senderConfig.isCompressionEnabled()); - - ArgumentCaptor recordFormatterArgumentCaptor = ArgumentCaptor.forClass(RecordFormatter.class); - ArgumentCaptor ingesterArgumentCaptor = ArgumentCaptor.forClass(Ingester.class); - verify(builder, times(1)).buildFromIngester( - recordFormatterArgumentCaptor.capture(), ingesterArgumentCaptor.capture()); - - RecordFormatter recordFormatter = recordFormatterArgumentCaptor.getValue(); - Class expectedAwsS3RecordFormatter = null; - String expectedS3KeySuffix = null; - switch (formatType) { - case MESSAGE_PACK: - expectedAwsS3RecordFormatter = MessagePackRecordFormatter.class; - expectedS3KeySuffix = ".msgpack.gz"; - break; - case JSONL: - expectedAwsS3RecordFormatter = JsonlRecordFormatter.class; - expectedS3KeySuffix = ".jsonl.gz"; - break; - case CSV: - expectedAwsS3RecordFormatter = CsvRecordFormatter.class; - expectedS3KeySuffix = ".csv.gz"; - break; - } - assertEquals(expectedAwsS3RecordFormatter, recordFormatter.getClass()); - - AwsS3Ingester ingester = (AwsS3Ingester) ingesterArgumentCaptor.getValue(); - assertEquals(sender, ingester.getSender()); - DefaultS3DestinationDecider destinationDecider = - (DefaultS3DestinationDecider) ingester.getS3DestinationDecider(); - assertNull(destinationDecider.getKeyPrefix()); - assertEquals(expectedS3KeySuffix, destinationDecider.getKeySuffix()); - assertEquals(UTC, destinationDecider.getZoneId()); - - ArgumentCaptor bufferConfigArgumentCaptor = ArgumentCaptor.forClass(Buffer.Config.class); - ArgumentCaptor flusherConfigArgumentCaptor = ArgumentCaptor.forClass(Flusher.Config.class); - verify(builder, times(1)).createFluency( - eq(recordFormatter), - eq(ingester), - bufferConfigArgumentCaptor.capture(), - flusherConfigArgumentCaptor.capture()); - - Buffer.Config bufferConfig = bufferConfigArgumentCaptor.getValue(); - assertEquals(512 * 1024 * 1024, bufferConfig.getMaxBufferSize()); - assertEquals(4 * 1024 * 1024, bufferConfig.getChunkInitialSize()); - assertEquals(64 * 1024 * 1024, bufferConfig.getChunkRetentionSize()); - assertEquals(30 * 1000, bufferConfig.getChunkRetentionTimeMillis()); - } - - @Test - void buildWithCustomConfig() - { - FluencyBuilderForAwsS3 builder = builderWithCustomConfig; - AwsS3Sender sender = mock(AwsS3Sender.class); - doReturn(sender).when(builder).createSender(any(AwsS3Sender.Config.class)); - - builder.setFormatType(FluencyBuilderForAwsS3.FormatType.JSONL); - Fluency fluency = builder.build(); - assertEquals(fluency, this.fluency); - - ArgumentCaptor configArgumentCaptor = ArgumentCaptor.forClass(AwsS3Sender.Config.class); - verify(builder, times(1)).createSender(configArgumentCaptor.capture()); - AwsS3Sender.Config senderConfig = configArgumentCaptor.getValue(); - assertEquals("https://foo.bar.org", senderConfig.getEndpoint()); - assertEquals(Region.US_EAST_1.toString(), senderConfig.getRegion()); - assertEquals("ACCESSKEYID", senderConfig.getAwsAccessKeyId()); - assertEquals("SECRETACCESSKEY", senderConfig.getAwsSecretAccessKey()); - assertEquals(4, senderConfig.getRetryMax()); - assertEquals(543, senderConfig.getRetryIntervalMs()); - assertEquals(65432, senderConfig.getMaxRetryIntervalMs()); - assertEquals(1.234f, senderConfig.getRetryFactor()); - assertEquals(99 * 1024, senderConfig.getWorkBufSize()); - assertFalse(senderConfig.isCompressionEnabled()); - - ArgumentCaptor recordFormatterArgumentCaptor = ArgumentCaptor.forClass(RecordFormatter.class); - ArgumentCaptor ingesterArgumentCaptor = ArgumentCaptor.forClass(Ingester.class); - verify(builder, times(1)).buildFromIngester( - recordFormatterArgumentCaptor.capture(), ingesterArgumentCaptor.capture()); - - RecordFormatter recordFormatter = recordFormatterArgumentCaptor.getValue(); - assertTrue(recordFormatter instanceof JsonlRecordFormatter); - - AwsS3Ingester ingester = (AwsS3Ingester) ingesterArgumentCaptor.getValue(); - assertEquals(sender, ingester.getSender()); - DefaultS3DestinationDecider destinationDecider = - (DefaultS3DestinationDecider) ingester.getS3DestinationDecider(); - assertEquals("mydata", destinationDecider.getKeyPrefix()); - assertEquals(".zzz", destinationDecider.getKeySuffix()); - assertEquals(ZoneId.of("JST", SHORT_IDS), destinationDecider.getZoneId()); - - ArgumentCaptor bufferConfigArgumentCaptor = ArgumentCaptor.forClass(Buffer.Config.class); - ArgumentCaptor flusherConfigArgumentCaptor = ArgumentCaptor.forClass(Flusher.Config.class); - verify(builder, times(1)).createFluency( - eq(recordFormatter), - eq(ingester), - bufferConfigArgumentCaptor.capture(), - flusherConfigArgumentCaptor.capture()); - - Buffer.Config bufferConfig = bufferConfigArgumentCaptor.getValue(); - assertEquals(42 * 1024 * 1024, bufferConfig.getMaxBufferSize()); - assertEquals(2 * 1024 * 1024, bufferConfig.getChunkInitialSize()); - assertEquals(9 * 1024 * 1024, bufferConfig.getChunkRetentionSize()); - assertEquals(1234, bufferConfig.getChunkRetentionTimeMillis()); + doReturn(fluency) + .when(builderWithCustomConfig) + .createFluency( + any(RecordFormatter.class), + any(Ingester.class), + any(Buffer.Config.class), + any(Flusher.Config.class)); + } + + @ParameterizedTest + @EnumSource(FluencyBuilderForAwsS3.FormatType.class) + void buildWithDefaultConfig(FluencyBuilderForAwsS3.FormatType formatType) { + FluencyBuilderForAwsS3 builder = builderWithDefaultConfig; + AwsS3Sender sender = mock(AwsS3Sender.class); + doReturn(sender).when(builder).createSender(any(AwsS3Sender.Config.class)); + + builder.setFormatType(formatType); + Fluency fluency = builder.build(); + assertEquals(fluency, this.fluency); + + ArgumentCaptor configArgumentCaptor = + ArgumentCaptor.forClass(AwsS3Sender.Config.class); + verify(builder, times(1)).createSender(configArgumentCaptor.capture()); + AwsS3Sender.Config senderConfig = configArgumentCaptor.getValue(); + assertNull(senderConfig.getEndpoint()); + assertNull(senderConfig.getRegion()); + assertNull(senderConfig.getAwsAccessKeyId()); + assertNull(senderConfig.getAwsSecretAccessKey()); + assertEquals(10, senderConfig.getRetryMax()); + assertEquals(1000, senderConfig.getRetryIntervalMs()); + assertEquals(30000, senderConfig.getMaxRetryIntervalMs()); + assertEquals(2.0, senderConfig.getRetryFactor()); + assertEquals(8192, senderConfig.getWorkBufSize()); + assertTrue(senderConfig.isCompressionEnabled()); + + ArgumentCaptor recordFormatterArgumentCaptor = + ArgumentCaptor.forClass(RecordFormatter.class); + ArgumentCaptor ingesterArgumentCaptor = ArgumentCaptor.forClass(Ingester.class); + verify(builder, times(1)) + .buildFromIngester( + recordFormatterArgumentCaptor.capture(), ingesterArgumentCaptor.capture()); + + RecordFormatter recordFormatter = recordFormatterArgumentCaptor.getValue(); + Class expectedAwsS3RecordFormatter = null; + String expectedS3KeySuffix = null; + switch (formatType) { + case MESSAGE_PACK: + expectedAwsS3RecordFormatter = MessagePackRecordFormatter.class; + expectedS3KeySuffix = ".msgpack.gz"; + break; + case JSONL: + expectedAwsS3RecordFormatter = JsonlRecordFormatter.class; + expectedS3KeySuffix = ".jsonl.gz"; + break; + case CSV: + expectedAwsS3RecordFormatter = CsvRecordFormatter.class; + expectedS3KeySuffix = ".csv.gz"; + break; } + assertEquals(expectedAwsS3RecordFormatter, recordFormatter.getClass()); + + AwsS3Ingester ingester = (AwsS3Ingester) ingesterArgumentCaptor.getValue(); + assertEquals(sender, ingester.getSender()); + DefaultS3DestinationDecider destinationDecider = + (DefaultS3DestinationDecider) ingester.getS3DestinationDecider(); + assertNull(destinationDecider.getKeyPrefix()); + assertEquals(expectedS3KeySuffix, destinationDecider.getKeySuffix()); + assertEquals(UTC, destinationDecider.getZoneId()); + + ArgumentCaptor bufferConfigArgumentCaptor = + ArgumentCaptor.forClass(Buffer.Config.class); + ArgumentCaptor flusherConfigArgumentCaptor = + ArgumentCaptor.forClass(Flusher.Config.class); + verify(builder, times(1)) + .createFluency( + eq(recordFormatter), + eq(ingester), + bufferConfigArgumentCaptor.capture(), + flusherConfigArgumentCaptor.capture()); + + Buffer.Config bufferConfig = bufferConfigArgumentCaptor.getValue(); + assertEquals(512 * 1024 * 1024, bufferConfig.getMaxBufferSize()); + assertEquals(4 * 1024 * 1024, bufferConfig.getChunkInitialSize()); + assertEquals(64 * 1024 * 1024, bufferConfig.getChunkRetentionSize()); + assertEquals(30 * 1000, bufferConfig.getChunkRetentionTimeMillis()); + } + + @Test + void buildWithCustomConfig() { + FluencyBuilderForAwsS3 builder = builderWithCustomConfig; + AwsS3Sender sender = mock(AwsS3Sender.class); + doReturn(sender).when(builder).createSender(any(AwsS3Sender.Config.class)); + + builder.setFormatType(FluencyBuilderForAwsS3.FormatType.JSONL); + Fluency fluency = builder.build(); + assertEquals(fluency, this.fluency); + + ArgumentCaptor configArgumentCaptor = + ArgumentCaptor.forClass(AwsS3Sender.Config.class); + verify(builder, times(1)).createSender(configArgumentCaptor.capture()); + AwsS3Sender.Config senderConfig = configArgumentCaptor.getValue(); + assertEquals("https://foo.bar.org", senderConfig.getEndpoint()); + assertEquals(Region.US_EAST_1.toString(), senderConfig.getRegion()); + assertEquals("ACCESSKEYID", senderConfig.getAwsAccessKeyId()); + assertEquals("SECRETACCESSKEY", senderConfig.getAwsSecretAccessKey()); + assertEquals(4, senderConfig.getRetryMax()); + assertEquals(543, senderConfig.getRetryIntervalMs()); + assertEquals(65432, senderConfig.getMaxRetryIntervalMs()); + assertEquals(1.234f, senderConfig.getRetryFactor()); + assertEquals(99 * 1024, senderConfig.getWorkBufSize()); + assertFalse(senderConfig.isCompressionEnabled()); + + ArgumentCaptor recordFormatterArgumentCaptor = + ArgumentCaptor.forClass(RecordFormatter.class); + ArgumentCaptor ingesterArgumentCaptor = ArgumentCaptor.forClass(Ingester.class); + verify(builder, times(1)) + .buildFromIngester( + recordFormatterArgumentCaptor.capture(), ingesterArgumentCaptor.capture()); + + RecordFormatter recordFormatter = recordFormatterArgumentCaptor.getValue(); + assertTrue(recordFormatter instanceof JsonlRecordFormatter); + + AwsS3Ingester ingester = (AwsS3Ingester) ingesterArgumentCaptor.getValue(); + assertEquals(sender, ingester.getSender()); + DefaultS3DestinationDecider destinationDecider = + (DefaultS3DestinationDecider) ingester.getS3DestinationDecider(); + assertEquals("mydata", destinationDecider.getKeyPrefix()); + assertEquals(".zzz", destinationDecider.getKeySuffix()); + assertEquals(ZoneId.of("JST", SHORT_IDS), destinationDecider.getZoneId()); + + ArgumentCaptor bufferConfigArgumentCaptor = + ArgumentCaptor.forClass(Buffer.Config.class); + ArgumentCaptor flusherConfigArgumentCaptor = + ArgumentCaptor.forClass(Flusher.Config.class); + verify(builder, times(1)) + .createFluency( + eq(recordFormatter), + eq(ingester), + bufferConfigArgumentCaptor.capture(), + flusherConfigArgumentCaptor.capture()); + + Buffer.Config bufferConfig = bufferConfigArgumentCaptor.getValue(); + assertEquals(42 * 1024 * 1024, bufferConfig.getMaxBufferSize()); + assertEquals(2 * 1024 * 1024, bufferConfig.getChunkInitialSize()); + assertEquals(9 * 1024 * 1024, bufferConfig.getChunkRetentionSize()); + assertEquals(1234, bufferConfig.getChunkRetentionTimeMillis()); + } } diff --git a/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/ingester/AwsS3IngesterTest.java b/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/ingester/AwsS3IngesterTest.java index b30131d7..b6882a02 100644 --- a/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/ingester/AwsS3IngesterTest.java +++ b/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/ingester/AwsS3IngesterTest.java @@ -16,15 +16,6 @@ package org.komamitsu.fluency.aws.s3.ingester; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.aws.s3.ingester.sender.AwsS3Sender; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.time.Instant; - import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; @@ -33,42 +24,46 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -class AwsS3IngesterTest -{ - private AwsS3Sender s3Sender; - private S3DestinationDecider destinationDecider; - private AwsS3Ingester ingester; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.aws.s3.ingester.sender.AwsS3Sender; + +class AwsS3IngesterTest { + private AwsS3Sender s3Sender; + private S3DestinationDecider destinationDecider; + private AwsS3Ingester ingester; - @BeforeEach - void setUp() - { - s3Sender = mock(AwsS3Sender.class); + @BeforeEach + void setUp() { + s3Sender = mock(AwsS3Sender.class); - destinationDecider = mock(S3DestinationDecider.class); - doReturn(new S3DestinationDecider.S3Destination("mybucket", "my/key/base.data")) - .when(destinationDecider).decide(anyString(), any(Instant.class)); + destinationDecider = mock(S3DestinationDecider.class); + doReturn(new S3DestinationDecider.S3Destination("mybucket", "my/key/base.data")) + .when(destinationDecider) + .decide(anyString(), any(Instant.class)); - ingester = new AwsS3Ingester(s3Sender, destinationDecider); - } + ingester = new AwsS3Ingester(s3Sender, destinationDecider); + } - @Test - void ingest() - throws IOException - { - ingester.ingest("foo.bar", - ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))); + @Test + void ingest() throws IOException { + ingester.ingest("foo.bar", ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))); - verify(s3Sender, times(1)) - .send(eq("mybucket"), - eq("my/key/base.data"), - eq(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8)))); - } + verify(s3Sender, times(1)) + .send( + eq("mybucket"), + eq("my/key/base.data"), + eq(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8)))); + } - @Test - void close() - { - ingester.close(); + @Test + void close() { + ingester.close(); - verify(s3Sender, times(1)).close(); - } -} \ No newline at end of file + verify(s3Sender, times(1)).close(); + } +} diff --git a/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/ingester/DefaultS3DestinationDeciderTest.java b/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/ingester/DefaultS3DestinationDeciderTest.java index 78e96cbc..02c45bb2 100644 --- a/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/ingester/DefaultS3DestinationDeciderTest.java +++ b/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/ingester/DefaultS3DestinationDeciderTest.java @@ -16,58 +16,53 @@ package org.komamitsu.fluency.aws.s3.ingester; -import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; import java.time.ZoneId; import java.time.ZonedDateTime; +import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -class DefaultS3DestinationDeciderTest -{ - // +09:00 - private static ZoneId TIMEZONE_JST = ZoneId.of("JST", ZoneId.SHORT_IDS); +class DefaultS3DestinationDeciderTest { + // +09:00 + private static ZoneId TIMEZONE_JST = ZoneId.of("JST", ZoneId.SHORT_IDS); - @Test - void decide() - { - DefaultS3DestinationDecider.Config config = new DefaultS3DestinationDecider.Config(); - config.setKeyPrefix("archives"); - config.setKeySuffix(".testdata"); - DefaultS3DestinationDecider decider = new DefaultS3DestinationDecider(config); - ZonedDateTime time = ZonedDateTime.of(2019, 12, 31, 23, 59, 59, 999999000, TIMEZONE_JST); - S3DestinationDecider.S3Destination destination = - decider.decide("web.access_log", time.toInstant()); - assertEquals("web.access_log", destination.getBucket()); - // JST is 9 hours ahead of UTC - assertEquals("archives/2019/12/31/14/59-59-999999.testdata", destination.getKey()); - } + @Test + void decide() { + DefaultS3DestinationDecider.Config config = new DefaultS3DestinationDecider.Config(); + config.setKeyPrefix("archives"); + config.setKeySuffix(".testdata"); + DefaultS3DestinationDecider decider = new DefaultS3DestinationDecider(config); + ZonedDateTime time = ZonedDateTime.of(2019, 12, 31, 23, 59, 59, 999999000, TIMEZONE_JST); + S3DestinationDecider.S3Destination destination = + decider.decide("web.access_log", time.toInstant()); + assertEquals("web.access_log", destination.getBucket()); + // JST is 9 hours ahead of UTC + assertEquals("archives/2019/12/31/14/59-59-999999.testdata", destination.getKey()); + } - @Test - void decideWithSpecificTimeZone() - { - DefaultS3DestinationDecider.Config config = new DefaultS3DestinationDecider.Config(); - config.setKeyPrefix("archives"); - config.setKeySuffix(".testdata"); - config.setZoneId(TIMEZONE_JST); - DefaultS3DestinationDecider decider = new DefaultS3DestinationDecider(config); - ZonedDateTime time = ZonedDateTime.of(2019, 12, 31, 23, 59, 59, 999999000, TIMEZONE_JST); - S3DestinationDecider.S3Destination destination = - decider.decide("web.access_log", time.toInstant()); - assertEquals("web.access_log", destination.getBucket()); - assertEquals("archives/2019/12/31/23/59-59-999999.testdata", destination.getKey()); - } + @Test + void decideWithSpecificTimeZone() { + DefaultS3DestinationDecider.Config config = new DefaultS3DestinationDecider.Config(); + config.setKeyPrefix("archives"); + config.setKeySuffix(".testdata"); + config.setZoneId(TIMEZONE_JST); + DefaultS3DestinationDecider decider = new DefaultS3DestinationDecider(config); + ZonedDateTime time = ZonedDateTime.of(2019, 12, 31, 23, 59, 59, 999999000, TIMEZONE_JST); + S3DestinationDecider.S3Destination destination = + decider.decide("web.access_log", time.toInstant()); + assertEquals("web.access_log", destination.getBucket()); + assertEquals("archives/2019/12/31/23/59-59-999999.testdata", destination.getKey()); + } - @Test - void decideWithoutPrefixNorSuffix() - { - DefaultS3DestinationDecider.Config config = new DefaultS3DestinationDecider.Config(); - DefaultS3DestinationDecider decider = new DefaultS3DestinationDecider(config); - ZonedDateTime time = ZonedDateTime.of(2019, 12, 31, 23, 59, 59, 999999000, TIMEZONE_JST); - S3DestinationDecider.S3Destination destination = - decider.decide("web.access_log", time.toInstant()); - assertEquals("web.access_log", destination.getBucket()); - // JST is 9 hours ahead of UTC - assertEquals("2019/12/31/14/59-59-999999", destination.getKey()); - } + @Test + void decideWithoutPrefixNorSuffix() { + DefaultS3DestinationDecider.Config config = new DefaultS3DestinationDecider.Config(); + DefaultS3DestinationDecider decider = new DefaultS3DestinationDecider(config); + ZonedDateTime time = ZonedDateTime.of(2019, 12, 31, 23, 59, 59, 999999000, TIMEZONE_JST); + S3DestinationDecider.S3Destination destination = + decider.decide("web.access_log", time.toInstant()); + assertEquals("web.access_log", destination.getBucket()); + // JST is 9 hours ahead of UTC + assertEquals("2019/12/31/14/59-59-999999", destination.getKey()); + } } diff --git a/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/ingester/sender/AwsS3SenderTest.java b/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/ingester/sender/AwsS3SenderTest.java index 66d2c035..11ef07a3 100644 --- a/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/ingester/sender/AwsS3SenderTest.java +++ b/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/ingester/sender/AwsS3SenderTest.java @@ -16,7 +16,24 @@ package org.komamitsu.fluency.aws.s3.ingester.sender; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + import com.google.common.io.ByteStreams; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.zip.GZIPInputStream; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -29,151 +46,129 @@ import software.amazon.awssdk.services.s3.S3ClientBuilder; import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.zip.GZIPInputStream; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -class AwsS3SenderTest -{ - @Test - void buildClientWithDefaults() - { - AwsS3Sender.Config config = new AwsS3Sender.Config(); - - S3Client s3Client = mock(S3Client.class); - S3ClientBuilder s3ClientBuilder = mock(S3ClientBuilder.class); - doReturn(s3Client).when(s3ClientBuilder).build(); - - new AwsS3Sender(s3ClientBuilder, config); - - verify(s3ClientBuilder, times(1)).build(); - verify(s3ClientBuilder, times(0)).endpointOverride(any()); - verify(s3ClientBuilder, times(0)).region(any()); - verify(s3ClientBuilder, times(0)).credentialsProvider(any()); - } - - @Test - void buildClientWithCustomizedConfig() - { - AwsS3Sender.Config config = new AwsS3Sender.Config(); - config.setEndpoint("https://another.s3endpoi.nt"); - config.setRegion("ap-northeast-1"); - config.setAwsAccessKeyId("foo"); - config.setAwsSecretAccessKey("bar"); - - S3Client s3Client = mock(S3Client.class); - S3ClientBuilder s3ClientBuilder = mock(S3ClientBuilder.class); - doReturn(s3Client).when(s3ClientBuilder).build(); - doAnswer(invocation -> { - AwsCredentialsProvider provider = invocation.getArgument(0); - AwsCredentials awsCredentials = provider.resolveCredentials(); - assertEquals("foo", awsCredentials.accessKeyId()); - assertEquals("bar", awsCredentials.secretAccessKey()); - return null; - }).when(s3ClientBuilder).credentialsProvider(any()); - - new AwsS3Sender(s3ClientBuilder, config); - - verify(s3ClientBuilder, times(1)).build(); - verify(s3ClientBuilder, times(1)).endpointOverride(eq(URI.create("https://another.s3endpoi.nt"))); - verify(s3ClientBuilder, times(1)).region(eq(Region.AP_NORTHEAST_1)); - verify(s3ClientBuilder, times(1)).credentialsProvider(any()); - } - - private void testSend( - AwsS3Sender.Config config, - boolean gzipCompressed, - int failures) - throws IOException - { - S3Client s3Client = mock(S3Client.class); - S3ClientBuilder s3ClientBuilder = mock(S3ClientBuilder.class); - doReturn(s3Client).when(s3ClientBuilder).build(); - - AtomicInteger retryCount = new AtomicInteger(); - doAnswer(invocation -> { - PutObjectRequest request = invocation.getArgument(0); - - assertEquals("hello.world", request.bucket()); - assertEquals("2345/01/31/23/59-59-99999.data", request.key()); - - RequestBody body = invocation.getArgument(1); - try (InputStream s3In = body.contentStreamProvider().newStream(); - InputStream in = gzipCompressed ? new GZIPInputStream(s3In) : s3In) { +class AwsS3SenderTest { + @Test + void buildClientWithDefaults() { + AwsS3Sender.Config config = new AwsS3Sender.Config(); + + S3Client s3Client = mock(S3Client.class); + S3ClientBuilder s3ClientBuilder = mock(S3ClientBuilder.class); + doReturn(s3Client).when(s3ClientBuilder).build(); + + new AwsS3Sender(s3ClientBuilder, config); + + verify(s3ClientBuilder, times(1)).build(); + verify(s3ClientBuilder, times(0)).endpointOverride(any()); + verify(s3ClientBuilder, times(0)).region(any()); + verify(s3ClientBuilder, times(0)).credentialsProvider(any()); + } + + @Test + void buildClientWithCustomizedConfig() { + AwsS3Sender.Config config = new AwsS3Sender.Config(); + config.setEndpoint("https://another.s3endpoi.nt"); + config.setRegion("ap-northeast-1"); + config.setAwsAccessKeyId("foo"); + config.setAwsSecretAccessKey("bar"); + + S3Client s3Client = mock(S3Client.class); + S3ClientBuilder s3ClientBuilder = mock(S3ClientBuilder.class); + doReturn(s3Client).when(s3ClientBuilder).build(); + doAnswer( + invocation -> { + AwsCredentialsProvider provider = invocation.getArgument(0); + AwsCredentials awsCredentials = provider.resolveCredentials(); + assertEquals("foo", awsCredentials.accessKeyId()); + assertEquals("bar", awsCredentials.secretAccessKey()); + return null; + }) + .when(s3ClientBuilder) + .credentialsProvider(any()); + + new AwsS3Sender(s3ClientBuilder, config); + + verify(s3ClientBuilder, times(1)).build(); + verify(s3ClientBuilder, times(1)) + .endpointOverride(eq(URI.create("https://another.s3endpoi.nt"))); + verify(s3ClientBuilder, times(1)).region(eq(Region.AP_NORTHEAST_1)); + verify(s3ClientBuilder, times(1)).credentialsProvider(any()); + } + + private void testSend(AwsS3Sender.Config config, boolean gzipCompressed, int failures) + throws IOException { + S3Client s3Client = mock(S3Client.class); + S3ClientBuilder s3ClientBuilder = mock(S3ClientBuilder.class); + doReturn(s3Client).when(s3ClientBuilder).build(); + + AtomicInteger retryCount = new AtomicInteger(); + doAnswer( + invocation -> { + PutObjectRequest request = invocation.getArgument(0); + + assertEquals("hello.world", request.bucket()); + assertEquals("2345/01/31/23/59-59-99999.data", request.key()); + + RequestBody body = invocation.getArgument(1); + try (InputStream s3In = body.contentStreamProvider().newStream(); + InputStream in = gzipCompressed ? new GZIPInputStream(s3In) : s3In) { byte[] content = ByteStreams.toByteArray(in); assertEquals("0123456789", new String(content, StandardCharsets.UTF_8)); - } + } - if (retryCount.getAndIncrement() < failures) { + if (retryCount.getAndIncrement() < failures) { throw new RuntimeException("Something happened"); - } - - return null; - }).when(s3Client).putObject(any(PutObjectRequest.class), any(RequestBody.class)); - - AwsS3Sender sender = new AwsS3Sender(s3ClientBuilder, config); - - sender.send("hello.world", "2345/01/31/23/59-59-99999.data", - ByteBuffer.wrap("0123456789".getBytes(StandardCharsets.UTF_8))); - - verify(s3Client, times(failures + 1)).putObject(any(PutObjectRequest.class), any(RequestBody.class)); - - sender.close(); - } - - @ParameterizedTest - @ValueSource(ints={0, 2}) - void send(int failures) - throws IOException - { - AwsS3Sender.Config config = new AwsS3Sender.Config(); - testSend(config, true, failures); - } - - @ParameterizedTest - @ValueSource(ints={0, 2}) - void sendWithoutCompression(int failures) - throws IOException - { - AwsS3Sender.Config config = new AwsS3Sender.Config(); - config.setCompressionEnabled(false); - - testSend(config, false, failures); - } - - @Test - void sendButRetryOver() - { - AwsS3Sender.Config config = new AwsS3Sender.Config(); - config.setRetryMax(2); - assertThrows(RetryableException.class, () -> testSend(config, true, 3)); - } - - @Test - void close() - { - S3Client s3Client = mock(S3Client.class); - S3ClientBuilder s3ClientBuilder = mock(S3ClientBuilder.class); - doReturn(s3Client).when(s3ClientBuilder).build(); - - AwsS3Sender sender = new AwsS3Sender(s3ClientBuilder); - sender.close(); - - verify(s3Client, times(1)).close(); - } - -} \ No newline at end of file + } + + return null; + }) + .when(s3Client) + .putObject(any(PutObjectRequest.class), any(RequestBody.class)); + + AwsS3Sender sender = new AwsS3Sender(s3ClientBuilder, config); + + sender.send( + "hello.world", + "2345/01/31/23/59-59-99999.data", + ByteBuffer.wrap("0123456789".getBytes(StandardCharsets.UTF_8))); + + verify(s3Client, times(failures + 1)) + .putObject(any(PutObjectRequest.class), any(RequestBody.class)); + + sender.close(); + } + + @ParameterizedTest + @ValueSource(ints = {0, 2}) + void send(int failures) throws IOException { + AwsS3Sender.Config config = new AwsS3Sender.Config(); + testSend(config, true, failures); + } + + @ParameterizedTest + @ValueSource(ints = {0, 2}) + void sendWithoutCompression(int failures) throws IOException { + AwsS3Sender.Config config = new AwsS3Sender.Config(); + config.setCompressionEnabled(false); + + testSend(config, false, failures); + } + + @Test + void sendButRetryOver() { + AwsS3Sender.Config config = new AwsS3Sender.Config(); + config.setRetryMax(2); + assertThrows(RetryableException.class, () -> testSend(config, true, 3)); + } + + @Test + void close() { + S3Client s3Client = mock(S3Client.class); + S3ClientBuilder s3ClientBuilder = mock(S3ClientBuilder.class); + doReturn(s3Client).when(s3ClientBuilder).build(); + + AwsS3Sender sender = new AwsS3Sender(s3ClientBuilder); + sender.close(); + + verify(s3Client, times(1)).close(); + } +} diff --git a/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/recordformat/CsvRecordFormatterTest.java b/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/recordformat/CsvRecordFormatterTest.java index c3125344..89ad5fd7 100644 --- a/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/recordformat/CsvRecordFormatterTest.java +++ b/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/recordformat/CsvRecordFormatterTest.java @@ -16,181 +16,188 @@ package org.komamitsu.fluency.aws.s3.recordformat; +import static org.junit.jupiter.api.Assertions.assertEquals; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.EnumSource; -import org.msgpack.jackson.dataformat.MessagePackFactory; - import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.function.Consumer; import java.util.function.Function; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.msgpack.jackson.dataformat.MessagePackFactory; -import static org.junit.jupiter.api.Assertions.assertEquals; - -class CsvRecordFormatterTest -{ - private static final String TAG = "foodb.bartbl"; - private static final long FUTURE_EPOCH = 4294967296L; - private static final Map RECORD_0 = ImmutableMap.of("name", "first", "age", 42, "email", "hello@world.com"); - private static final Map RECORD_1 = ImmutableMap.of("name", "second", "age", 55, "time", FUTURE_EPOCH, "comment", "zzzzzz"); - private static final Map RECORD_2 = ImmutableMap.of("job", "knight", "name", "third", "age", 99, "ignored", "foobar"); - private static final Consumer CONFIGURATOR_NORMAL = config -> {}; - private static final Consumer CONFIGURATOR_QUOTE = config -> config.setQuote("\""); - private static final Consumer CONFIGURATOR_TAB_DELIM = config -> config.setDelimiter("\t"); - private static final Consumer CONFIGURATOR_QUOTE_TAB_DELIM = config -> { +class CsvRecordFormatterTest { + private static final String TAG = "foodb.bartbl"; + private static final long FUTURE_EPOCH = 4294967296L; + private static final Map RECORD_0 = + ImmutableMap.of("name", "first", "age", 42, "email", "hello@world.com"); + private static final Map RECORD_1 = + ImmutableMap.of("name", "second", "age", 55, "time", FUTURE_EPOCH, "comment", "zzzzzz"); + private static final Map RECORD_2 = + ImmutableMap.of("job", "knight", "name", "third", "age", 99, "ignored", "foobar"); + private static final Consumer CONFIGURATOR_NORMAL = config -> {}; + private static final Consumer CONFIGURATOR_QUOTE = + config -> config.setQuote("\""); + private static final Consumer CONFIGURATOR_TAB_DELIM = + config -> config.setDelimiter("\t"); + private static final Consumer CONFIGURATOR_QUOTE_TAB_DELIM = + config -> { config.setQuote("\""); config.setDelimiter("\t"); - }; - private static final Function QUOTER_NOTHING = - x -> x == null ? "" : x.toString(); - private static final Function QUOTER_QUOTE = - x -> x == null ? "" : "\"" + x + "\""; - private CsvRecordFormatter.Config config; - - enum Option { - NORMAL("%s,%s,%s,%s,,%s,%s\n", - CONFIGURATOR_NORMAL, - QUOTER_NOTHING), - QUOTE("%s,%s,%s,%s,,%s,%s\n", - CONFIGURATOR_QUOTE, - QUOTER_QUOTE), - TAB_DELIM("%s\t%s\t%s\t%s\t\t%s\t%s\n", - CONFIGURATOR_TAB_DELIM, - QUOTER_NOTHING), - QUOTE_AND_TAB_DELIM("%s\t%s\t%s\t%s\t\t%s\t%s\n", - CONFIGURATOR_QUOTE_TAB_DELIM, - QUOTER_QUOTE); - - private final String format; - private final Consumer configurator; - private final Function quoter; - - Option(String format, Consumer configurator, Function quoter) - { - this.format = format; - this.configurator = configurator; - this.quoter = quoter; - } + }; + private static final Function QUOTER_NOTHING = x -> x == null ? "" : x.toString(); + private static final Function QUOTER_QUOTE = + x -> x == null ? "" : "\"" + x + "\""; + private CsvRecordFormatter.Config config; + + enum Option { + NORMAL("%s,%s,%s,%s,,%s,%s\n", CONFIGURATOR_NORMAL, QUOTER_NOTHING), + QUOTE("%s,%s,%s,%s,,%s,%s\n", CONFIGURATOR_QUOTE, QUOTER_QUOTE), + TAB_DELIM("%s\t%s\t%s\t%s\t\t%s\t%s\n", CONFIGURATOR_TAB_DELIM, QUOTER_NOTHING), + QUOTE_AND_TAB_DELIM("%s\t%s\t%s\t%s\t\t%s\t%s\n", CONFIGURATOR_QUOTE_TAB_DELIM, QUOTER_QUOTE); + + private final String format; + private final Consumer configurator; + private final Function quoter; + + Option( + String format, + Consumer configurator, + Function quoter) { + this.format = format; + this.configurator = configurator; + this.quoter = quoter; } - - @BeforeEach - void setUp() - { - config = new CsvRecordFormatter.Config(); - config.setColumnNames(ImmutableList.of("time", "name", "age", "email", "none", "comment", "job")); - } - - private void assertRecord0(byte[] formatted, long expectedTime, Option option) - { - assertEquals( - String.format(option.format, - option.quoter.apply(expectedTime), // time - option.quoter.apply("first"), // name - option.quoter.apply(42), // age - option.quoter.apply("hello@world.com"), // email - option.quoter.apply(null), // comment - option.quoter.apply(null) // job - ), - new String(formatted, StandardCharsets.UTF_8)); - } - - private void assertRecord1(byte[] formatted, long expectedTime, Option option) - { - assertEquals( - String.format(option.format, - option.quoter.apply(expectedTime), // time - option.quoter.apply("second"), // name - option.quoter.apply(55), // age - option.quoter.apply(null), // email - option.quoter.apply("zzzzzz"), // comment - option.quoter.apply(null) // job - ), - new String(formatted, StandardCharsets.UTF_8)); - } - - private void assertRecord2(byte[] formatted, long expectedTime, Option option) - { - assertEquals( - String.format(option.format, - option.quoter.apply(expectedTime), // time - option.quoter.apply("third"), // name - option.quoter.apply(99), // age - option.quoter.apply(null), // email - option.quoter.apply(null), // comment - option.quoter.apply("knight") // job - ), - new String(formatted, StandardCharsets.UTF_8)); - } - - @ParameterizedTest - @EnumSource(Option.class) - void format(Option option) + } + + @BeforeEach + void setUp() { + config = new CsvRecordFormatter.Config(); + config.setColumnNames( + ImmutableList.of("time", "name", "age", "email", "none", "comment", "job")); + } + + private void assertRecord0(byte[] formatted, long expectedTime, Option option) { + assertEquals( + String.format( + option.format, + option.quoter.apply(expectedTime), // time + option.quoter.apply("first"), // name + option.quoter.apply(42), // age + option.quoter.apply("hello@world.com"), // email + option.quoter.apply(null), // comment + option.quoter.apply(null) // job + ), + new String(formatted, StandardCharsets.UTF_8)); + } + + private void assertRecord1(byte[] formatted, long expectedTime, Option option) { + assertEquals( + String.format( + option.format, + option.quoter.apply(expectedTime), // time + option.quoter.apply("second"), // name + option.quoter.apply(55), // age + option.quoter.apply(null), // email + option.quoter.apply("zzzzzz"), // comment + option.quoter.apply(null) // job + ), + new String(formatted, StandardCharsets.UTF_8)); + } + + private void assertRecord2(byte[] formatted, long expectedTime, Option option) { + assertEquals( + String.format( + option.format, + option.quoter.apply(expectedTime), // time + option.quoter.apply("third"), // name + option.quoter.apply(99), // age + option.quoter.apply(null), // email + option.quoter.apply(null), // comment + option.quoter.apply("knight") // job + ), + new String(formatted, StandardCharsets.UTF_8)); + } + + @ParameterizedTest + @EnumSource(Option.class) + void format(Option option) { + option.configurator.accept(config); + CsvRecordFormatter recordFormatter = new CsvRecordFormatter(config); + long now = System.currentTimeMillis() / 1000; + assertRecord0(recordFormatter.format(TAG, now, RECORD_0), now, option); + assertRecord1(recordFormatter.format(TAG, now, RECORD_1), FUTURE_EPOCH, option); + assertRecord2(recordFormatter.format(TAG, now, RECORD_2), now, option); + } + + @ParameterizedTest + @EnumSource(Option.class) + void formatFromMessagePackBytes(Option option) throws IOException { + option.configurator.accept(config); + CsvRecordFormatter recordFormatter = new CsvRecordFormatter(config); + long now = System.currentTimeMillis() / 1000; + ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); { - option.configurator.accept(config); - CsvRecordFormatter recordFormatter = new CsvRecordFormatter(config); - long now = System.currentTimeMillis() / 1000; - assertRecord0(recordFormatter.format(TAG, now, RECORD_0), now, option); - assertRecord1(recordFormatter.format(TAG, now, RECORD_1), FUTURE_EPOCH, option); - assertRecord2(recordFormatter.format(TAG, now, RECORD_2), now, option); + byte[] bytes = objectMapper.writeValueAsBytes(RECORD_0); + assertRecord0( + recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), now, option); } - - @ParameterizedTest - @EnumSource(Option.class) - void formatFromMessagePackBytes(Option option) - throws IOException { - option.configurator.accept(config); - CsvRecordFormatter recordFormatter = new CsvRecordFormatter(config); - long now = System.currentTimeMillis() / 1000; - ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); - { - byte[] bytes = objectMapper.writeValueAsBytes(RECORD_0); - assertRecord0(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), now, option); - } - { - byte[] bytes = objectMapper.writeValueAsBytes(RECORD_1); - assertRecord1(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), FUTURE_EPOCH, option); - } - { - byte[] bytes = objectMapper.writeValueAsBytes(RECORD_2); - assertRecord2(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), now, option); - } + byte[] bytes = objectMapper.writeValueAsBytes(RECORD_1); + assertRecord1( + recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), + FUTURE_EPOCH, + option); } - - private ByteBuffer convertMapToMessagePackByteBuffer(Map record, boolean isDirect) - throws JsonProcessingException { - ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); - byte[] bytes = objectMapper.writeValueAsBytes(record); - ByteBuffer byteBuffer; - if (isDirect) { - byteBuffer = ByteBuffer.allocateDirect(bytes.length); - } - else { - byteBuffer = ByteBuffer.allocate(bytes.length); - } - byteBuffer.put(bytes); - byteBuffer.flip(); - return byteBuffer; + byte[] bytes = objectMapper.writeValueAsBytes(RECORD_2); + assertRecord2( + recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), now, option); } - - @ParameterizedTest - @EnumSource(Option.class) - void formatFromMessagePackByteBuffer(Option option) - throws IOException - { - option.configurator.accept(config); - CsvRecordFormatter recordFormatter = new CsvRecordFormatter(config); - long now = System.currentTimeMillis() / 1000; - assertRecord0(recordFormatter.formatFromMessagePack(TAG, now, convertMapToMessagePackByteBuffer(RECORD_0, false)), now, option); - assertRecord1(recordFormatter.formatFromMessagePack(TAG, now, convertMapToMessagePackByteBuffer(RECORD_1, true)), FUTURE_EPOCH, option); - assertRecord2(recordFormatter.formatFromMessagePack(TAG, now, convertMapToMessagePackByteBuffer(RECORD_2, true)), now, option); + } + + private ByteBuffer convertMapToMessagePackByteBuffer(Map record, boolean isDirect) + throws JsonProcessingException { + ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); + byte[] bytes = objectMapper.writeValueAsBytes(record); + ByteBuffer byteBuffer; + if (isDirect) { + byteBuffer = ByteBuffer.allocateDirect(bytes.length); + } else { + byteBuffer = ByteBuffer.allocate(bytes.length); } -} \ No newline at end of file + byteBuffer.put(bytes); + byteBuffer.flip(); + return byteBuffer; + } + + @ParameterizedTest + @EnumSource(Option.class) + void formatFromMessagePackByteBuffer(Option option) throws IOException { + option.configurator.accept(config); + CsvRecordFormatter recordFormatter = new CsvRecordFormatter(config); + long now = System.currentTimeMillis() / 1000; + assertRecord0( + recordFormatter.formatFromMessagePack( + TAG, now, convertMapToMessagePackByteBuffer(RECORD_0, false)), + now, + option); + assertRecord1( + recordFormatter.formatFromMessagePack( + TAG, now, convertMapToMessagePackByteBuffer(RECORD_1, true)), + FUTURE_EPOCH, + option); + assertRecord2( + recordFormatter.formatFromMessagePack( + TAG, now, convertMapToMessagePackByteBuffer(RECORD_2, true)), + now, + option); + } +} diff --git a/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/recordformat/JsonlRecordFormatterTest.java b/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/recordformat/JsonlRecordFormatterTest.java index 651a2132..d00a1ce5 100644 --- a/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/recordformat/JsonlRecordFormatterTest.java +++ b/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/recordformat/JsonlRecordFormatterTest.java @@ -16,127 +16,123 @@ package org.komamitsu.fluency.aws.s3.recordformat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Map; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.msgpack.jackson.dataformat.MessagePackFactory; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Map; +class JsonlRecordFormatterTest { + private static final String TAG = "foodb.bartbl"; + private static final long FUTURE_EPOCH = 4294967296L; + private static final Map RECORD_0 = + ImmutableMap.of("name", "first", "age", 42, "email", "hello@world.com"); + private static final Map RECORD_1 = + ImmutableMap.of("name", "second", "age", 55, "time", FUTURE_EPOCH, "comment", "zzzzzz"); + private static final Map RECORD_2 = + ImmutableMap.of("job", "knight", "name", "third", "age", 99); + private final ObjectMapper objectMapper = new ObjectMapper(); + private JsonlRecordFormatter recordFormatter; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; + @BeforeEach + void setUp() { + recordFormatter = new JsonlRecordFormatter(new JsonlRecordFormatter.Config()); + } -class JsonlRecordFormatterTest -{ - private static final String TAG = "foodb.bartbl"; - private static final long FUTURE_EPOCH = 4294967296L; - private static final Map RECORD_0 = ImmutableMap.of("name", "first", "age", 42, "email", "hello@world.com"); - private static final Map RECORD_1 = ImmutableMap.of("name", "second", "age", 55, "time", FUTURE_EPOCH, "comment", "zzzzzz"); - private static final Map RECORD_2 = ImmutableMap.of("job", "knight", "name", "third", "age", 99); - private final ObjectMapper objectMapper = new ObjectMapper(); - private JsonlRecordFormatter recordFormatter; + private void assertRecord0(byte[] formatted, long expectedTime) throws IOException { + JsonNode jsonNode = objectMapper.readTree(formatted); + assertTrue(jsonNode.isObject()); + assertEquals(4, jsonNode.size()); + assertEquals(expectedTime, jsonNode.get("time").asLong()); + assertEquals("first", jsonNode.get("name").asText()); + assertEquals(42, jsonNode.get("age").asInt()); + assertEquals("hello@world.com", jsonNode.get("email").asText()); + } - @BeforeEach - void setUp() - { - recordFormatter = new JsonlRecordFormatter(new JsonlRecordFormatter.Config()); - } + private void assertRecord1(byte[] formatted, long expectedTime) throws IOException { + JsonNode jsonNode = objectMapper.readTree(formatted); + assertTrue(jsonNode.isObject()); + assertEquals(4, jsonNode.size()); + assertEquals(expectedTime, jsonNode.get("time").asLong()); + assertEquals("second", jsonNode.get("name").asText()); + assertEquals(55, jsonNode.get("age").asInt()); + assertEquals("zzzzzz", jsonNode.get("comment").asText()); + } - private void assertRecord0(byte[] formatted, long expectedTime) - throws IOException - { - JsonNode jsonNode = objectMapper.readTree(formatted); - assertTrue(jsonNode.isObject()); - assertEquals(4, jsonNode.size()); - assertEquals(expectedTime, jsonNode.get("time").asLong()); - assertEquals("first", jsonNode.get("name").asText()); - assertEquals(42, jsonNode.get("age").asInt()); - assertEquals("hello@world.com", jsonNode.get("email").asText()); - } + private void assertRecord2(byte[] formatted, long expectedTime) throws IOException { + JsonNode jsonNode = objectMapper.readTree(formatted); + assertTrue(jsonNode.isObject()); + assertEquals(4, jsonNode.size()); + assertEquals(expectedTime, jsonNode.get("time").asLong()); + assertEquals("third", jsonNode.get("name").asText()); + assertEquals(99, jsonNode.get("age").asInt()); + assertEquals("knight", jsonNode.get("job").asText()); + } - private void assertRecord1(byte[] formatted, long expectedTime) - throws IOException - { - JsonNode jsonNode = objectMapper.readTree(formatted); - assertTrue(jsonNode.isObject()); - assertEquals(4, jsonNode.size()); - assertEquals(expectedTime, jsonNode.get("time").asLong()); - assertEquals("second", jsonNode.get("name").asText()); - assertEquals(55, jsonNode.get("age").asInt()); - assertEquals("zzzzzz", jsonNode.get("comment").asText()); - } + @Test + void format() throws IOException { + long now = System.currentTimeMillis() / 1000; + assertRecord0(recordFormatter.format(TAG, now, RECORD_0), now); + assertRecord1(recordFormatter.format(TAG, now, RECORD_1), FUTURE_EPOCH); + assertRecord2(recordFormatter.format(TAG, now, RECORD_2), now); + } - private void assertRecord2(byte[] formatted, long expectedTime) - throws IOException + @Test + void formatFromMessagePackBytes() throws IOException { + long now = System.currentTimeMillis() / 1000; + ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); { - JsonNode jsonNode = objectMapper.readTree(formatted); - assertTrue(jsonNode.isObject()); - assertEquals(4, jsonNode.size()); - assertEquals(expectedTime, jsonNode.get("time").asLong()); - assertEquals("third", jsonNode.get("name").asText()); - assertEquals(99, jsonNode.get("age").asInt()); - assertEquals("knight", jsonNode.get("job").asText()); + byte[] bytes = objectMapper.writeValueAsBytes(RECORD_0); + assertRecord0(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), now); } - - @Test - void format() - throws IOException { - long now = System.currentTimeMillis() / 1000; - assertRecord0(recordFormatter.format(TAG, now, RECORD_0), now); - assertRecord1(recordFormatter.format(TAG, now, RECORD_1), FUTURE_EPOCH); - assertRecord2(recordFormatter.format(TAG, now, RECORD_2), now); + byte[] bytes = objectMapper.writeValueAsBytes(RECORD_1); + assertRecord1( + recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), FUTURE_EPOCH); } - - @Test - void formatFromMessagePackBytes() - throws IOException { - long now = System.currentTimeMillis() / 1000; - ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); - { - byte[] bytes = objectMapper.writeValueAsBytes(RECORD_0); - assertRecord0(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), now); - } - { - byte[] bytes = objectMapper.writeValueAsBytes(RECORD_1); - assertRecord1(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), FUTURE_EPOCH); - } - { - byte[] bytes = objectMapper.writeValueAsBytes(RECORD_2); - assertRecord2(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), now); - } + byte[] bytes = objectMapper.writeValueAsBytes(RECORD_2); + assertRecord2(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), now); } + } - private ByteBuffer convertMapToMessagePackByteBuffer(Map record, boolean isDirect) - throws JsonProcessingException - { - ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); - byte[] bytes = objectMapper.writeValueAsBytes(record); - ByteBuffer byteBuffer; - if (isDirect) { - byteBuffer = ByteBuffer.allocateDirect(bytes.length); - } - else { - byteBuffer = ByteBuffer.allocate(bytes.length); - } - byteBuffer.put(bytes); - byteBuffer.flip(); - return byteBuffer; + private ByteBuffer convertMapToMessagePackByteBuffer(Map record, boolean isDirect) + throws JsonProcessingException { + ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); + byte[] bytes = objectMapper.writeValueAsBytes(record); + ByteBuffer byteBuffer; + if (isDirect) { + byteBuffer = ByteBuffer.allocateDirect(bytes.length); + } else { + byteBuffer = ByteBuffer.allocate(bytes.length); } + byteBuffer.put(bytes); + byteBuffer.flip(); + return byteBuffer; + } - @Test - void formatFromMessagePackByteBuffer() - throws IOException - { - long now = System.currentTimeMillis() / 1000; - assertRecord0(recordFormatter.formatFromMessagePack(TAG, now, convertMapToMessagePackByteBuffer(RECORD_0, false)), now); - assertRecord1(recordFormatter.formatFromMessagePack(TAG, now, convertMapToMessagePackByteBuffer(RECORD_1, true)), FUTURE_EPOCH); - assertRecord2(recordFormatter.formatFromMessagePack(TAG, now, convertMapToMessagePackByteBuffer(RECORD_2, true)), now); - } -} \ No newline at end of file + @Test + void formatFromMessagePackByteBuffer() throws IOException { + long now = System.currentTimeMillis() / 1000; + assertRecord0( + recordFormatter.formatFromMessagePack( + TAG, now, convertMapToMessagePackByteBuffer(RECORD_0, false)), + now); + assertRecord1( + recordFormatter.formatFromMessagePack( + TAG, now, convertMapToMessagePackByteBuffer(RECORD_1, true)), + FUTURE_EPOCH); + assertRecord2( + recordFormatter.formatFromMessagePack( + TAG, now, convertMapToMessagePackByteBuffer(RECORD_2, true)), + now); + } +} diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/BufferFullException.java b/fluency-core/src/main/java/org/komamitsu/fluency/BufferFullException.java index c0363c4d..213eb775 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/BufferFullException.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/BufferFullException.java @@ -18,11 +18,8 @@ import java.io.IOException; -public class BufferFullException - extends IOException -{ - public BufferFullException(String s) - { - super(s); - } +public class BufferFullException extends IOException { + public BufferFullException(String s) { + super(s); + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/EventTime.java b/fluency-core/src/main/java/org/komamitsu/fluency/EventTime.java index a78e2434..5a7da282 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/EventTime.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/EventTime.java @@ -20,160 +20,140 @@ import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.fasterxml.jackson.databind.ser.std.StdSerializer; -import org.msgpack.jackson.dataformat.MessagePackExtensionType; -import org.msgpack.jackson.dataformat.MessagePackGenerator; - import java.io.IOException; import java.nio.ByteBuffer; +import org.msgpack.jackson.dataformat.MessagePackExtensionType; +import org.msgpack.jackson.dataformat.MessagePackGenerator; @JsonSerialize(using = EventTime.Serializer.class) -public class EventTime -{ - private final long seconds; - private final long nanoseconds; - - /** - * Constructs an EventTime. - * - * @param seconds the epoch seconds. This should be a 32-bit value. - * @param nanoseconds the nanoseconds. This should be a 32-bit value. - */ - public EventTime(long seconds, long nanoseconds) - { - if (seconds >> 32 != 0) { - throw new IllegalArgumentException("`seconds` should be a 32-bit value"); - } - - if (nanoseconds >> 32 != 0) { - throw new IllegalArgumentException("`nanoseconds` should be a 32-bit value"); - } - - this.seconds = seconds; - this.nanoseconds = nanoseconds; - } - - /** - * Constructs an EventTime. - * - * @param epochSeconds the epoch seconds. This should be a 32-bit value. - */ - public static EventTime fromEpoch(long epochSeconds) - { - return new EventTime(epochSeconds, 0); +public class EventTime { + private final long seconds; + private final long nanoseconds; + + /** + * Constructs an EventTime. + * + * @param seconds the epoch seconds. This should be a 32-bit value. + * @param nanoseconds the nanoseconds. This should be a 32-bit value. + */ + public EventTime(long seconds, long nanoseconds) { + if (seconds >> 32 != 0) { + throw new IllegalArgumentException("`seconds` should be a 32-bit value"); } - /** - * Constructs an EventTime. - * - * @param epochSeconds the epoch seconds. This should be a 32-bit value. - * @param nanoseconds the nanoseconds. This should be a 32-bit value. - */ - public static EventTime fromEpoch(long epochSeconds, long nanoseconds) - { - return new EventTime(epochSeconds, nanoseconds); + if (nanoseconds >> 32 != 0) { + throw new IllegalArgumentException("`nanoseconds` should be a 32-bit value"); } - /** - * Constructs an EventTime. - * - * @param epochMillisecond the epoch milli seconds. - * This should be a 32-bit value. - */ - public static EventTime fromEpochMilli(long epochMillisecond) - { - return new EventTime(epochMillisecond / 1000, (epochMillisecond % 1000) * 1000000); + this.seconds = seconds; + this.nanoseconds = nanoseconds; + } + + /** + * Constructs an EventTime. + * + * @param epochSeconds the epoch seconds. This should be a 32-bit value. + */ + public static EventTime fromEpoch(long epochSeconds) { + return new EventTime(epochSeconds, 0); + } + + /** + * Constructs an EventTime. + * + * @param epochSeconds the epoch seconds. This should be a 32-bit value. + * @param nanoseconds the nanoseconds. This should be a 32-bit value. + */ + public static EventTime fromEpoch(long epochSeconds, long nanoseconds) { + return new EventTime(epochSeconds, nanoseconds); + } + + /** + * Constructs an EventTime. + * + * @param epochMillisecond the epoch milli seconds. This should be a 32-bit value. + */ + public static EventTime fromEpochMilli(long epochMillisecond) { + return new EventTime(epochMillisecond / 1000, (epochMillisecond % 1000) * 1000000); + } + + public long getSeconds() { + return seconds; + } + + public long getNanoseconds() { + return nanoseconds; + } + + /** @deprecated As of release 1.9, replaced by {@link #getNanoseconds()} */ + @Deprecated + public long getNanoSeconds() { + return nanoseconds; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - public long getSeconds() - { - return seconds; + if (!(o instanceof EventTime)) { + return false; } - public long getNanoseconds() - { - return nanoseconds; - } + EventTime eventTime = (EventTime) o; - /** - * @deprecated As of release 1.9, replaced by {@link #getNanoseconds()} - */ - @Deprecated - public long getNanoSeconds() - { - return nanoseconds; + if (seconds != eventTime.seconds) { + return false; } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (!(o instanceof EventTime)) { - return false; - } - - EventTime eventTime = (EventTime) o; - - if (seconds != eventTime.seconds) { - return false; - } - return nanoseconds == eventTime.nanoseconds; + return nanoseconds == eventTime.nanoseconds; + } + + @Override + public int hashCode() { + int result = (int) (seconds ^ (seconds >>> 32)); + result = 31 * result + (int) (nanoseconds ^ (nanoseconds >>> 32)); + return result; + } + + @Override + public String toString() { + return "EventTime{" + "seconds=" + seconds + ", nanoseconds=" + nanoseconds + '}'; + } + + public static class Serializer extends StdSerializer { + public Serializer() { + super(EventTime.class); } - @Override - public int hashCode() - { - int result = (int) (seconds ^ (seconds >>> 32)); - result = 31 * result + (int) (nanoseconds ^ (nanoseconds >>> 32)); - return result; + protected Serializer(Class t) { + super(t); } @Override - public String toString() - { - return "EventTime{" + - "seconds=" + seconds + - ", nanoseconds=" + nanoseconds + - '}'; - } - - public static class Serializer - extends StdSerializer - { - public Serializer() - { - super(EventTime.class); - } - - protected Serializer(Class t) - { - super(t); - } - - @Override - public void serialize(EventTime value, JsonGenerator gen, SerializerProvider provider) - throws IOException - { - if (!(gen instanceof MessagePackGenerator)) { - throw new IllegalStateException("" + - "This class should be serialized by MessagePackGenerator, but `gen` is " + gen.getClass()); - } - - MessagePackGenerator messagePackGenerator = (MessagePackGenerator) gen; - - // https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#eventtime-ext-format - // - // +-------+----+----+----+----+----+----+----+----+----+ - // | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | - // +-------+----+----+----+----+----+----+----+----+----+ - // | D7 | 00 | seconds from epoch| nanosecond | - // +-------+----+----+----+----+----+----+----+----+----+ - // |fixext8|type| 32bits integer BE | 32bits integer BE | - // +-------+----+----+----+----+----+----+----+----+----+ - ByteBuffer buffer = ByteBuffer.allocate(8); - buffer.putInt((int) value.seconds).putInt((int) value.nanoseconds); - messagePackGenerator.writeExtensionType(new MessagePackExtensionType((byte) 0x0, buffer.array())); - } + public void serialize(EventTime value, JsonGenerator gen, SerializerProvider provider) + throws IOException { + if (!(gen instanceof MessagePackGenerator)) { + throw new IllegalStateException( + "" + + "This class should be serialized by MessagePackGenerator, but `gen` is " + + gen.getClass()); + } + + MessagePackGenerator messagePackGenerator = (MessagePackGenerator) gen; + + // https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#eventtime-ext-format + // + // +-------+----+----+----+----+----+----+----+----+----+ + // | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | + // +-------+----+----+----+----+----+----+----+----+----+ + // | D7 | 00 | seconds from epoch| nanosecond | + // +-------+----+----+----+----+----+----+----+----+----+ + // |fixext8|type| 32bits integer BE | 32bits integer BE | + // +-------+----+----+----+----+----+----+----+----+----+ + ByteBuffer buffer = ByteBuffer.allocate(8); + buffer.putInt((int) value.seconds).putInt((int) value.nanoseconds); + messagePackGenerator.writeExtensionType( + new MessagePackExtensionType((byte) 0x0, buffer.array())); } + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/Fluency.java b/fluency-core/src/main/java/org/komamitsu/fluency/Fluency.java index 9b5bdf6c..6bc051a3 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/Fluency.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/Fluency.java @@ -16,191 +16,161 @@ package org.komamitsu.fluency; -import org.komamitsu.fluency.buffer.Buffer; -import org.komamitsu.fluency.flusher.Flusher; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.Closeable; import java.io.Flushable; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Map; import java.util.concurrent.TimeUnit; +import org.komamitsu.fluency.buffer.Buffer; +import org.komamitsu.fluency.flusher.Flusher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class Fluency - implements Flushable, Closeable -{ - private static final Logger LOG = LoggerFactory.getLogger(Fluency.class); - private final Buffer buffer; - private final Flusher flusher; - private final Emitter emitter = new Emitter(); - - // Fluency has this public constructor, but using FluentBuilder is usually more recommended. - public Fluency(Buffer buffer, Flusher flusher) - { - this.buffer = buffer; - this.flusher = flusher; - } - - public void emit(final String tag, final long timestamp, final Map data) - throws IOException - { - emitter.emit(() -> buffer.append(tag, timestamp, data)); - } - - public void emit(String tag, Map data) - throws IOException - { - emit(tag, System.currentTimeMillis() / 1000, data); - } - - public void emit(final String tag, final EventTime eventTime, final Map data) - throws IOException - { - emitter.emit(() -> buffer.append(tag, eventTime, data)); - } - - public void emit(final String tag, final long timestamp, final byte[] mapValue, final int offset, final int len) - throws IOException - { - emitter.emit(() -> buffer.appendMessagePackMapValue(tag, timestamp, mapValue, offset, len)); - } - - public void emit(String tag, byte[] mapValue, int offset, int len) - throws IOException - { - emit(tag, System.currentTimeMillis() / 1000, mapValue, offset, len); - } - - public void emit(final String tag, final EventTime eventTime, final byte[] mapValue, final int offset, final int len) - throws IOException - { - emitter.emit(() -> buffer.appendMessagePackMapValue(tag, eventTime, mapValue, offset, len)); - } - - public void emit(final String tag, final long timestamp, final ByteBuffer mapValue) - throws IOException - { - emitter.emit(() -> buffer.appendMessagePackMapValue(tag, timestamp, mapValue)); - } - - public void emit(String tag, ByteBuffer mapValue) - throws IOException - { - emit(tag, System.currentTimeMillis() / 1000, mapValue); - } - - public void emit(final String tag, final EventTime eventTime, final ByteBuffer mapValue) - throws IOException - { - emitter.emit(() -> buffer.appendMessagePackMapValue(tag, eventTime, mapValue)); - } - - @Override - public void flush() - throws IOException - { +public class Fluency implements Flushable, Closeable { + private static final Logger LOG = LoggerFactory.getLogger(Fluency.class); + private final Buffer buffer; + private final Flusher flusher; + private final Emitter emitter = new Emitter(); + + // Fluency has this public constructor, but using FluentBuilder is usually more recommended. + public Fluency(Buffer buffer, Flusher flusher) { + this.buffer = buffer; + this.flusher = flusher; + } + + public void emit(final String tag, final long timestamp, final Map data) + throws IOException { + emitter.emit(() -> buffer.append(tag, timestamp, data)); + } + + public void emit(String tag, Map data) throws IOException { + emit(tag, System.currentTimeMillis() / 1000, data); + } + + public void emit(final String tag, final EventTime eventTime, final Map data) + throws IOException { + emitter.emit(() -> buffer.append(tag, eventTime, data)); + } + + public void emit( + final String tag, + final long timestamp, + final byte[] mapValue, + final int offset, + final int len) + throws IOException { + emitter.emit(() -> buffer.appendMessagePackMapValue(tag, timestamp, mapValue, offset, len)); + } + + public void emit(String tag, byte[] mapValue, int offset, int len) throws IOException { + emit(tag, System.currentTimeMillis() / 1000, mapValue, offset, len); + } + + public void emit( + final String tag, + final EventTime eventTime, + final byte[] mapValue, + final int offset, + final int len) + throws IOException { + emitter.emit(() -> buffer.appendMessagePackMapValue(tag, eventTime, mapValue, offset, len)); + } + + public void emit(final String tag, final long timestamp, final ByteBuffer mapValue) + throws IOException { + emitter.emit(() -> buffer.appendMessagePackMapValue(tag, timestamp, mapValue)); + } + + public void emit(String tag, ByteBuffer mapValue) throws IOException { + emit(tag, System.currentTimeMillis() / 1000, mapValue); + } + + public void emit(final String tag, final EventTime eventTime, final ByteBuffer mapValue) + throws IOException { + emitter.emit(() -> buffer.appendMessagePackMapValue(tag, eventTime, mapValue)); + } + + @Override + public void flush() throws IOException { + flusher.flush(); + } + + @Override + public void close() throws IOException { + flusher.close(); + } + + public void clearBackupFiles() { + buffer.clearBackupFiles(); + } + + public long getAllocatedBufferSize() { + return buffer.getAllocatedSize(); + } + + public long getBufferedDataSize() { + return buffer.getBufferedDataSize(); + } + + public boolean isTerminated() { + return flusher.isTerminated(); + } + + public boolean waitUntilAllBufferFlushed(int maxWaitSeconds) throws InterruptedException { + int intervalMilli = 500; + for (int i = 0; i < maxWaitSeconds * (1000 / intervalMilli); i++) { + long bufferedDataSize = getBufferedDataSize(); + LOG.debug("Waiting for flushing all buffer: {}", bufferedDataSize); + if (bufferedDataSize == 0) { + return true; + } + TimeUnit.MILLISECONDS.sleep(intervalMilli); + } + LOG.warn("Buffered data still remains: {}", getBufferedDataSize()); + return false; + } + + public boolean waitUntilFlusherTerminated(int maxWaitSeconds) throws InterruptedException { + int intervalMilli = 500; + for (int i = 0; i < maxWaitSeconds * (1000 / intervalMilli); i++) { + boolean terminated = isTerminated(); + LOG.debug("Waiting until the flusher is terminated: {}", terminated); + if (terminated) { + return true; + } + TimeUnit.MILLISECONDS.sleep(intervalMilli); + } + LOG.warn("The flusher isn't terminated"); + return false; + } + + public Buffer getBuffer() { + return buffer; + } + + public Flusher getFlusher() { + return flusher; + } + + @Override + public String toString() { + return "Fluency{" + "buffer=" + buffer + ", flusher=" + flusher + '}'; + } + + private interface Append { + void append() throws IOException; + } + + private class Emitter { + void emit(Append appender) throws IOException { + try { + appender.append(); + } catch (BufferFullException e) { + LOG.error("emit() failed due to buffer full. Flushing buffer. Please try again..."); flusher.flush(); + throw e; + } } - - @Override - public void close() - throws IOException - { - flusher.close(); - } - - public void clearBackupFiles() - { - buffer.clearBackupFiles(); - } - - public long getAllocatedBufferSize() - { - return buffer.getAllocatedSize(); - } - - public long getBufferedDataSize() - { - return buffer.getBufferedDataSize(); - } - - public boolean isTerminated() - { - return flusher.isTerminated(); - } - - public boolean waitUntilAllBufferFlushed(int maxWaitSeconds) - throws InterruptedException - { - int intervalMilli = 500; - for (int i = 0; i < maxWaitSeconds * (1000 / intervalMilli); i++) { - long bufferedDataSize = getBufferedDataSize(); - LOG.debug("Waiting for flushing all buffer: {}", bufferedDataSize); - if (bufferedDataSize == 0) { - return true; - } - TimeUnit.MILLISECONDS.sleep(intervalMilli); - } - LOG.warn("Buffered data still remains: {}", getBufferedDataSize()); - return false; - } - - public boolean waitUntilFlusherTerminated(int maxWaitSeconds) - throws InterruptedException - { - int intervalMilli = 500; - for (int i = 0; i < maxWaitSeconds * (1000 / intervalMilli); i++) { - boolean terminated = isTerminated(); - LOG.debug("Waiting until the flusher is terminated: {}", terminated); - if (terminated) { - return true; - } - TimeUnit.MILLISECONDS.sleep(intervalMilli); - } - LOG.warn("The flusher isn't terminated"); - return false; - } - - public Buffer getBuffer() - { - return buffer; - } - - public Flusher getFlusher() - { - return flusher; - } - - @Override - public String toString() - { - return "Fluency{" + - "buffer=" + buffer + - ", flusher=" + flusher + - '}'; - } - - private interface Append - { - void append() - throws IOException; - } - - private class Emitter - { - void emit(Append appender) - throws IOException - { - try { - appender.append(); - } - catch (BufferFullException e) { - LOG.error("emit() failed due to buffer full. Flushing buffer. Please try again..."); - flusher.flush(); - throw e; - } - } - } + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/FluencyBuilder.java b/fluency-core/src/main/java/org/komamitsu/fluency/FluencyBuilder.java index 50179baf..97153381 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/FluencyBuilder.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/FluencyBuilder.java @@ -23,218 +23,200 @@ import org.komamitsu.fluency.recordformat.RecordFormatter; import org.msgpack.core.annotations.VisibleForTesting; -public class FluencyBuilder -{ - private Long maxBufferSize; - private Integer bufferChunkInitialSize; - private Integer bufferChunkRetentionSize; - private Integer bufferChunkRetentionTimeMillis; - private Integer flushAttemptIntervalMillis; - private String fileBackupDir; - private Integer waitUntilBufferFlushed; - private Integer waitUntilFlusherTerminated; - private Boolean jvmHeapBufferMode; - private ErrorHandler errorHandler; - - // TODO: Contain sender's retry setting here? - - public Long getMaxBufferSize() - { - return maxBufferSize; - } - - public void setMaxBufferSize(Long maxBufferSize) - { - this.maxBufferSize = maxBufferSize; - } - - public Integer getBufferChunkInitialSize() - { - return bufferChunkInitialSize; - } - - public void setBufferChunkInitialSize(Integer bufferChunkInitialSize) - { - this.bufferChunkInitialSize = bufferChunkInitialSize; - } - - public Integer getBufferChunkRetentionSize() - { - return bufferChunkRetentionSize; - } - - public void setBufferChunkRetentionSize(Integer bufferChunkRetentionSize) - { - this.bufferChunkRetentionSize = bufferChunkRetentionSize; - } - - public Integer getBufferChunkRetentionTimeMillis() - { - return bufferChunkRetentionTimeMillis; - } - - public void setBufferChunkRetentionTimeMillis(Integer bufferChunkRetentionTimeMillis) - { - this.bufferChunkRetentionTimeMillis = bufferChunkRetentionTimeMillis; - } - - /** - * @deprecated As of release 2.4.0, replaced by {@link #getFlushAttemptIntervalMillis()} - */ - @Deprecated - public Integer getFlushIntervalMillis() - { - return flushAttemptIntervalMillis; - } - - /** - * @deprecated As of release 2.4.0, replaced by {@link #setFlushAttemptIntervalMillis(Integer flushAttemptIntervalMillis)} - */ - @Deprecated - public void setFlushIntervalMillis(Integer flushAttemptIntervalMillis) - { - this.flushAttemptIntervalMillis = flushAttemptIntervalMillis; - } - - public Integer getFlushAttemptIntervalMillis() - { - return flushAttemptIntervalMillis; - } +public class FluencyBuilder { + private Long maxBufferSize; + private Integer bufferChunkInitialSize; + private Integer bufferChunkRetentionSize; + private Integer bufferChunkRetentionTimeMillis; + private Integer flushAttemptIntervalMillis; + private String fileBackupDir; + private Integer waitUntilBufferFlushed; + private Integer waitUntilFlusherTerminated; + private Boolean jvmHeapBufferMode; + private ErrorHandler errorHandler; + + // TODO: Contain sender's retry setting here? + + public Long getMaxBufferSize() { + return maxBufferSize; + } + + public void setMaxBufferSize(Long maxBufferSize) { + this.maxBufferSize = maxBufferSize; + } + + public Integer getBufferChunkInitialSize() { + return bufferChunkInitialSize; + } + + public void setBufferChunkInitialSize(Integer bufferChunkInitialSize) { + this.bufferChunkInitialSize = bufferChunkInitialSize; + } + + public Integer getBufferChunkRetentionSize() { + return bufferChunkRetentionSize; + } + + public void setBufferChunkRetentionSize(Integer bufferChunkRetentionSize) { + this.bufferChunkRetentionSize = bufferChunkRetentionSize; + } + + public Integer getBufferChunkRetentionTimeMillis() { + return bufferChunkRetentionTimeMillis; + } + + public void setBufferChunkRetentionTimeMillis(Integer bufferChunkRetentionTimeMillis) { + this.bufferChunkRetentionTimeMillis = bufferChunkRetentionTimeMillis; + } + + /** @deprecated As of release 2.4.0, replaced by {@link #getFlushAttemptIntervalMillis()} */ + @Deprecated + public Integer getFlushIntervalMillis() { + return flushAttemptIntervalMillis; + } + + /** + * @deprecated As of release 2.4.0, replaced by {@link #setFlushAttemptIntervalMillis(Integer + * flushAttemptIntervalMillis)} + */ + @Deprecated + public void setFlushIntervalMillis(Integer flushAttemptIntervalMillis) { + this.flushAttemptIntervalMillis = flushAttemptIntervalMillis; + } + + public Integer getFlushAttemptIntervalMillis() { + return flushAttemptIntervalMillis; + } + + public void setFlushAttemptIntervalMillis(Integer flushAttemptIntervalMillis) { + this.flushAttemptIntervalMillis = flushAttemptIntervalMillis; + } + + public String getFileBackupDir() { + return fileBackupDir; + } + + public void setFileBackupDir(String fileBackupDir) { + this.fileBackupDir = fileBackupDir; + } + + public Integer getWaitUntilBufferFlushed() { + return waitUntilBufferFlushed; + } + + public void setWaitUntilBufferFlushed(Integer waitUntilBufferFlushed) { + this.waitUntilBufferFlushed = waitUntilBufferFlushed; + } + + public Integer getWaitUntilFlusherTerminated() { + return waitUntilFlusherTerminated; + } + + public void setWaitUntilFlusherTerminated(Integer waitUntilFlusherTerminated) { + this.waitUntilFlusherTerminated = waitUntilFlusherTerminated; + } + + public Boolean getJvmHeapBufferMode() { + return jvmHeapBufferMode; + } + + public void setJvmHeapBufferMode(Boolean jvmHeapBufferMode) { + this.jvmHeapBufferMode = jvmHeapBufferMode; + } + + public ErrorHandler getErrorHandler() { + return errorHandler; + } + + public void setErrorHandler(ErrorHandler errorHandler) { + this.errorHandler = errorHandler; + } + + @Override + public String toString() { + return "BaseFluencyBuilder{" + + "maxBufferSize=" + + maxBufferSize + + ", bufferChunkInitialSize=" + + bufferChunkInitialSize + + ", bufferChunkRetentionSize=" + + bufferChunkRetentionSize + + ", bufferChunkRetentionTimeMillis=" + + bufferChunkRetentionTimeMillis + + ", flushAttemptIntervalMillis=" + + flushAttemptIntervalMillis + + ", fileBackupDir='" + + fileBackupDir + + '\'' + + ", waitUntilBufferFlushed=" + + waitUntilBufferFlushed + + ", waitUntilFlusherTerminated=" + + waitUntilFlusherTerminated + + ", jvmHeapBufferMode=" + + jvmHeapBufferMode + + ", errorHandler=" + + errorHandler + + '}'; + } + + @VisibleForTesting + public Fluency createFluency( + RecordFormatter recordFormatter, + Ingester ingester, + Buffer.Config bufferConfig, + Flusher.Config flusherConfig) { + Buffer buffer = new Buffer(bufferConfig, recordFormatter); + Flusher flusher = new Flusher(flusherConfig, buffer, ingester); + return new Fluency(buffer, flusher); + } + + public Fluency buildFromIngester(RecordFormatter recordFormatter, Ingester ingester) { + Buffer.Config bufferConfig = new Buffer.Config(); + configureBufferConfig(bufferConfig); - public void setFlushAttemptIntervalMillis(Integer flushAttemptIntervalMillis) - { - this.flushAttemptIntervalMillis = flushAttemptIntervalMillis; - } - - public String getFileBackupDir() - { - return fileBackupDir; - } + Flusher.Config flusherConfig = new Flusher.Config(); + configureFlusherConfig(flusherConfig); - public void setFileBackupDir(String fileBackupDir) - { - this.fileBackupDir = fileBackupDir; - } - - public Integer getWaitUntilBufferFlushed() - { - return waitUntilBufferFlushed; - } + return createFluency(recordFormatter, ingester, bufferConfig, flusherConfig); + } - public void setWaitUntilBufferFlushed(Integer waitUntilBufferFlushed) - { - this.waitUntilBufferFlushed = waitUntilBufferFlushed; + private void configureBufferConfig(Buffer.Config bufferConfig) { + if (getMaxBufferSize() != null) { + bufferConfig.setMaxBufferSize(getMaxBufferSize()); } - public Integer getWaitUntilFlusherTerminated() - { - return waitUntilFlusherTerminated; + if (getBufferChunkInitialSize() != null) { + bufferConfig.setChunkInitialSize(getBufferChunkInitialSize()); } - public void setWaitUntilFlusherTerminated(Integer waitUntilFlusherTerminated) - { - this.waitUntilFlusherTerminated = waitUntilFlusherTerminated; + if (getBufferChunkRetentionSize() != null) { + bufferConfig.setChunkRetentionSize(getBufferChunkRetentionSize()); } - public Boolean getJvmHeapBufferMode() - { - return jvmHeapBufferMode; + if (getBufferChunkRetentionTimeMillis() != null) { + bufferConfig.setChunkRetentionTimeMillis(getBufferChunkRetentionTimeMillis()); } - public void setJvmHeapBufferMode(Boolean jvmHeapBufferMode) - { - this.jvmHeapBufferMode = jvmHeapBufferMode; + if (getFileBackupDir() != null) { + bufferConfig.setFileBackupDir(getFileBackupDir()); } - public ErrorHandler getErrorHandler() - { - return errorHandler; + if (getJvmHeapBufferMode() != null) { + bufferConfig.setJvmHeapBufferMode(getJvmHeapBufferMode()); } + } - public void setErrorHandler(ErrorHandler errorHandler) - { - this.errorHandler = errorHandler; + protected void configureFlusherConfig(Flusher.Config flusherConfig) { + if (getFlushAttemptIntervalMillis() != null) { + flusherConfig.setFlushAttemptIntervalMillis(getFlushAttemptIntervalMillis()); } - @Override - public String toString() - { - return "BaseFluencyBuilder{" + - "maxBufferSize=" + maxBufferSize + - ", bufferChunkInitialSize=" + bufferChunkInitialSize + - ", bufferChunkRetentionSize=" + bufferChunkRetentionSize + - ", bufferChunkRetentionTimeMillis=" + bufferChunkRetentionTimeMillis + - ", flushAttemptIntervalMillis=" + flushAttemptIntervalMillis + - ", fileBackupDir='" + fileBackupDir + '\'' + - ", waitUntilBufferFlushed=" + waitUntilBufferFlushed + - ", waitUntilFlusherTerminated=" + waitUntilFlusherTerminated + - ", jvmHeapBufferMode=" + jvmHeapBufferMode + - ", errorHandler=" + errorHandler + - '}'; + if (getWaitUntilBufferFlushed() != null) { + flusherConfig.setWaitUntilBufferFlushed(getWaitUntilBufferFlushed()); } - @VisibleForTesting - public Fluency createFluency( - RecordFormatter recordFormatter, - Ingester ingester, - Buffer.Config bufferConfig, - Flusher.Config flusherConfig) - { - Buffer buffer = new Buffer(bufferConfig, recordFormatter); - Flusher flusher = new Flusher(flusherConfig, buffer, ingester); - return new Fluency(buffer, flusher); - } - - public Fluency buildFromIngester(RecordFormatter recordFormatter, Ingester ingester) - { - Buffer.Config bufferConfig = new Buffer.Config(); - configureBufferConfig(bufferConfig); - - Flusher.Config flusherConfig = new Flusher.Config(); - configureFlusherConfig(flusherConfig); - - return createFluency(recordFormatter, ingester, bufferConfig, flusherConfig); - } - - private void configureBufferConfig(Buffer.Config bufferConfig) - { - if (getMaxBufferSize() != null) { - bufferConfig.setMaxBufferSize(getMaxBufferSize()); - } - - if (getBufferChunkInitialSize() != null) { - bufferConfig.setChunkInitialSize(getBufferChunkInitialSize()); - } - - if (getBufferChunkRetentionSize() != null) { - bufferConfig.setChunkRetentionSize(getBufferChunkRetentionSize()); - } - - if (getBufferChunkRetentionTimeMillis() != null) { - bufferConfig.setChunkRetentionTimeMillis(getBufferChunkRetentionTimeMillis()); - } - - if (getFileBackupDir() != null) { - bufferConfig.setFileBackupDir(getFileBackupDir()); - } - - if (getJvmHeapBufferMode() != null) { - bufferConfig.setJvmHeapBufferMode(getJvmHeapBufferMode()); - } - } - - protected void configureFlusherConfig(Flusher.Config flusherConfig) - { - if (getFlushAttemptIntervalMillis() != null) { - flusherConfig.setFlushAttemptIntervalMillis(getFlushAttemptIntervalMillis()); - } - - if (getWaitUntilBufferFlushed() != null) { - flusherConfig.setWaitUntilBufferFlushed(getWaitUntilBufferFlushed()); - } - - if (getWaitUntilFlusherTerminated() != null) { - flusherConfig.setWaitUntilTerminated(getWaitUntilFlusherTerminated()); - } + if (getWaitUntilFlusherTerminated() != null) { + flusherConfig.setWaitUntilTerminated(getWaitUntilFlusherTerminated()); } + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/NonRetryableException.java b/fluency-core/src/main/java/org/komamitsu/fluency/NonRetryableException.java index 39d6b27c..03c776a4 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/NonRetryableException.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/NonRetryableException.java @@ -16,16 +16,12 @@ package org.komamitsu.fluency; -public class NonRetryableException - extends RuntimeException -{ - public NonRetryableException(String message) - { - super(message); - } +public class NonRetryableException extends RuntimeException { + public NonRetryableException(String message) { + super(message); + } - public NonRetryableException(String message, Throwable cause) - { - super(message, cause); - } + public NonRetryableException(String message, Throwable cause) { + super(message, cause); + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/RetryableException.java b/fluency-core/src/main/java/org/komamitsu/fluency/RetryableException.java index 7616bd3b..31c9d854 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/RetryableException.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/RetryableException.java @@ -16,16 +16,12 @@ package org.komamitsu.fluency; -public class RetryableException - extends RuntimeException -{ - public RetryableException(String message) - { - super(message); - } +public class RetryableException extends RuntimeException { + public RetryableException(String message) { + super(message); + } - public RetryableException(String message, Throwable cause) - { - super(message, cause); - } + public RetryableException(String message, Throwable cause) { + super(message, cause); + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/buffer/Buffer.java b/fluency-core/src/main/java/org/komamitsu/fluency/buffer/Buffer.java index b0335f67..b5a55ce7 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/buffer/Buffer.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/buffer/Buffer.java @@ -16,16 +16,6 @@ package org.komamitsu.fluency.buffer; -import org.komamitsu.fluency.BufferFullException; -import org.komamitsu.fluency.EventTime; -import org.komamitsu.fluency.ingester.Ingester; -import org.komamitsu.fluency.recordformat.RecordFormatter; -import org.komamitsu.fluency.validation.Validatable; -import org.komamitsu.fluency.validation.annotation.DecimalMin; -import org.komamitsu.fluency.validation.annotation.Min; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.Closeable; import java.io.File; import java.io.IOException; @@ -40,611 +30,562 @@ import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicLong; +import org.komamitsu.fluency.BufferFullException; +import org.komamitsu.fluency.EventTime; +import org.komamitsu.fluency.ingester.Ingester; +import org.komamitsu.fluency.recordformat.RecordFormatter; +import org.komamitsu.fluency.validation.Validatable; +import org.komamitsu.fluency.validation.annotation.DecimalMin; +import org.komamitsu.fluency.validation.annotation.Min; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class Buffer - implements Closeable -{ - private static final Logger LOG = LoggerFactory.getLogger(Buffer.class); - private final FileBackup fileBackup; - private final RecordFormatter recordFormatter; - private final Config config; - - private final Map retentionBuffers = new HashMap<>(); - private final LinkedBlockingQueue flushableBuffers = new LinkedBlockingQueue<>(); - private final Queue backupBuffers = new ConcurrentLinkedQueue<>(); - private final BufferPool bufferPool; - - public Buffer(RecordFormatter recordFormatter) - { - this(new Config(), recordFormatter); - } - - public Buffer(final Config config, RecordFormatter recordFormatter) - { - config.validateValues(); - this.config = config; - - if (config.getFileBackupDir() != null) { - fileBackup = new FileBackup(new File(config.getFileBackupDir()), this, config.getFileBackupPrefix()); - } - else { - fileBackup = null; - } - - this.recordFormatter = recordFormatter; - - bufferPool = new BufferPool( - config.getChunkInitialSize(), config.getMaxBufferSize(), config.jvmHeapBufferMode); - - init(); - } - - private void init() - { - if (fileBackup != null) { - for (FileBackup.SavedBuffer savedBuffer : fileBackup.getSavedFiles()) { - savedBuffer.open((params, channel) -> { - try { - LOG.info("Loading buffer: params={}, buffer.size={}", params, channel.size()); - } - catch (IOException e) { - LOG.error("Failed to access the backup file: params={}", params, e); - } - loadBufferFromFile(params, channel); - }); - } - } - } +public class Buffer implements Closeable { + private static final Logger LOG = LoggerFactory.getLogger(Buffer.class); + private final FileBackup fileBackup; + private final RecordFormatter recordFormatter; + private final Config config; + + private final Map retentionBuffers = new HashMap<>(); + private final LinkedBlockingQueue flushableBuffers = new LinkedBlockingQueue<>(); + private final Queue backupBuffers = new ConcurrentLinkedQueue<>(); + private final BufferPool bufferPool; + + public Buffer(RecordFormatter recordFormatter) { + this(new Config(), recordFormatter); + } + + public Buffer(final Config config, RecordFormatter recordFormatter) { + config.validateValues(); + this.config = config; + + if (config.getFileBackupDir() != null) { + fileBackup = + new FileBackup(new File(config.getFileBackupDir()), this, config.getFileBackupPrefix()); + } else { + fileBackup = null; + } + + this.recordFormatter = recordFormatter; + + bufferPool = + new BufferPool( + config.getChunkInitialSize(), config.getMaxBufferSize(), config.jvmHeapBufferMode); + + init(); + } + + private void init() { + if (fileBackup != null) { + for (FileBackup.SavedBuffer savedBuffer : fileBackup.getSavedFiles()) { + savedBuffer.open( + (params, channel) -> { + try { + LOG.info("Loading buffer: params={}, buffer.size={}", params, channel.size()); + } catch (IOException e) { + LOG.error("Failed to access the backup file: params={}", params, e); + } + loadBufferFromFile(params, channel); + }); + } + } + } + + protected void saveBuffer(List params, ByteBuffer buffer) { + if (fileBackup == null) { + return; + } + LOG.info("Saving buffer: params={}, buffer={}", params, buffer); + fileBackup.saveBuffer(params, buffer); + } + + public void flush(Ingester ingester, boolean force) throws IOException { + LOG.trace("flush(): force={}, bufferUsage={}", force, getBufferUsage()); + flushInternal(ingester, force); + } + + @Override + public void close() { + try { + LOG.debug("Saving all buffers if needed"); + saveAllBuffersToFile(); + } catch (Exception e) { + LOG.warn("Failed to save all buffers", e); + } + LOG.debug("Closing buffers"); + closeInternal(); + } + + private long getMaxSize() { + return config.getMaxBufferSize(); + } + + public float getBufferUsage() { + return (float) getAllocatedSize() / getMaxSize(); + } - protected void saveBuffer(List params, ByteBuffer buffer) - { - if (fileBackup == null) { - return; - } - LOG.info("Saving buffer: params={}, buffer={}", params, buffer); - fileBackup.saveBuffer(params, buffer); + public void clearBackupFiles() { + if (fileBackup != null) { + for (FileBackup.SavedBuffer buffer : fileBackup.getSavedFiles()) { + buffer.remove(); + } } - - public void flush(Ingester ingester, boolean force) - throws IOException - { - LOG.trace("flush(): force={}, bufferUsage={}", force, getBufferUsage()); - flushInternal(ingester, force); + } + + public long getMaxBufferSize() { + return config.getMaxBufferSize(); + } + + public String getFileBackupPrefix() { + return config.getFileBackupPrefix(); + } + + public String getFileBackupDir() { + return config.getFileBackupDir(); + } + + private RetentionBuffer prepareBuffer(String tag, int writeSize) throws BufferFullException { + RetentionBuffer retentionBuffer = retentionBuffers.get(tag); + if (retentionBuffer != null && retentionBuffer.getByteBuffer().remaining() > writeSize) { + return retentionBuffer; + } + + int existingDataSize = 0; + int newBufferChunkRetentionSize; + if (retentionBuffer == null) { + newBufferChunkRetentionSize = config.getChunkInitialSize(); + } else { + existingDataSize = retentionBuffer.getByteBuffer().position(); + newBufferChunkRetentionSize = + (int) (retentionBuffer.getByteBuffer().capacity() * config.getChunkExpandRatio()); + } + + while (newBufferChunkRetentionSize < (writeSize + existingDataSize)) { + newBufferChunkRetentionSize *= config.getChunkExpandRatio(); + } + + ByteBuffer acquiredBuffer = bufferPool.acquireBuffer(newBufferChunkRetentionSize); + if (acquiredBuffer == null) { + throw new BufferFullException( + "Buffer is full. config=" + config + ", bufferPool=" + bufferPool); + } + + RetentionBuffer newBuffer = new RetentionBuffer(acquiredBuffer, System.currentTimeMillis()); + if (retentionBuffer != null) { + ByteBuffer buf = retentionBuffer.getByteBuffer().duplicate(); + buf.flip(); + newBuffer.getByteBuffer().put(buf); + bufferPool.returnBuffer(retentionBuffer.getByteBuffer()); + } + LOG.trace("prepareBuffer(): allocate a new buffer. tag={}, buffer={}", tag, newBuffer); + + retentionBuffers.put(tag, newBuffer); + return newBuffer; + } + + private void loadDataToRetentionBuffers(String tag, ByteBuffer src) throws IOException { + synchronized (retentionBuffers) { + RetentionBuffer retentionBuffer = retentionBuffers.get(tag); + if (retentionBuffer != null) { + moveRetentionBufferIfNeeded(tag, retentionBuffer, src.remaining()); + } + RetentionBuffer buffer = prepareBuffer(tag, src.remaining()); + buffer.getByteBuffer().put(src); + } + } + + protected void loadBufferFromFile(List params, FileChannel channel) { + if (params.size() != 1) { + throw new IllegalArgumentException("The number of params should be 1: params=" + params); + } + String tag = params.get(0); + + try { + MappedByteBuffer src = channel.map(FileChannel.MapMode.PRIVATE, 0, channel.size()); + loadDataToRetentionBuffers(tag, src); + } catch (Exception e) { + LOG.error("Failed to load data to flushableBuffers: params={}, channel={}", params, channel); + } + } + + private void saveBuffer(TaggableBuffer buffer) { + saveBuffer(Collections.singletonList(buffer.getTag()), buffer.getByteBuffer()); + } + + protected void saveAllBuffersToFile() throws IOException { + moveRetentionBuffersToFlushable(true); // Just in case + + TaggableBuffer flushableBuffer; + while ((flushableBuffer = flushableBuffers.poll()) != null) { + saveBuffer(flushableBuffer); + } + while ((flushableBuffer = backupBuffers.poll()) != null) { + saveBuffer(flushableBuffer); + } + } + + private void appendMapInternal(String tag, Object timestamp, Map data) + throws IOException { + loadDataToRetentionBuffers(tag, ByteBuffer.wrap(recordFormatter.format(tag, timestamp, data))); + } + + private void appendMessagePackMapValueInternal( + String tag, Object timestamp, byte[] mapValue, int offset, int len) throws IOException { + loadDataToRetentionBuffers( + tag, + ByteBuffer.wrap( + recordFormatter.formatFromMessagePack(tag, timestamp, mapValue, offset, len))); + } + + private void appendMessagePackMapValueInternal(String tag, Object timestamp, ByteBuffer mapValue) + throws IOException { + loadDataToRetentionBuffers( + tag, ByteBuffer.wrap(recordFormatter.formatFromMessagePack(tag, timestamp, mapValue))); + } + + public void append(String tag, long timestamp, Map data) throws IOException { + appendMapInternal(tag, timestamp, data); + } + + public void append(String tag, EventTime timestamp, Map data) throws IOException { + appendMapInternal(tag, timestamp, data); + } + + public void appendMessagePackMapValue( + String tag, long timestamp, byte[] mapValue, int offset, int len) throws IOException { + appendMessagePackMapValueInternal(tag, timestamp, mapValue, offset, len); + } + + public void appendMessagePackMapValue( + String tag, EventTime timestamp, byte[] mapValue, int offset, int len) throws IOException { + appendMessagePackMapValueInternal(tag, timestamp, mapValue, offset, len); + } + + public void appendMessagePackMapValue(String tag, long timestamp, ByteBuffer mapValue) + throws IOException { + appendMessagePackMapValueInternal(tag, timestamp, mapValue); + } + + public void appendMessagePackMapValue(String tag, EventTime timestamp, ByteBuffer mapValue) + throws IOException { + appendMessagePackMapValueInternal(tag, timestamp, mapValue); + } + + private void moveRetentionBufferIfNeeded(String tag, RetentionBuffer buffer, int additionalSize) + throws IOException { + if (buffer.getByteBuffer().position() + additionalSize > config.getChunkRetentionSize()) { + moveRetentionBufferToFlushable(tag, buffer); + } + } + + private void moveRetentionBuffersToFlushable(boolean force) throws IOException { + long expiredThreshold = System.currentTimeMillis() - config.getChunkRetentionTimeMillis(); + + synchronized (retentionBuffers) { + for (Map.Entry entry : retentionBuffers.entrySet()) { + // it can be null because moveRetentionBufferToFlushable() can set null + if (entry.getValue() != null) { + if (force || entry.getValue().getCreatedTimeMillis() < expiredThreshold) { + moveRetentionBufferToFlushable(entry.getKey(), entry.getValue()); + } + } + } + } + } + + private void moveRetentionBufferToFlushable(String tag, RetentionBuffer buffer) + throws IOException { + try { + LOG.trace("moveRetentionBufferToFlushable(): tag={}, buffer={}", tag, buffer); + ByteBuffer buf = buffer.getByteBuffer().duplicate(); + buf.flip(); + flushableBuffers.put(new TaggableBuffer(tag, buf)); + retentionBuffers.put(tag, null); + } catch (InterruptedException e) { + throw new IOException("Failed to move retention buffer due to interruption", e); + } + } + + // TODO: Can't Buffer hold `intester` as an instance variable? + protected void flushInternal(Ingester ingester, boolean force) throws IOException { + moveRetentionBuffersToFlushable(force); + + TaggableBuffer flushableBuffer; + while (!Thread.currentThread().isInterrupted() + && (flushableBuffer = flushableBuffers.poll()) != null) { + boolean keepBuffer = false; + try { + LOG.trace( + "flushInternal(): bufferUsage={}, flushableBuffer={}", + getBufferUsage(), + flushableBuffer); + String tag = flushableBuffer.getTag(); + ByteBuffer dataBuffer = flushableBuffer.getByteBuffer(); + ingester.ingest(tag, dataBuffer); + } catch (IOException e) { + LOG.warn( + "Failed to send data. The data is going to be saved into the buffer again: data={}", + flushableBuffer); + keepBuffer = true; + throw e; + } finally { + if (keepBuffer) { + try { + flushableBuffers.put(flushableBuffer); + } catch (InterruptedException e1) { + LOG.warn( + "Failed to save the data into the buffer. Trying to save it in extra buffer: chunk={}", + flushableBuffer); + Thread.currentThread().interrupt(); + backupBuffers.add(flushableBuffer); + } + } else { + bufferPool.returnBuffer(flushableBuffer.getByteBuffer()); + } + } + } + } + + protected synchronized void closeInternal() { + retentionBuffers.clear(); + bufferPool.releaseBuffers(); + } + + public long getAllocatedSize() { + return bufferPool.getAllocatedSize(); + } + + public long getBufferedDataSize() { + long size = 0; + synchronized (retentionBuffers) { + for (Map.Entry buffer : retentionBuffers.entrySet()) { + if (buffer.getValue() != null && buffer.getValue().getByteBuffer() != null) { + size += buffer.getValue().getByteBuffer().position(); + } + } + } + for (TaggableBuffer buffer : flushableBuffers) { + if (buffer.getByteBuffer() != null) { + size += buffer.getByteBuffer().remaining(); + } + } + return size; + } + + public boolean getJvmHeapBufferMode() { + return bufferPool.getJvmHeapBufferMode(); + } + + public String bufferFormatType() { + // To keep backward compatibility + return "packed_forward"; + } + + public int getChunkInitialSize() { + return config.getChunkInitialSize(); + } + + public float getChunkExpandRatio() { + return config.getChunkExpandRatio(); + } + + public int getChunkRetentionSize() { + return config.getChunkRetentionSize(); + } + + public int getChunkRetentionTimeMillis() { + return config.getChunkRetentionTimeMillis(); + } + + @Override + public String toString() { + return "PackedForwardBuffer{" + + "retentionBuffers=" + + retentionBuffers + + ", flushableBuffers=" + + flushableBuffers + + ", backupBuffers=" + + backupBuffers + + ", bufferPool=" + + bufferPool + + ", config=" + + config + + "} " + + super.toString(); + } + + private static class RetentionBuffer { + private final AtomicLong createdTimeMillis; + private final ByteBuffer byteBuffer; + + RetentionBuffer(ByteBuffer byteBuffer, long currentTimeMillis) { + this.byteBuffer = byteBuffer; + this.createdTimeMillis = new AtomicLong(currentTimeMillis); + } + + long getCreatedTimeMillis() { + return createdTimeMillis.get(); + } + + ByteBuffer getByteBuffer() { + return byteBuffer; } @Override - public void close() - { - try { - LOG.debug("Saving all buffers if needed"); - saveAllBuffersToFile(); - } - catch (Exception e) { - LOG.warn("Failed to save all buffers", e); - } - LOG.debug("Closing buffers"); - closeInternal(); + public String toString() { + return "RetentionBuffer{" + + "createdTimeMillis=" + + createdTimeMillis + + ", byteBuffer=" + + byteBuffer + + '}'; } + } - private long getMaxSize() - { - return config.getMaxBufferSize(); - } + private static class TaggableBuffer { + private final String tag; + private final ByteBuffer byteBuffer; - public float getBufferUsage() - { - return (float) getAllocatedSize() / getMaxSize(); + public TaggableBuffer(String tag, ByteBuffer byteBuffer) { + this.tag = tag; + this.byteBuffer = byteBuffer; } - public void clearBackupFiles() - { - if (fileBackup != null) { - for (FileBackup.SavedBuffer buffer : fileBackup.getSavedFiles()) { - buffer.remove(); - } - } - } - - public long getMaxBufferSize() - { - return config.getMaxBufferSize(); + public String getTag() { + return tag; } - public String getFileBackupPrefix() - { - return config.getFileBackupPrefix(); + public ByteBuffer getByteBuffer() { + return byteBuffer; } - public String getFileBackupDir() - { - return config.getFileBackupDir(); - } - - private RetentionBuffer prepareBuffer(String tag, int writeSize) - throws BufferFullException - { - RetentionBuffer retentionBuffer = retentionBuffers.get(tag); - if (retentionBuffer != null && retentionBuffer.getByteBuffer().remaining() > writeSize) { - return retentionBuffer; - } - - int existingDataSize = 0; - int newBufferChunkRetentionSize; - if (retentionBuffer == null) { - newBufferChunkRetentionSize = config.getChunkInitialSize(); - } - else { - existingDataSize = retentionBuffer.getByteBuffer().position(); - newBufferChunkRetentionSize = (int) (retentionBuffer.getByteBuffer().capacity() * config.getChunkExpandRatio()); - } - - while (newBufferChunkRetentionSize < (writeSize + existingDataSize)) { - newBufferChunkRetentionSize *= config.getChunkExpandRatio(); - } - - ByteBuffer acquiredBuffer = bufferPool.acquireBuffer(newBufferChunkRetentionSize); - if (acquiredBuffer == null) { - throw new BufferFullException("Buffer is full. config=" + config + ", bufferPool=" + bufferPool); - } - - RetentionBuffer newBuffer = new RetentionBuffer(acquiredBuffer, System.currentTimeMillis()); - if (retentionBuffer != null) { - ByteBuffer buf = retentionBuffer.getByteBuffer().duplicate(); - buf.flip(); - newBuffer.getByteBuffer().put(buf); - bufferPool.returnBuffer(retentionBuffer.getByteBuffer()); - } - LOG.trace("prepareBuffer(): allocate a new buffer. tag={}, buffer={}", tag, newBuffer); - - retentionBuffers.put(tag, newBuffer); - return newBuffer; - } - - private void loadDataToRetentionBuffers(String tag, ByteBuffer src) - throws IOException - { - synchronized (retentionBuffers) { - RetentionBuffer retentionBuffer = retentionBuffers.get(tag); - if (retentionBuffer != null) { - moveRetentionBufferIfNeeded(tag, retentionBuffer, src.remaining()); - } - RetentionBuffer buffer = prepareBuffer(tag, src.remaining()); - buffer.getByteBuffer().put(src); - } - } - - protected void loadBufferFromFile(List params, FileChannel channel) - { - if (params.size() != 1) { - throw new IllegalArgumentException("The number of params should be 1: params=" + params); - } - String tag = params.get(0); - - try { - MappedByteBuffer src = channel.map(FileChannel.MapMode.PRIVATE, 0, channel.size()); - loadDataToRetentionBuffers(tag, src); - } - catch (Exception e) { - LOG.error("Failed to load data to flushableBuffers: params={}, channel={}", params, channel); - } + @Override + public String toString() { + return "TaggableBuffer{" + "tag='" + tag + '\'' + ", byteBuffer=" + byteBuffer + '}'; } + } - private void saveBuffer(TaggableBuffer buffer) - { - saveBuffer(Collections.singletonList(buffer.getTag()), buffer.getByteBuffer()); - } + public static class Config implements Validatable { + private long maxBufferSize = 512 * 1024 * 1024; + private String fileBackupDir; + private String fileBackupPrefix; // Mainly for testing - protected void saveAllBuffersToFile() - throws IOException - { - moveRetentionBuffersToFlushable(true); // Just in case + private int chunkInitialSize = 1024 * 1024; - TaggableBuffer flushableBuffer; - while ((flushableBuffer = flushableBuffers.poll()) != null) { - saveBuffer(flushableBuffer); - } - while ((flushableBuffer = backupBuffers.poll()) != null) { - saveBuffer(flushableBuffer); - } - } + @DecimalMin("1.2") + private float chunkExpandRatio = 2.0f; - private void appendMapInternal(String tag, Object timestamp, Map data) - throws IOException - { - loadDataToRetentionBuffers(tag, - ByteBuffer.wrap(recordFormatter.format(tag, timestamp, data))); - } + private int chunkRetentionSize = 4 * 1024 * 1024; - private void appendMessagePackMapValueInternal(String tag, Object timestamp, byte[] mapValue, int offset, int len) - throws IOException - { - loadDataToRetentionBuffers(tag, - ByteBuffer.wrap(recordFormatter.formatFromMessagePack(tag, timestamp, mapValue, offset, len))); - } + @Min(50) + private int chunkRetentionTimeMillis = 1000; - private void appendMessagePackMapValueInternal(String tag, Object timestamp, ByteBuffer mapValue) - throws IOException - { - loadDataToRetentionBuffers(tag, - ByteBuffer.wrap(recordFormatter.formatFromMessagePack(tag, timestamp, mapValue))); - } + private boolean jvmHeapBufferMode = false; - public void append(String tag, long timestamp, Map data) - throws IOException - { - appendMapInternal(tag, timestamp, data); + public long getMaxBufferSize() { + return maxBufferSize; } - public void append(String tag, EventTime timestamp, Map data) - throws IOException - { - appendMapInternal(tag, timestamp, data); + public void setMaxBufferSize(long maxBufferSize) { + this.maxBufferSize = maxBufferSize; } - public void appendMessagePackMapValue(String tag, long timestamp, byte[] mapValue, int offset, int len) - throws IOException - { - appendMessagePackMapValueInternal(tag, timestamp, mapValue, offset, len); + public String getFileBackupDir() { + return fileBackupDir; } - public void appendMessagePackMapValue(String tag, EventTime timestamp, byte[] mapValue, int offset, int len) - throws IOException - { - appendMessagePackMapValueInternal(tag, timestamp, mapValue, offset, len); + public void setFileBackupDir(String fileBackupDir) { + this.fileBackupDir = fileBackupDir; } - public void appendMessagePackMapValue(String tag, long timestamp, ByteBuffer mapValue) - throws IOException - { - appendMessagePackMapValueInternal(tag, timestamp, mapValue); + public String getFileBackupPrefix() { + return fileBackupPrefix; } - public void appendMessagePackMapValue(String tag, EventTime timestamp, ByteBuffer mapValue) - throws IOException - { - appendMessagePackMapValueInternal(tag, timestamp, mapValue); + public void setFileBackupPrefix(String fileBackupPrefix) { + this.fileBackupPrefix = fileBackupPrefix; } - private void moveRetentionBufferIfNeeded(String tag, RetentionBuffer buffer, int additionalSize) - throws IOException - { - if (buffer.getByteBuffer().position() + additionalSize > config.getChunkRetentionSize()) { - moveRetentionBufferToFlushable(tag, buffer); - } + public int getChunkInitialSize() { + return chunkInitialSize; } - private void moveRetentionBuffersToFlushable(boolean force) - throws IOException - { - long expiredThreshold = System.currentTimeMillis() - config.getChunkRetentionTimeMillis(); - - synchronized (retentionBuffers) { - for (Map.Entry entry : retentionBuffers.entrySet()) { - // it can be null because moveRetentionBufferToFlushable() can set null - if (entry.getValue() != null) { - if (force || entry.getValue().getCreatedTimeMillis() < expiredThreshold) { - moveRetentionBufferToFlushable(entry.getKey(), entry.getValue()); - } - } - } - } + public void setChunkInitialSize(int chunkInitialSize) { + this.chunkInitialSize = chunkInitialSize; } - private void moveRetentionBufferToFlushable(String tag, RetentionBuffer buffer) - throws IOException - { - try { - LOG.trace("moveRetentionBufferToFlushable(): tag={}, buffer={}", tag, buffer); - ByteBuffer buf = buffer.getByteBuffer().duplicate(); - buf.flip(); - flushableBuffers.put(new TaggableBuffer(tag, buf)); - retentionBuffers.put(tag, null); - } - catch (InterruptedException e) { - throw new IOException("Failed to move retention buffer due to interruption", e); - } + public float getChunkExpandRatio() { + return chunkExpandRatio; } - // TODO: Can't Buffer hold `intester` as an instance variable? - protected void flushInternal(Ingester ingester, boolean force) - throws IOException - { - moveRetentionBuffersToFlushable(force); - - TaggableBuffer flushableBuffer; - while (!Thread.currentThread().isInterrupted() && - (flushableBuffer = flushableBuffers.poll()) != null) { - boolean keepBuffer = false; - try { - LOG.trace("flushInternal(): bufferUsage={}, flushableBuffer={}", getBufferUsage(), flushableBuffer); - String tag = flushableBuffer.getTag(); - ByteBuffer dataBuffer = flushableBuffer.getByteBuffer(); - ingester.ingest(tag, dataBuffer); - } - catch (IOException e) { - LOG.warn("Failed to send data. The data is going to be saved into the buffer again: data={}", flushableBuffer); - keepBuffer = true; - throw e; - } - finally { - if (keepBuffer) { - try { - flushableBuffers.put(flushableBuffer); - } - catch (InterruptedException e1) { - LOG.warn("Failed to save the data into the buffer. Trying to save it in extra buffer: chunk={}", flushableBuffer); - Thread.currentThread().interrupt(); - backupBuffers.add(flushableBuffer); - } - } - else { - bufferPool.returnBuffer(flushableBuffer.getByteBuffer()); - } - } - } + public void setChunkExpandRatio(float chunkExpandRatio) { + this.chunkExpandRatio = chunkExpandRatio; } - protected synchronized void closeInternal() - { - retentionBuffers.clear(); - bufferPool.releaseBuffers(); + public int getChunkRetentionSize() { + return chunkRetentionSize; } - public long getAllocatedSize() - { - return bufferPool.getAllocatedSize(); + public void setChunkRetentionSize(int chunkRetentionSize) { + this.chunkRetentionSize = chunkRetentionSize; } - public long getBufferedDataSize() - { - long size = 0; - synchronized (retentionBuffers) { - for (Map.Entry buffer : retentionBuffers.entrySet()) { - if (buffer.getValue() != null && buffer.getValue().getByteBuffer() != null) { - size += buffer.getValue().getByteBuffer().position(); - } - } - } - for (TaggableBuffer buffer : flushableBuffers) { - if (buffer.getByteBuffer() != null) { - size += buffer.getByteBuffer().remaining(); - } - } - return size; + public int getChunkRetentionTimeMillis() { + return chunkRetentionTimeMillis; } - public boolean getJvmHeapBufferMode() - { - return bufferPool.getJvmHeapBufferMode(); + public void setChunkRetentionTimeMillis(int chunkRetentionTimeMillis) { + this.chunkRetentionTimeMillis = chunkRetentionTimeMillis; } - public String bufferFormatType() - { - // To keep backward compatibility - return "packed_forward"; + public boolean getJvmHeapBufferMode() { + return jvmHeapBufferMode; } - public int getChunkInitialSize() - { - return config.getChunkInitialSize(); + public void setJvmHeapBufferMode(boolean jvmHeapBufferMode) { + this.jvmHeapBufferMode = jvmHeapBufferMode; } - public float getChunkExpandRatio() - { - return config.getChunkExpandRatio(); - } + void validateValues() { + validate(); - public int getChunkRetentionSize() - { - return config.getChunkRetentionSize(); - } + if (chunkInitialSize >= chunkRetentionSize) { + throw new IllegalArgumentException( + String.format( + "Buffer Chunk Retention Size (%d) should be more than Initial Buffer Chunk Size (%d)", + chunkRetentionSize, chunkInitialSize)); + } - public int getChunkRetentionTimeMillis() - { - return config.getChunkRetentionTimeMillis(); + if (chunkRetentionSize >= maxBufferSize) { + throw new IllegalArgumentException( + String.format( + "Max Total Buffer Size (%d) should be more than Buffer Chunk Retention Size (%d)", + maxBufferSize, chunkRetentionSize)); + } } @Override - public String toString() - { - return "PackedForwardBuffer{" + - "retentionBuffers=" + retentionBuffers + - ", flushableBuffers=" + flushableBuffers + - ", backupBuffers=" + backupBuffers + - ", bufferPool=" + bufferPool + - ", config=" + config + - "} " + super.toString(); - } - - private static class RetentionBuffer - { - private final AtomicLong createdTimeMillis; - private final ByteBuffer byteBuffer; - - RetentionBuffer(ByteBuffer byteBuffer, long currentTimeMillis) - { - this.byteBuffer = byteBuffer; - this.createdTimeMillis = new AtomicLong(currentTimeMillis); - } - - long getCreatedTimeMillis() - { - return createdTimeMillis.get(); - } - - ByteBuffer getByteBuffer() - { - return byteBuffer; - } - - @Override - public String toString() - { - return "RetentionBuffer{" + - "createdTimeMillis=" + createdTimeMillis + - ", byteBuffer=" + byteBuffer + - '}'; - } - } - - private static class TaggableBuffer - { - private final String tag; - private final ByteBuffer byteBuffer; - - public TaggableBuffer(String tag, ByteBuffer byteBuffer) - { - this.tag = tag; - this.byteBuffer = byteBuffer; - } - - public String getTag() - { - return tag; - } - - public ByteBuffer getByteBuffer() - { - return byteBuffer; - } - - @Override - public String toString() - { - return "TaggableBuffer{" + - "tag='" + tag + '\'' + - ", byteBuffer=" + byteBuffer + - '}'; - } - } - - public static class Config - implements Validatable - { - private long maxBufferSize = 512 * 1024 * 1024; - private String fileBackupDir; - private String fileBackupPrefix; // Mainly for testing - - private int chunkInitialSize = 1024 * 1024; - @DecimalMin("1.2") - private float chunkExpandRatio = 2.0f; - private int chunkRetentionSize = 4 * 1024 * 1024; - @Min(50) - private int chunkRetentionTimeMillis = 1000; - private boolean jvmHeapBufferMode = false; - - public long getMaxBufferSize() - { - return maxBufferSize; - } - - public void setMaxBufferSize(long maxBufferSize) - { - this.maxBufferSize = maxBufferSize; - } - - public String getFileBackupDir() - { - return fileBackupDir; - } - - public void setFileBackupDir(String fileBackupDir) - { - this.fileBackupDir = fileBackupDir; - } - - public String getFileBackupPrefix() - { - return fileBackupPrefix; - } - - public void setFileBackupPrefix(String fileBackupPrefix) - { - this.fileBackupPrefix = fileBackupPrefix; - } - - public int getChunkInitialSize() - { - return chunkInitialSize; - } - - public void setChunkInitialSize(int chunkInitialSize) - { - this.chunkInitialSize = chunkInitialSize; - } - - public float getChunkExpandRatio() - { - return chunkExpandRatio; - } - - public void setChunkExpandRatio(float chunkExpandRatio) - { - this.chunkExpandRatio = chunkExpandRatio; - } - - public int getChunkRetentionSize() - { - return chunkRetentionSize; - } - - public void setChunkRetentionSize(int chunkRetentionSize) - { - this.chunkRetentionSize = chunkRetentionSize; - } - - public int getChunkRetentionTimeMillis() - { - return chunkRetentionTimeMillis; - } - - public void setChunkRetentionTimeMillis(int chunkRetentionTimeMillis) - { - this.chunkRetentionTimeMillis = chunkRetentionTimeMillis; - } - - public boolean getJvmHeapBufferMode() - { - return jvmHeapBufferMode; - } - - public void setJvmHeapBufferMode(boolean jvmHeapBufferMode) - { - this.jvmHeapBufferMode = jvmHeapBufferMode; - } - - void validateValues() - { - validate(); - - if (chunkInitialSize >= chunkRetentionSize) { - throw new IllegalArgumentException( - String.format( - "Buffer Chunk Retention Size (%d) should be more than Initial Buffer Chunk Size (%d)", - chunkRetentionSize, chunkInitialSize)); - } - - if (chunkRetentionSize >= maxBufferSize) { - throw new IllegalArgumentException( - String.format( - "Max Total Buffer Size (%d) should be more than Buffer Chunk Retention Size (%d)", - maxBufferSize, chunkRetentionSize)); - } - } - - @Override - public String toString() - { - return "Config{" + - "maxBufferSize=" + maxBufferSize + - ", fileBackupDir='" + fileBackupDir + '\'' + - ", fileBackupPrefix='" + fileBackupPrefix + '\'' + - ", chunkInitialSize=" + chunkInitialSize + - ", chunkExpandRatio=" + chunkExpandRatio + - ", chunkRetentionSize=" + chunkRetentionSize + - ", chunkRetentionTimeMillis=" + chunkRetentionTimeMillis + - ", jvmHeapBufferMode=" + jvmHeapBufferMode + - '}'; - } - } + public String toString() { + return "Config{" + + "maxBufferSize=" + + maxBufferSize + + ", fileBackupDir='" + + fileBackupDir + + '\'' + + ", fileBackupPrefix='" + + fileBackupPrefix + + '\'' + + ", chunkInitialSize=" + + chunkInitialSize + + ", chunkExpandRatio=" + + chunkExpandRatio + + ", chunkRetentionSize=" + + chunkRetentionSize + + ", chunkRetentionTimeMillis=" + + chunkRetentionTimeMillis + + ", jvmHeapBufferMode=" + + jvmHeapBufferMode + + '}'; + } + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/buffer/BufferPool.java b/fluency-core/src/main/java/org/komamitsu/fluency/buffer/BufferPool.java index 5e1d2c2a..187f142d 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/buffer/BufferPool.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/buffer/BufferPool.java @@ -16,129 +16,125 @@ package org.komamitsu.fluency.buffer; -import org.msgpack.core.annotations.VisibleForTesting; - import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicLong; +import org.msgpack.core.annotations.VisibleForTesting; -class BufferPool -{ - @VisibleForTesting final Map> bufferPool = new HashMap<>(); - private final AtomicLong allocatedSize = new AtomicLong(); - private final int initialBufferSize; - private final long maxBufferSize; - private final boolean jvmHeapBufferMode; - - public BufferPool(int initialBufferSize, long maxBufferSize) - { - this(initialBufferSize, maxBufferSize, false); +class BufferPool { + @VisibleForTesting + final Map> bufferPool = new HashMap<>(); + + private final AtomicLong allocatedSize = new AtomicLong(); + private final int initialBufferSize; + private final long maxBufferSize; + private final boolean jvmHeapBufferMode; + + public BufferPool(int initialBufferSize, long maxBufferSize) { + this(initialBufferSize, maxBufferSize, false); + } + + public BufferPool(int initialBufferSize, long maxBufferSize, boolean jvmHeapBufferMode) { + this.initialBufferSize = initialBufferSize; + this.maxBufferSize = maxBufferSize; + this.jvmHeapBufferMode = jvmHeapBufferMode; + } + + public ByteBuffer acquireBuffer(int bufferSize) { + int normalizedBufferSize = initialBufferSize; + while (normalizedBufferSize < bufferSize) { + normalizedBufferSize *= 2; } - public BufferPool(int initialBufferSize, long maxBufferSize, boolean jvmHeapBufferMode) - { - this.initialBufferSize = initialBufferSize; - this.maxBufferSize = maxBufferSize; - this.jvmHeapBufferMode = jvmHeapBufferMode; + LinkedBlockingQueue buffers; + synchronized (bufferPool) { + buffers = bufferPool.get(normalizedBufferSize); + if (buffers == null) { + buffers = new LinkedBlockingQueue<>(); + bufferPool.put(normalizedBufferSize, buffers); + } } - public ByteBuffer acquireBuffer(int bufferSize) - { - int normalizedBufferSize = initialBufferSize; - while (normalizedBufferSize < bufferSize) { - normalizedBufferSize *= 2; - } - - LinkedBlockingQueue buffers; - synchronized (bufferPool) { - buffers = bufferPool.get(normalizedBufferSize); - if (buffers == null) { - buffers = new LinkedBlockingQueue<>(); - bufferPool.put(normalizedBufferSize, buffers); - } - } - - ByteBuffer buffer = buffers.poll(); - if (buffer != null) { - return buffer; - } - - /* - synchronized (allocatedSize) { - if (allocatedSize.get() + normalizedBufferSize > maxBufferSize) { - return null; // `null` means the buffer is full. - } - allocatedSize.addAndGet(normalizedBufferSize); - return ByteBuffer.allocateDirect(normalizedBufferSize); - } - */ - - while (true) { - long currentAllocatedSize = allocatedSize.get(); - if (currentAllocatedSize + normalizedBufferSize > maxBufferSize) { - releaseBuffers(); - return null; // `null` means the buffer is full. - } - if (currentAllocatedSize == allocatedSize.getAndAdd(normalizedBufferSize)) { - ByteBuffer buf; - if (jvmHeapBufferMode) { - buf = ByteBuffer.allocate(normalizedBufferSize); - } - else { - buf = ByteBuffer.allocateDirect(normalizedBufferSize); - } - return buf; - } - allocatedSize.getAndAdd(-normalizedBufferSize); - } + ByteBuffer buffer = buffers.poll(); + if (buffer != null) { + return buffer; } - public void returnBuffer(ByteBuffer byteBuffer) - { - LinkedBlockingQueue buffers = bufferPool.get(byteBuffer.capacity()); - if (buffers == null) { - throw new IllegalStateException("`buffers` shouldn't be null"); + /* + synchronized (allocatedSize) { + if (allocatedSize.get() + normalizedBufferSize > maxBufferSize) { + return null; // `null` means the buffer is full. } - - byteBuffer.position(0); - byteBuffer.limit(byteBuffer.capacity()); - buffers.offer(byteBuffer); + allocatedSize.addAndGet(normalizedBufferSize); + return ByteBuffer.allocateDirect(normalizedBufferSize); } - - public long getAllocatedSize() - { - return allocatedSize.get(); - } - - public void releaseBuffers() - { - // TODO: Stop releasing all the buffers when having released buffers to some extent - synchronized (bufferPool) { - for (Map.Entry> entry : bufferPool.entrySet()) { - ByteBuffer buffer; - while ((buffer = entry.getValue().poll()) != null) { - allocatedSize.addAndGet(-buffer.capacity()); - } - } + */ + + while (true) { + long currentAllocatedSize = allocatedSize.get(); + if (currentAllocatedSize + normalizedBufferSize > maxBufferSize) { + releaseBuffers(); + return null; // `null` means the buffer is full. + } + if (currentAllocatedSize == allocatedSize.getAndAdd(normalizedBufferSize)) { + ByteBuffer buf; + if (jvmHeapBufferMode) { + buf = ByteBuffer.allocate(normalizedBufferSize); + } else { + buf = ByteBuffer.allocateDirect(normalizedBufferSize); } + return buf; + } + allocatedSize.getAndAdd(-normalizedBufferSize); } + } - public boolean getJvmHeapBufferMode() - { - return jvmHeapBufferMode; + public void returnBuffer(ByteBuffer byteBuffer) { + LinkedBlockingQueue buffers = bufferPool.get(byteBuffer.capacity()); + if (buffers == null) { + throw new IllegalStateException("`buffers` shouldn't be null"); } - @Override - public String toString() - { - return "BufferPool{" + - "bufferPool=" + bufferPool + - ", allocatedSize=" + allocatedSize + - ", initialBufferSize=" + initialBufferSize + - ", maxBufferSize=" + maxBufferSize + - ", jvmHeapBufferMode=" + jvmHeapBufferMode + - '}'; + byteBuffer.position(0); + byteBuffer.limit(byteBuffer.capacity()); + buffers.offer(byteBuffer); + } + + public long getAllocatedSize() { + return allocatedSize.get(); + } + + public void releaseBuffers() { + // TODO: Stop releasing all the buffers when having released buffers to some extent + synchronized (bufferPool) { + for (Map.Entry> entry : bufferPool.entrySet()) { + ByteBuffer buffer; + while ((buffer = entry.getValue().poll()) != null) { + allocatedSize.addAndGet(-buffer.capacity()); + } + } } + } + + public boolean getJvmHeapBufferMode() { + return jvmHeapBufferMode; + } + + @Override + public String toString() { + return "BufferPool{" + + "bufferPool=" + + bufferPool + + ", allocatedSize=" + + allocatedSize + + ", initialBufferSize=" + + initialBufferSize + + ", maxBufferSize=" + + maxBufferSize + + ", jvmHeapBufferMode=" + + jvmHeapBufferMode + + '}'; + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/buffer/FileBackup.java b/fluency-core/src/main/java/org/komamitsu/fluency/buffer/FileBackup.java index 257aca4c..c7b9c730 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/buffer/FileBackup.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/buffer/FileBackup.java @@ -16,9 +16,6 @@ package org.komamitsu.fluency.buffer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.Closeable; import java.io.File; import java.io.FileOutputStream; @@ -33,199 +30,198 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class FileBackup -{ - private static final Logger LOG = LoggerFactory.getLogger(FileBackup.class); - private static final String PARAM_DELIM_IN_FILENAME = "#"; - private static final String EXT_FILENAME = ".buf"; - private final File backupDir; - private final Buffer userBuffer; - private final Pattern pattern; - private final String prefix; - - public FileBackup(File backupDir, Buffer userBuffer, String prefix) - { - if (backupDir.mkdir()) { - LOG.info("Created backupDir: dir={}", backupDir); - } - if (!backupDir.isDirectory() || !backupDir.canRead() || !backupDir.canWrite()) { - throw new IllegalArgumentException("backupDir[" + backupDir + "] needs to be a readable & writable directory"); - } - this.backupDir = backupDir; - this.userBuffer = userBuffer; - this.prefix = prefix; - this.pattern = Pattern.compile(String.format("^%s%s%s([\\w.\\-%s]+)%s$", userBuffer.bufferFormatType(), prefix(), PARAM_DELIM_IN_FILENAME, PARAM_DELIM_IN_FILENAME, EXT_FILENAME)); - LOG.debug(this.toString()); +public class FileBackup { + private static final Logger LOG = LoggerFactory.getLogger(FileBackup.class); + private static final String PARAM_DELIM_IN_FILENAME = "#"; + private static final String EXT_FILENAME = ".buf"; + private final File backupDir; + private final Buffer userBuffer; + private final Pattern pattern; + private final String prefix; + + public FileBackup(File backupDir, Buffer userBuffer, String prefix) { + if (backupDir.mkdir()) { + LOG.info("Created backupDir: dir={}", backupDir); } - - private String prefix() - { - return prefix == null ? "" : "_" + prefix; + if (!backupDir.isDirectory() || !backupDir.canRead() || !backupDir.canWrite()) { + throw new IllegalArgumentException( + "backupDir[" + backupDir + "] needs to be a readable & writable directory"); } - - @Override - public String toString() - { - return "FileBackup{" + - "backupDir=" + backupDir + - ", userBuffer=" + userBuffer + - ", pattern=" + pattern + - ", prefix='" + prefix + '\'' + - '}'; + this.backupDir = backupDir; + this.userBuffer = userBuffer; + this.prefix = prefix; + this.pattern = + Pattern.compile( + String.format( + "^%s%s%s([\\w.\\-%s]+)%s$", + userBuffer.bufferFormatType(), + prefix(), + PARAM_DELIM_IN_FILENAME, + PARAM_DELIM_IN_FILENAME, + EXT_FILENAME)); + LOG.debug(this.toString()); + } + + private String prefix() { + return prefix == null ? "" : "_" + prefix; + } + + @Override + public String toString() { + return "FileBackup{" + + "backupDir=" + + backupDir + + ", userBuffer=" + + userBuffer + + ", pattern=" + + pattern + + ", prefix='" + + prefix + + '\'' + + '}'; + } + + public List getSavedFiles() { + File[] files = backupDir.listFiles(); + if (files == null) { + LOG.warn("Failed to list the backup directory. {}", backupDir); + return new ArrayList<>(); } - public List getSavedFiles() - { - File[] files = backupDir.listFiles(); - if (files == null) { - LOG.warn("Failed to list the backup directory. {}", backupDir); - return new ArrayList<>(); - } - - LOG.debug("Checking backup files. files.length={}", files.length); - List savedBuffers = new ArrayList<>(); - for (File f : files) { - Matcher matcher = pattern.matcher(f.toPath().getFileName().toString()); - if (matcher.find()) { - if (matcher.groupCount() != 1) { - LOG.warn("Invalid backup filename: file={}", f.getName()); - } - else { - String concatParams = matcher.group(1); - String[] params = concatParams.split(PARAM_DELIM_IN_FILENAME); - LinkedList paramList = new LinkedList<>(Arrays.asList(params)); - LOG.debug("Saved buffer params={}", paramList); - paramList.removeLast(); - savedBuffers.add(new SavedBuffer(f, paramList)); - } - } - else { - LOG.trace("Found a file in backup dir, but the file path doesn't match the pattern. file={}", f.getAbsolutePath()); - } + LOG.debug("Checking backup files. files.length={}", files.length); + List savedBuffers = new ArrayList<>(); + for (File f : files) { + Matcher matcher = pattern.matcher(f.toPath().getFileName().toString()); + if (matcher.find()) { + if (matcher.groupCount() != 1) { + LOG.warn("Invalid backup filename: file={}", f.getName()); + } else { + String concatParams = matcher.group(1); + String[] params = concatParams.split(PARAM_DELIM_IN_FILENAME); + LinkedList paramList = new LinkedList<>(Arrays.asList(params)); + LOG.debug("Saved buffer params={}", paramList); + paramList.removeLast(); + savedBuffers.add(new SavedBuffer(f, paramList)); + } + } else { + LOG.trace( + "Found a file in backup dir, but the file path doesn't match the pattern. file={}", + f.getAbsolutePath()); + } + } + return savedBuffers; + } + + public void saveBuffer(List params, ByteBuffer buffer) { + List copiedParams = new ArrayList<>(params); + copiedParams.add(String.valueOf(System.nanoTime())); + + boolean isFirst = true; + StringBuilder sb = new StringBuilder(); + for (String param : copiedParams) { + if (isFirst) { + isFirst = false; + } else { + sb.append(PARAM_DELIM_IN_FILENAME); + } + sb.append(param); + } + String filename = + this.userBuffer.bufferFormatType() + + prefix() + + PARAM_DELIM_IN_FILENAME + + sb.toString() + + EXT_FILENAME; + + File file = new File(backupDir, filename); + LOG.debug("Backing up buffer: path={}, size={}", file.getAbsolutePath(), buffer.remaining()); + + FileChannel channel = null; + try { + channel = new FileOutputStream(file).getChannel(); + channel.write(buffer); + } catch (Exception e) { + LOG.error( + "Failed to save buffer to file: params={}, path={}, buffer={}", + copiedParams, + file.getAbsolutePath(), + buffer, + e); + } finally { + if (channel != null) { + try { + channel.close(); + } catch (IOException e) { + LOG.warn("Failed to close Channel: channel={}", channel); } - return savedBuffers; + } } + } - public void saveBuffer(List params, ByteBuffer buffer) - { - List copiedParams = new ArrayList<>(params); - copiedParams.add(String.valueOf(System.nanoTime())); - - boolean isFirst = true; - StringBuilder sb = new StringBuilder(); - for (String param : copiedParams) { - if (isFirst) { - isFirst = false; - } - else { - sb.append(PARAM_DELIM_IN_FILENAME); - } - sb.append(param); - } - String filename = this.userBuffer.bufferFormatType() + prefix() + PARAM_DELIM_IN_FILENAME + sb.toString() + EXT_FILENAME; + public static class SavedBuffer implements Closeable { + private final List params; + private final File savedFile; + private FileChannel channel; - File file = new File(backupDir, filename); - LOG.debug("Backing up buffer: path={}, size={}", file.getAbsolutePath(), buffer.remaining()); + public SavedBuffer(File savedFile, List params) { + this.savedFile = savedFile; + this.params = params; + } - FileChannel channel = null; + public void open(Callback callback) { + try { + channel = new RandomAccessFile(savedFile, "rw").getChannel(); + callback.process(params, channel); + success(); + } catch (Exception e) { + LOG.error("Failed to process file. Skipping the file: file={}", savedFile, e); + } finally { try { - channel = new FileOutputStream(file).getChannel(); - channel.write(buffer); - } - catch (Exception e) { - LOG.error("Failed to save buffer to file: params={}, path={}, buffer={}", copiedParams, file.getAbsolutePath(), buffer, e); - } - finally { - if (channel != null) { - try { - channel.close(); - } - catch (IOException e) { - LOG.warn("Failed to close Channel: channel={}", channel); - } - } + close(); + } catch (IOException e) { + LOG.warn("Failed to close file: file={}", savedFile, e); } + } } - public static class SavedBuffer - implements Closeable - { - private final List params; - private final File savedFile; - private FileChannel channel; - - public SavedBuffer(File savedFile, List params) - { - this.savedFile = savedFile; - this.params = params; - } - - public void open(Callback callback) - { - try { - channel = new RandomAccessFile(savedFile, "rw").getChannel(); - callback.process(params, channel); - success(); - } - catch (Exception e) { - LOG.error("Failed to process file. Skipping the file: file={}", savedFile, e); - } - finally { - try { - close(); - } - catch (IOException e) { - LOG.warn("Failed to close file: file={}", savedFile, e); - } - } - } - - public void remove() - { - if (!savedFile.delete()) { - LOG.warn("Failed to delete file: file={}", savedFile); - } - } + public void remove() { + if (!savedFile.delete()) { + LOG.warn("Failed to delete file: file={}", savedFile); + } + } - private void success() - { - try { - close(); - } - catch (IOException e) { - LOG.warn("Failed to close file: file={}", savedFile, e); - } - finally { - remove(); - } - } + private void success() { + try { + close(); + } catch (IOException e) { + LOG.warn("Failed to close file: file={}", savedFile, e); + } finally { + remove(); + } + } - @Override - public void close() - throws IOException - { - if (channel != null && channel.isOpen()) { - channel.close(); - channel = null; - } - } + @Override + public void close() throws IOException { + if (channel != null && channel.isOpen()) { + channel.close(); + channel = null; + } + } - @Override - public String toString() { - return "SavedBuffer{" + - "params=" + params + - ", savedFile=" + savedFile + - '}'; - } + @Override + public String toString() { + return "SavedBuffer{" + "params=" + params + ", savedFile=" + savedFile + '}'; + } - public Path getPath() { - return savedFile.toPath(); - } + public Path getPath() { + return savedFile.toPath(); + } - public interface Callback - { - void process(List params, FileChannel channel); - } + public interface Callback { + void process(List params, FileChannel channel); } + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/flusher/Flusher.java b/fluency-core/src/main/java/org/komamitsu/fluency/flusher/Flusher.java index 97a42463..91047e76 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/flusher/Flusher.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/flusher/Flusher.java @@ -16,15 +16,6 @@ package org.komamitsu.fluency.flusher; -import org.komamitsu.fluency.buffer.Buffer; -import org.komamitsu.fluency.ingester.Ingester; -import org.komamitsu.fluency.util.ExecutorServiceUtils; -import org.komamitsu.fluency.validation.Validatable; -import org.komamitsu.fluency.validation.annotation.Max; -import org.komamitsu.fluency.validation.annotation.Min; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.Closeable; import java.io.Flushable; import java.util.concurrent.BlockingQueue; @@ -36,271 +27,248 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import org.komamitsu.fluency.buffer.Buffer; +import org.komamitsu.fluency.ingester.Ingester; +import org.komamitsu.fluency.util.ExecutorServiceUtils; +import org.komamitsu.fluency.validation.Validatable; +import org.komamitsu.fluency.validation.annotation.Max; +import org.komamitsu.fluency.validation.annotation.Min; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class Flusher - implements Flushable, Closeable -{ - private static final Logger LOG = LoggerFactory.getLogger(Flusher.class); - private static final int DEFAULT_EVENT_QUEUE_SIZE = 16; - protected final Buffer buffer; - protected final Ingester ingester; - private final AtomicBoolean isTerminated = new AtomicBoolean(); - private final Config config; - private final BlockingQueue eventQueue = new LinkedBlockingQueue<>(DEFAULT_EVENT_QUEUE_SIZE); - private final ExecutorService executorService = ExecutorServiceUtils.newSingleThreadDaemonExecutor(); - - public Flusher(Config config, Buffer buffer, Ingester ingester) - { - config.validateValues(); - this.config = config; - this.buffer = buffer; - this.ingester = ingester; - executorService.execute(this::runLoop); - } - - private void runLoop() { - Boolean wakeup = null; - do { - try { - wakeup = eventQueue.poll(config.getFlushAttemptIntervalMillis(), TimeUnit.MILLISECONDS); - boolean force = wakeup != null; - buffer.flush(ingester, force); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - catch (Throwable e) { - LOG.error("Failed to flush", e); - } - } - while (!executorService.isShutdown()); - - if (wakeup == null) { - // The above run loop can quit without force buffer flush in the following cases - // - close() is called right after the repeated non-force buffer flush executed in the run loop - // - // In these cases, remaining buffers won't be flushed. - // So force buffer flush is executed here just in case - try { - buffer.flush(ingester, true); - } - catch (Throwable e) { - LOG.error("Failed to flush", e); - } - } +public class Flusher implements Flushable, Closeable { + private static final Logger LOG = LoggerFactory.getLogger(Flusher.class); + private static final int DEFAULT_EVENT_QUEUE_SIZE = 16; + protected final Buffer buffer; + protected final Ingester ingester; + private final AtomicBoolean isTerminated = new AtomicBoolean(); + private final Config config; + private final BlockingQueue eventQueue = + new LinkedBlockingQueue<>(DEFAULT_EVENT_QUEUE_SIZE); + private final ExecutorService executorService = + ExecutorServiceUtils.newSingleThreadDaemonExecutor(); + + public Flusher(Config config, Buffer buffer, Ingester ingester) { + config.validateValues(); + this.config = config; + this.buffer = buffer; + this.ingester = ingester; + executorService.execute(this::runLoop); + } + + private void runLoop() { + Boolean wakeup = null; + do { + try { + wakeup = eventQueue.poll(config.getFlushAttemptIntervalMillis(), TimeUnit.MILLISECONDS); + boolean force = wakeup != null; + buffer.flush(ingester, force); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Throwable e) { + LOG.error("Failed to flush", e); + } + } while (!executorService.isShutdown()); + + if (wakeup == null) { + // The above run loop can quit without force buffer flush in the following cases + // - close() is called right after the repeated non-force buffer flush executed in the run + // loop + // + // In these cases, remaining buffers won't be flushed. + // So force buffer flush is executed here just in case + try { + buffer.flush(ingester, true); + } catch (Throwable e) { + LOG.error("Failed to flush", e); + } } + } - public Buffer getBuffer() - { - return buffer; - } + public Buffer getBuffer() { + return buffer; + } - @Override - public void flush() - { - eventQueue.offer(true); - } + @Override + public void flush() { + eventQueue.offer(true); + } - private void flushBufferQuietly() - { - LOG.trace("Flushing the buffer"); + private void flushBufferQuietly() { + LOG.trace("Flushing the buffer"); - try { - flush(); - } - catch (Throwable e) { - LOG.error("Failed to call flush()", e); - } + try { + flush(); + } catch (Throwable e) { + LOG.error("Failed to call flush()", e); } + } - private void finishExecutorQuietly() - { - LOG.trace("Finishing the executor"); + private void finishExecutorQuietly() { + LOG.trace("Finishing the executor"); - ExecutorServiceUtils.finishExecutorService( - executorService, - config.getWaitUntilBufferFlushed()); - } + ExecutorServiceUtils.finishExecutorService(executorService, config.getWaitUntilBufferFlushed()); + } - private void closeBufferQuietly() - { - LOG.trace("Closing the buffer"); + private void closeBufferQuietly() { + LOG.trace("Closing the buffer"); - try { - buffer.close(); - } - catch (Throwable e) { - LOG.warn("Failed to close the buffer", e); - } + try { + buffer.close(); + } catch (Throwable e) { + LOG.warn("Failed to close the buffer", e); } + } - private void closeIngesterQuietly() - { - LOG.trace("Closing the ingester"); - - try { - // Close the socket at the end to prevent the server from failing to read from the connection - ingester.close(); - } - catch (Throwable e) { - LOG.error("Failed to close the ingester", e); - } - } + private void closeIngesterQuietly() { + LOG.trace("Closing the ingester"); - private void closeResourcesQuietly() - { - ExecutorService executorService = Executors.newSingleThreadExecutor(); - try { - Future future = executorService.submit(() -> { + try { + // Close the socket at the end to prevent the server from failing to read from the connection + ingester.close(); + } catch (Throwable e) { + LOG.error("Failed to close the ingester", e); + } + } + + private void closeResourcesQuietly() { + ExecutorService executorService = Executors.newSingleThreadExecutor(); + try { + Future future = + executorService.submit( + () -> { closeBufferQuietly(); closeIngesterQuietly(); isTerminated.set(true); return null; - }); - future.get(getWaitUntilTerminated(), TimeUnit.SECONDS); - } - catch (InterruptedException e) { - LOG.warn("Interrupted", e); - Thread.currentThread().interrupt(); - } - catch (ExecutionException e) { - LOG.warn("closeBuffer() failed", e); - } - catch (TimeoutException e) { - LOG.warn("closeBuffer() timed out", e); - } - finally { - executorService.shutdown();; - } + }); + future.get(getWaitUntilTerminated(), TimeUnit.SECONDS); + } catch (InterruptedException e) { + LOG.warn("Interrupted", e); + Thread.currentThread().interrupt(); + } catch (ExecutionException e) { + LOG.warn("closeBuffer() failed", e); + } catch (TimeoutException e) { + LOG.warn("closeBuffer() timed out", e); + } finally { + executorService.shutdown(); + ; + } + } + + @Override + public void close() { + flushBufferQuietly(); + + finishExecutorQuietly(); + + closeResourcesQuietly(); + } + + public boolean isTerminated() { + return isTerminated.get(); + } + + public Ingester getIngester() { + return ingester; + } + + /** @deprecated As of release 2.4.0, replaced by {@link #getFlushAttemptIntervalMillis()} */ + @Deprecated + public int getFlushIntervalMillis() { + return config.getFlushAttemptIntervalMillis(); + } + + public int getFlushAttemptIntervalMillis() { + return config.getFlushAttemptIntervalMillis(); + } + + public int getWaitUntilBufferFlushed() { + return config.getWaitUntilBufferFlushed(); + } + + public int getWaitUntilTerminated() { + return config.getWaitUntilTerminated(); + } + + @Override + public String toString() { + return "Flusher{" + + "isTerminated=" + + isTerminated + + ", buffer=" + + buffer + + ", ingester=" + + ingester + + ", config=" + + config + + '}'; + } + + public static class Config implements Validatable { + @Min(20) + @Max(2000) + private int flushAttemptIntervalMillis = 600; + + @Min(1) + private int waitUntilBufferFlushed = 60; + + @Min(1) + private int waitUntilTerminated = 60; + + /** @deprecated As of release 2.4.0, replaced by {@link #getFlushAttemptIntervalMillis()} */ + @Deprecated + public int getFlushIntervalMillis() { + return flushAttemptIntervalMillis; } - @Override - public void close() - { - flushBufferQuietly(); - - finishExecutorQuietly(); - - closeResourcesQuietly(); + /** + * @deprecated As of release 2.4.0, replaced by {@link #setFlushAttemptIntervalMillis(int + * flushAttemptIntervalMillis)} + */ + @Deprecated + public void setFlushIntervalMillis(int flushAttemptIntervalMillis) { + this.flushAttemptIntervalMillis = flushAttemptIntervalMillis; } - public boolean isTerminated() - { - return isTerminated.get(); + public int getFlushAttemptIntervalMillis() { + return flushAttemptIntervalMillis; } - public Ingester getIngester() - { - return ingester; + public void setFlushAttemptIntervalMillis(int flushAttemptIntervalMillis) { + this.flushAttemptIntervalMillis = flushAttemptIntervalMillis; } - /** - * @deprecated As of release 2.4.0, replaced by {@link #getFlushAttemptIntervalMillis()} - */ - @Deprecated - public int getFlushIntervalMillis() - { - return config.getFlushAttemptIntervalMillis(); + public int getWaitUntilBufferFlushed() { + return waitUntilBufferFlushed; } - public int getFlushAttemptIntervalMillis() - { - return config.getFlushAttemptIntervalMillis(); + public void setWaitUntilBufferFlushed(int waitUntilBufferFlushed) { + this.waitUntilBufferFlushed = waitUntilBufferFlushed; } - public int getWaitUntilBufferFlushed() - { - return config.getWaitUntilBufferFlushed(); + public int getWaitUntilTerminated() { + return waitUntilTerminated; } - public int getWaitUntilTerminated() - { - return config.getWaitUntilTerminated(); + public void setWaitUntilTerminated(int waitUntilTerminated) { + this.waitUntilTerminated = waitUntilTerminated; } - @Override - public String toString() - { - return "Flusher{" + - "isTerminated=" + isTerminated + - ", buffer=" + buffer + - ", ingester=" + ingester + - ", config=" + config + - '}'; + void validateValues() { + validate(); } - public static class Config - implements Validatable - { - @Min(20) - @Max(2000) - private int flushAttemptIntervalMillis = 600; - @Min(1) - private int waitUntilBufferFlushed = 60; - @Min(1) - private int waitUntilTerminated = 60; - - /** - * @deprecated As of release 2.4.0, replaced by {@link #getFlushAttemptIntervalMillis()} - */ - @Deprecated - public int getFlushIntervalMillis() - { - return flushAttemptIntervalMillis; - } - - /** - * @deprecated As of release 2.4.0, replaced by {@link #setFlushAttemptIntervalMillis(int flushAttemptIntervalMillis)} - */ - @Deprecated - public void setFlushIntervalMillis(int flushAttemptIntervalMillis) - { - this.flushAttemptIntervalMillis = flushAttemptIntervalMillis; - } - - public int getFlushAttemptIntervalMillis() - { - return flushAttemptIntervalMillis; - } - - public void setFlushAttemptIntervalMillis(int flushAttemptIntervalMillis) - { - this.flushAttemptIntervalMillis = flushAttemptIntervalMillis; - } - - public int getWaitUntilBufferFlushed() - { - return waitUntilBufferFlushed; - } - - public void setWaitUntilBufferFlushed(int waitUntilBufferFlushed) - { - this.waitUntilBufferFlushed = waitUntilBufferFlushed; - } - - public int getWaitUntilTerminated() - { - return waitUntilTerminated; - } - - public void setWaitUntilTerminated(int waitUntilTerminated) - { - this.waitUntilTerminated = waitUntilTerminated; - } - - void validateValues() - { - validate(); - } - - @Override - public String toString() - { - return "Config{" + - "flushAttemptIntervalMillis=" + flushAttemptIntervalMillis + - ", waitUntilBufferFlushed=" + waitUntilBufferFlushed + - ", waitUntilTerminated=" + waitUntilTerminated + - '}'; - } + @Override + public String toString() { + return "Config{" + + "flushAttemptIntervalMillis=" + + flushAttemptIntervalMillis + + ", waitUntilBufferFlushed=" + + waitUntilBufferFlushed + + ", waitUntilTerminated=" + + waitUntilTerminated + + '}'; } + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/ingester/Ingester.java b/fluency-core/src/main/java/org/komamitsu/fluency/ingester/Ingester.java index b84d157f..8c9b09c0 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/ingester/Ingester.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/ingester/Ingester.java @@ -16,17 +16,13 @@ package org.komamitsu.fluency.ingester; -import org.komamitsu.fluency.ingester.sender.Sender; - import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; +import org.komamitsu.fluency.ingester.sender.Sender; -public interface Ingester - extends Closeable -{ - void ingest(String tag, ByteBuffer dataBuffer) - throws IOException; +public interface Ingester extends Closeable { + void ingest(String tag, ByteBuffer dataBuffer) throws IOException; - Sender getSender(); + Sender getSender(); } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/ingester/sender/ErrorHandler.java b/fluency-core/src/main/java/org/komamitsu/fluency/ingester/sender/ErrorHandler.java index 7a6d0d50..3c46f0eb 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/ingester/sender/ErrorHandler.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/ingester/sender/ErrorHandler.java @@ -16,7 +16,6 @@ package org.komamitsu.fluency.ingester.sender; -public interface ErrorHandler -{ - void handle(Throwable e); +public interface ErrorHandler { + void handle(Throwable e); } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/ingester/sender/Sender.java b/fluency-core/src/main/java/org/komamitsu/fluency/ingester/sender/Sender.java index 89314581..3a3d63c1 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/ingester/sender/Sender.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/ingester/sender/Sender.java @@ -16,28 +16,21 @@ package org.komamitsu.fluency.ingester.sender; -public interface Sender -{ - class Config - { - private ErrorHandler errorHandler; +public interface Sender { + class Config { + private ErrorHandler errorHandler; - public ErrorHandler getErrorHandler() - { - return errorHandler; - } + public ErrorHandler getErrorHandler() { + return errorHandler; + } - public void setErrorHandler(ErrorHandler errorHandler) - { - this.errorHandler = errorHandler; - } + public void setErrorHandler(ErrorHandler errorHandler) { + this.errorHandler = errorHandler; + } - @Override - public String toString() - { - return "Config{" + - "senderErrorHandler=" + errorHandler + - '}'; - } + @Override + public String toString() { + return "Config{" + "senderErrorHandler=" + errorHandler + '}'; } + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/AbstractRecordFormatter.java b/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/AbstractRecordFormatter.java index 0979de04..dff72256 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/AbstractRecordFormatter.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/AbstractRecordFormatter.java @@ -18,55 +18,49 @@ import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; import org.komamitsu.fluency.EventTime; import org.msgpack.jackson.dataformat.MessagePackFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Map; - -public abstract class AbstractRecordFormatter - implements RecordFormatter -{ - private static final Logger LOG = LoggerFactory.getLogger(AbstractRecordFormatter.class); - protected final ObjectMapper objectMapperForMessagePack = new ObjectMapper(new MessagePackFactory()); - protected final Config config; +public abstract class AbstractRecordFormatter implements RecordFormatter { + private static final Logger LOG = LoggerFactory.getLogger(AbstractRecordFormatter.class); + protected final ObjectMapper objectMapperForMessagePack = + new ObjectMapper(new MessagePackFactory()); + protected final Config config; - public AbstractRecordFormatter(Config config) - { - this.config = config; - registerObjectMapperModules(objectMapperForMessagePack); - } + public AbstractRecordFormatter(Config config) { + this.config = config; + registerObjectMapperModules(objectMapperForMessagePack); + } - public abstract byte[] format(String tag, Object timestamp, Map data); + public abstract byte[] format(String tag, Object timestamp, Map data); - public abstract byte[] formatFromMessagePack(String tag, Object timestamp, byte[] mapValue, int offset, int len); + public abstract byte[] formatFromMessagePack( + String tag, Object timestamp, byte[] mapValue, int offset, int len); - public abstract byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue); + public abstract byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue); - public abstract String formatName(); + public abstract String formatName(); - protected long getEpoch(Object timestamp) - { - if (timestamp instanceof EventTime) { - return ((EventTime) timestamp).getSeconds(); - } - else if (timestamp instanceof Number) { - return ((Number) timestamp).longValue(); - } - else { - LOG.warn("Invalid timestamp. Using current time: timestamp={}", timestamp); - return System.currentTimeMillis() / 1000; - } + protected long getEpoch(Object timestamp) { + if (timestamp instanceof EventTime) { + return ((EventTime) timestamp).getSeconds(); + } else if (timestamp instanceof Number) { + return ((Number) timestamp).longValue(); + } else { + LOG.warn("Invalid timestamp. Using current time: timestamp={}", timestamp); + return System.currentTimeMillis() / 1000; } + } - protected void registerObjectMapperModules(ObjectMapper objectMapper) - { - List jacksonModules = config.getJacksonModules(); - for (Module module : jacksonModules) { - objectMapper.registerModule(module); - } + protected void registerObjectMapperModules(ObjectMapper objectMapper) { + List jacksonModules = config.getJacksonModules(); + for (Module module : jacksonModules) { + objectMapper.registerModule(module); } + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/MessagePackRecordFormatter.java b/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/MessagePackRecordFormatter.java index 343ad4c6..c5d47326 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/MessagePackRecordFormatter.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/MessagePackRecordFormatter.java @@ -16,85 +16,75 @@ package org.komamitsu.fluency.recordformat; +import java.nio.ByteBuffer; +import java.util.Map; import org.komamitsu.fluency.recordformat.recordaccessor.MapRecordAccessor; import org.komamitsu.fluency.recordformat.recordaccessor.MessagePackRecordAccessor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.nio.ByteBuffer; -import java.util.Map; +public class MessagePackRecordFormatter extends AbstractRecordFormatter { + private static final Logger LOG = LoggerFactory.getLogger(MessagePackRecordFormatter.class); -public class MessagePackRecordFormatter - extends AbstractRecordFormatter -{ - private static final Logger LOG = LoggerFactory.getLogger(MessagePackRecordFormatter.class); + public MessagePackRecordFormatter() { + this(new Config()); + } - public MessagePackRecordFormatter() - { - this(new Config()); - } + public MessagePackRecordFormatter(Config config) { + super(config); + } - public MessagePackRecordFormatter(Config config) - { - super(config); + @Override + public byte[] format(String tag, Object timestamp, Map data) { + try { + MapRecordAccessor recordAccessor = new MapRecordAccessor(data); + recordAccessor.setTimestamp(getEpoch(timestamp)); + return recordAccessor.toMessagePack(objectMapperForMessagePack); + } catch (Throwable e) { + LOG.error( + String.format( + "Failed to format a Map record: cause=%s, tag=%s, timestamp=%s, recordCount=%d", + e.getMessage(), tag, timestamp, data.size())); + throw e; } + } - @Override - public byte[] format(String tag, Object timestamp, Map data) - { - try { - MapRecordAccessor recordAccessor = new MapRecordAccessor(data); - recordAccessor.setTimestamp(getEpoch(timestamp)); - return recordAccessor.toMessagePack(objectMapperForMessagePack); - } - catch (Throwable e) { - LOG.error(String.format( - "Failed to format a Map record: cause=%s, tag=%s, timestamp=%s, recordCount=%d", - e.getMessage(), tag, timestamp, data.size())); - throw e; - } + @Override + public byte[] formatFromMessagePack( + String tag, Object timestamp, byte[] mapValue, int offset, int len) { + try { + MessagePackRecordAccessor recordAccessor = + new MessagePackRecordAccessor(ByteBuffer.wrap(mapValue, offset, len)); + recordAccessor.setTimestamp(getEpoch(timestamp)); + return recordAccessor.toMessagePack(objectMapperForMessagePack); + } catch (Throwable e) { + LOG.error( + String.format( + "Failed to format a MessagePack record: cause=%s, tag=%s, timestamp=%s, offset=%d, len=%d", + e.getMessage(), tag, timestamp, offset, len)); + throw e; } + } - @Override - public byte[] formatFromMessagePack(String tag, Object timestamp, byte[] mapValue, int offset, int len) - { - try { - MessagePackRecordAccessor recordAccessor = new MessagePackRecordAccessor(ByteBuffer.wrap(mapValue, offset, len)); - recordAccessor.setTimestamp(getEpoch(timestamp)); - return recordAccessor.toMessagePack(objectMapperForMessagePack); - } - catch (Throwable e) { - LOG.error(String.format( - "Failed to format a MessagePack record: cause=%s, tag=%s, timestamp=%s, offset=%d, len=%d", - e.getMessage(), tag, timestamp, offset, len)); - throw e; - } + @Override + public byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue) { + try { + MessagePackRecordAccessor recordAccessor = new MessagePackRecordAccessor(mapValue); + recordAccessor.setTimestamp(getEpoch(timestamp)); + return recordAccessor.toMessagePack(objectMapperForMessagePack); + } catch (Throwable e) { + LOG.error( + String.format( + "Failed to format a MessagePack record: cause=%s, tag=%s, timestamp=%s, bytebuf=%s", + e.getMessage(), tag, timestamp, mapValue)); + throw e; } + } - @Override - public byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue) - { - try { - MessagePackRecordAccessor recordAccessor = new MessagePackRecordAccessor(mapValue); - recordAccessor.setTimestamp(getEpoch(timestamp)); - return recordAccessor.toMessagePack(objectMapperForMessagePack); - } - catch (Throwable e) { - LOG.error(String.format( - "Failed to format a MessagePack record: cause=%s, tag=%s, timestamp=%s, bytebuf=%s", - e.getMessage(), tag, timestamp, mapValue)); - throw e; - } - } - - @Override - public String formatName() - { - return "msgpack"; - } + @Override + public String formatName() { + return "msgpack"; + } - public static class Config - extends RecordFormatter.Config - { - } + public static class Config extends RecordFormatter.Config {} } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/RecordFormatter.java b/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/RecordFormatter.java index 996b8c81..393981ff 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/RecordFormatter.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/RecordFormatter.java @@ -17,34 +17,29 @@ package org.komamitsu.fluency.recordformat; import com.fasterxml.jackson.databind.Module; - import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; import java.util.Map; -public interface RecordFormatter -{ - byte[] format(String tag, Object timestamp, Map data); +public interface RecordFormatter { + byte[] format(String tag, Object timestamp, Map data); - byte[] formatFromMessagePack(String tag, Object timestamp, byte[] mapValue, int offset, int len); + byte[] formatFromMessagePack(String tag, Object timestamp, byte[] mapValue, int offset, int len); - byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue); + byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue); - String formatName(); + String formatName(); - class Config - { - private List jacksonModules = Collections.emptyList(); + class Config { + private List jacksonModules = Collections.emptyList(); - public List getJacksonModules() - { - return jacksonModules; - } + public List getJacksonModules() { + return jacksonModules; + } - public void setJacksonModules(List jacksonModules) - { - this.jacksonModules = jacksonModules; - } + public void setJacksonModules(List jacksonModules) { + this.jacksonModules = jacksonModules; } + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/recordaccessor/MapRecordAccessor.java b/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/recordaccessor/MapRecordAccessor.java index 26ab9190..0b64b699 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/recordaccessor/MapRecordAccessor.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/recordaccessor/MapRecordAccessor.java @@ -18,55 +18,45 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; - import java.util.HashMap; import java.util.Map; -public class MapRecordAccessor - implements RecordAccessor -{ - private Map map; - - public MapRecordAccessor(Map map) - { - this.map = new HashMap<>(map); - } +public class MapRecordAccessor implements RecordAccessor { + private Map map; - @Override - public String getAsString(String key) - { - Object value = map.get(key); - if (value == null) { - return null; - } - return value.toString(); - } + public MapRecordAccessor(Map map) { + this.map = new HashMap<>(map); + } - @Override - public void setTimestamp(long timestamp) - { - map.putIfAbsent("time", timestamp); + @Override + public String getAsString(String key) { + Object value = map.get(key); + if (value == null) { + return null; } - - @Override - public byte[] toMessagePack(ObjectMapper objectMapperForMessagePack) - { - try { - return objectMapperForMessagePack.writeValueAsBytes(map); - } - catch (JsonProcessingException e) { - throw new IllegalStateException("Failed to serialize the map to MessagePack format", e); - } + return value.toString(); + } + + @Override + public void setTimestamp(long timestamp) { + map.putIfAbsent("time", timestamp); + } + + @Override + public byte[] toMessagePack(ObjectMapper objectMapperForMessagePack) { + try { + return objectMapperForMessagePack.writeValueAsBytes(map); + } catch (JsonProcessingException e) { + throw new IllegalStateException("Failed to serialize the map to MessagePack format", e); } - - @Override - public String toJson(ObjectMapper objectMapperForJson) - { - try { - return objectMapperForJson.writeValueAsString(map); - } - catch (JsonProcessingException e) { - throw new IllegalStateException("Failed to serialize the map to JSON format", e); - } + } + + @Override + public String toJson(ObjectMapper objectMapperForJson) { + try { + return objectMapperForJson.writeValueAsString(map); + } catch (JsonProcessingException e) { + throw new IllegalStateException("Failed to serialize the map to JSON format", e); } + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/recordaccessor/MessagePackRecordAccessor.java b/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/recordaccessor/MessagePackRecordAccessor.java index 7ed29a31..81640474 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/recordaccessor/MessagePackRecordAccessor.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/recordaccessor/MessagePackRecordAccessor.java @@ -17,6 +17,10 @@ package org.komamitsu.fluency.recordformat.recordaccessor; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Map; import org.msgpack.core.MessagePack; import org.msgpack.core.MessagePacker; import org.msgpack.core.MessageUnpacker; @@ -24,96 +28,79 @@ import org.msgpack.value.Value; import org.msgpack.value.ValueFactory; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Map; - -public class MessagePackRecordAccessor - implements RecordAccessor -{ - private static final StringValue KEY_TIME = ValueFactory.newString("time"); - private MapValueBytes mapValueBytes; - - private static class MapValueBytes { - private final byte[] byteArray; - private final Map map; +public class MessagePackRecordAccessor implements RecordAccessor { + private static final StringValue KEY_TIME = ValueFactory.newString("time"); + private MapValueBytes mapValueBytes; - MapValueBytes(ByteBuffer byteBuffer) - { - byte[] mapValueBytes = new byte[byteBuffer.remaining()]; - byteBuffer.get(mapValueBytes); - try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(mapValueBytes)) { - this.map = unpacker.unpackValue().asMapValue().map(); - this.byteArray = mapValueBytes; - } - catch (IOException e) { - throw new IllegalArgumentException("Invalid MessagePack ByteBuffer", e); - } - } - - byte[] byteArray() - { - return byteArray; - } + private static class MapValueBytes { + private final byte[] byteArray; + private final Map map; - Map map() - { - return map; - } + MapValueBytes(ByteBuffer byteBuffer) { + byte[] mapValueBytes = new byte[byteBuffer.remaining()]; + byteBuffer.get(mapValueBytes); + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(mapValueBytes)) { + this.map = unpacker.unpackValue().asMapValue().map(); + this.byteArray = mapValueBytes; + } catch (IOException e) { + throw new IllegalArgumentException("Invalid MessagePack ByteBuffer", e); + } } - public MessagePackRecordAccessor(ByteBuffer byteBuffer) - { - mapValueBytes = new MapValueBytes(byteBuffer); + byte[] byteArray() { + return byteArray; } - @Override - public String getAsString(String key) - { - Value value = mapValueBytes.map().get(ValueFactory.newString(key)); - if (value == null) { - return null; - } - return value.toString(); + Map map() { + return map; } + } - @Override - public void setTimestamp(long timestamp) - { - int mapSize = mapValueBytes.map().size(); - try (ByteArrayOutputStream output = new ByteArrayOutputStream(mapValueBytes.byteArray().length + 16)) { - try (MessagePacker packer = MessagePack.newDefaultPacker(output)) { - if (mapValueBytes.map().containsKey(KEY_TIME)) { - packer.packMapHeader(mapSize); - } - else { - packer.packMapHeader(mapSize + 1); - KEY_TIME.writeTo(packer); - packer.packLong(timestamp); - } + public MessagePackRecordAccessor(ByteBuffer byteBuffer) { + mapValueBytes = new MapValueBytes(byteBuffer); + } - for (Map.Entry entry : mapValueBytes.map().entrySet()) { - packer.packValue(entry.getKey()); - packer.packValue(entry.getValue()); - } - } - mapValueBytes = new MapValueBytes(ByteBuffer.wrap(output.toByteArray())); + @Override + public String getAsString(String key) { + Value value = mapValueBytes.map().get(ValueFactory.newString(key)); + if (value == null) { + return null; + } + return value.toString(); + } + + @Override + public void setTimestamp(long timestamp) { + int mapSize = mapValueBytes.map().size(); + try (ByteArrayOutputStream output = + new ByteArrayOutputStream(mapValueBytes.byteArray().length + 16)) { + try (MessagePacker packer = MessagePack.newDefaultPacker(output)) { + if (mapValueBytes.map().containsKey(KEY_TIME)) { + packer.packMapHeader(mapSize); + } else { + packer.packMapHeader(mapSize + 1); + KEY_TIME.writeTo(packer); + packer.packLong(timestamp); } - catch (IOException e) { - throw new IllegalStateException("Failed to upsert `time` field", e); + + for (Map.Entry entry : mapValueBytes.map().entrySet()) { + packer.packValue(entry.getKey()); + packer.packValue(entry.getValue()); } + } + mapValueBytes = new MapValueBytes(ByteBuffer.wrap(output.toByteArray())); + } catch (IOException e) { + throw new IllegalStateException("Failed to upsert `time` field", e); } + } - @Override - public byte[] toMessagePack(ObjectMapper objectMapperForMessagePack) - { - return mapValueBytes.byteArray(); - } + @Override + public byte[] toMessagePack(ObjectMapper objectMapperForMessagePack) { + return mapValueBytes.byteArray(); + } - @Override - public String toJson(ObjectMapper objectMapperForJson) - { - return ValueFactory.newMap(mapValueBytes.map()).toJson(); - } + @Override + public String toJson(ObjectMapper objectMapperForJson) { + return ValueFactory.newMap(mapValueBytes.map()).toJson(); + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/recordaccessor/RecordAccessor.java b/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/recordaccessor/RecordAccessor.java index e79292f7..56502a17 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/recordaccessor/RecordAccessor.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/recordaccessor/RecordAccessor.java @@ -16,16 +16,14 @@ package org.komamitsu.fluency.recordformat.recordaccessor; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -public interface RecordAccessor -{ - String getAsString(String key); +public interface RecordAccessor { + String getAsString(String key); - void setTimestamp(long timestamp); + void setTimestamp(long timestamp); - byte[] toMessagePack(ObjectMapper objectMapperForMessagePack); + byte[] toMessagePack(ObjectMapper objectMapperForMessagePack); - String toJson(ObjectMapper objectMapperForJson); + String toJson(ObjectMapper objectMapperForJson); } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/util/ExecutorServiceUtils.java b/fluency-core/src/main/java/org/komamitsu/fluency/util/ExecutorServiceUtils.java index 643e84ae..05d9b851 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/util/ExecutorServiceUtils.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/util/ExecutorServiceUtils.java @@ -16,73 +16,65 @@ package org.komamitsu.fluency.util; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class ExecutorServiceUtils -{ - private static final Logger LOG = LoggerFactory.getLogger(ExecutorServiceUtils.class); +public class ExecutorServiceUtils { + private static final Logger LOG = LoggerFactory.getLogger(ExecutorServiceUtils.class); - /** - * Creates an Executor that is based on daemon threads. - * This allows the program to quit without explicitly - * calling shutdown on the pool - * - * @return the newly created single-threaded Executor - */ - public static ExecutorService newSingleThreadDaemonExecutor() { - return Executors.newSingleThreadExecutor(r -> { - Thread t = Executors.defaultThreadFactory().newThread(r); - t.setDaemon(true); - return t; + /** + * Creates an Executor that is based on daemon threads. This allows the program to quit without + * explicitly calling shutdown on the pool + * + * @return the newly created single-threaded Executor + */ + public static ExecutorService newSingleThreadDaemonExecutor() { + return Executors.newSingleThreadExecutor( + r -> { + Thread t = Executors.defaultThreadFactory().newThread(r); + t.setDaemon(true); + return t; }); - } - - /** - * Creates a scheduled thread pool where each thread has the daemon - * property set to true. This allows the program to quit without - * explicitly calling shutdown on the pool - * - * @param corePoolSize the number of threads to keep in the pool, - * even if they are idle + } - * @return a newly created scheduled thread pool - */ - public static ScheduledExecutorService newScheduledDaemonThreadPool(int corePoolSize) { - return Executors.newScheduledThreadPool(corePoolSize, r -> { - Thread t = Executors.defaultThreadFactory().newThread(r); - t.setDaemon(true); - return t; + /** + * Creates a scheduled thread pool where each thread has the daemon property set to true. This + * allows the program to quit without explicitly calling shutdown on the pool + * + * @param corePoolSize the number of threads to keep in the pool, even if they are idle + * @return a newly created scheduled thread pool + */ + public static ScheduledExecutorService newScheduledDaemonThreadPool(int corePoolSize) { + return Executors.newScheduledThreadPool( + corePoolSize, + r -> { + Thread t = Executors.defaultThreadFactory().newThread(r); + t.setDaemon(true); + return t; }); - } + } + public static void finishExecutorService(ExecutorService executorService) { + finishExecutorService(executorService, 3); + } - public static void finishExecutorService(ExecutorService executorService) - { - finishExecutorService(executorService, 3); + public static void finishExecutorService(ExecutorService executorService, long waitSecond) { + executorService.shutdown(); + try { + executorService.awaitTermination(waitSecond, TimeUnit.SECONDS); + } catch (InterruptedException e) { + LOG.warn("1st awaitTermination was interrupted", e); + Thread.currentThread().interrupt(); + } catch (Throwable e) { + LOG.warn("Failed to await the termination of executorService", e); } - public static void finishExecutorService(ExecutorService executorService, long waitSecond) - { - executorService.shutdown(); - try { - executorService.awaitTermination(waitSecond, TimeUnit.SECONDS); - } - catch (InterruptedException e) { - LOG.warn("1st awaitTermination was interrupted", e); - Thread.currentThread().interrupt(); - } - catch (Throwable e) { - LOG.warn("Failed to await the termination of executorService", e); - } - - if (!executorService.isTerminated()) { - executorService.shutdownNow(); - } + if (!executorService.isTerminated()) { + executorService.shutdownNow(); } + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/util/Tuple.java b/fluency-core/src/main/java/org/komamitsu/fluency/util/Tuple.java index f8beeab4..80a28846 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/util/Tuple.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/util/Tuple.java @@ -16,24 +16,20 @@ package org.komamitsu.fluency.util; -public class Tuple -{ - private final F first; - private final S second; +public class Tuple { + private final F first; + private final S second; - public Tuple(F first, S second) - { - this.first = first; - this.second = second; - } + public Tuple(F first, S second) { + this.first = first; + this.second = second; + } - public F getFirst() - { - return first; - } + public F getFirst() { + return first; + } - public S getSecond() - { - return second; - } + public S getSecond() { + return second; + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/util/Tuple3.java b/fluency-core/src/main/java/org/komamitsu/fluency/util/Tuple3.java index dfd9543d..47f8ff63 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/util/Tuple3.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/util/Tuple3.java @@ -16,31 +16,26 @@ package org.komamitsu.fluency.util; -public class Tuple3 -{ - private final F first; - private final S second; - private final T third; +public class Tuple3 { + private final F first; + private final S second; + private final T third; - public Tuple3(F first, S second, T third) - { - this.first = first; - this.second = second; - this.third = third; - } + public Tuple3(F first, S second, T third) { + this.first = first; + this.second = second; + this.third = third; + } - public F getFirst() - { - return first; - } + public F getFirst() { + return first; + } - public S getSecond() - { - return second; - } + public S getSecond() { + return second; + } - public T getThird() - { - return third; - } + public T getThird() { + return third; + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/validation/Validatable.java b/fluency-core/src/main/java/org/komamitsu/fluency/validation/Validatable.java index 7a0a326b..5292ce60 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/validation/Validatable.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/validation/Validatable.java @@ -16,137 +16,131 @@ package org.komamitsu.fluency.validation; -import org.komamitsu.fluency.validation.annotation.DecimalMax; -import org.komamitsu.fluency.validation.annotation.DecimalMin; -import org.komamitsu.fluency.validation.annotation.Max; -import org.komamitsu.fluency.validation.annotation.Min; - import java.lang.annotation.Annotation; import java.lang.reflect.Field; import java.math.BigDecimal; import java.util.Arrays; import java.util.List; import java.util.function.BiFunction; +import org.komamitsu.fluency.validation.annotation.DecimalMax; +import org.komamitsu.fluency.validation.annotation.DecimalMin; +import org.komamitsu.fluency.validation.annotation.Max; +import org.komamitsu.fluency.validation.annotation.Min; -public interface Validatable -{ - class ValidationList { - private static class Validation - { - Class annotationClass; - BiFunction isValid; - String messageTemplate; +public interface Validatable { + class ValidationList { + private static class Validation { + Class annotationClass; + BiFunction isValid; + String messageTemplate; - Validation( - Class annotationClass, - BiFunction isValid, - String messageTemplate) - { - this.annotationClass = annotationClass; - this.isValid = isValid; - this.messageTemplate = messageTemplate; - } - } + Validation( + Class annotationClass, + BiFunction isValid, + String messageTemplate) { + this.annotationClass = annotationClass; + this.isValid = isValid; + this.messageTemplate = messageTemplate; + } + } - private static final Validation VALIDATION_MAX = new Validation( - Max.class, - (annotation, actual) -> { - Max maxAnnotation = (Max) annotation; - if (maxAnnotation.inclusive()) { - return maxAnnotation.value() >= actual.longValue(); - } - else { - return maxAnnotation.value() > actual.longValue(); - } - }, - "This field (%s) is more than (%s)"); + private static final Validation VALIDATION_MAX = + new Validation( + Max.class, + (annotation, actual) -> { + Max maxAnnotation = (Max) annotation; + if (maxAnnotation.inclusive()) { + return maxAnnotation.value() >= actual.longValue(); + } else { + return maxAnnotation.value() > actual.longValue(); + } + }, + "This field (%s) is more than (%s)"); - private static final Validation VALIDATION_MIN = new Validation( - Min.class, - (annotation, actual) -> { - Min minAnnotation = (Min) annotation; - if (minAnnotation.inclusive()) { - return minAnnotation.value() <= actual.longValue(); - } - else { - return minAnnotation.value() < actual.longValue(); - } - }, - "This field (%s) is less than (%s)"); + private static final Validation VALIDATION_MIN = + new Validation( + Min.class, + (annotation, actual) -> { + Min minAnnotation = (Min) annotation; + if (minAnnotation.inclusive()) { + return minAnnotation.value() <= actual.longValue(); + } else { + return minAnnotation.value() < actual.longValue(); + } + }, + "This field (%s) is less than (%s)"); - private static final Validation VALIDATION_DECIMAL_MAX = new Validation( - DecimalMax.class, - (annotation, actual) -> { - DecimalMax maxAnnotation = (DecimalMax) annotation; - BigDecimal limitValue = new BigDecimal(maxAnnotation.value()); - BigDecimal actualValue = new BigDecimal(actual.toString()); - if (maxAnnotation.inclusive()) { - return limitValue.compareTo(actualValue) >= 0; - } - else { - return limitValue.compareTo(actualValue) > 0; - } - }, - "This field (%s) is more than (%s)"); + private static final Validation VALIDATION_DECIMAL_MAX = + new Validation( + DecimalMax.class, + (annotation, actual) -> { + DecimalMax maxAnnotation = (DecimalMax) annotation; + BigDecimal limitValue = new BigDecimal(maxAnnotation.value()); + BigDecimal actualValue = new BigDecimal(actual.toString()); + if (maxAnnotation.inclusive()) { + return limitValue.compareTo(actualValue) >= 0; + } else { + return limitValue.compareTo(actualValue) > 0; + } + }, + "This field (%s) is more than (%s)"); - private static final Validation VALIDATION_DECIMAL_MIN = new Validation( - DecimalMin.class, - (annotation, actual) -> { - DecimalMin maxAnnotation = (DecimalMin) annotation; - BigDecimal limitValue = new BigDecimal(maxAnnotation.value()); - BigDecimal actualValue = new BigDecimal(actual.toString()); - if (maxAnnotation.inclusive()) { - return limitValue.compareTo(actualValue) <= 0; - } - else { - return limitValue.compareTo(actualValue) < 0; - } - }, - "This field (%s) is less than (%s)"); + private static final Validation VALIDATION_DECIMAL_MIN = + new Validation( + DecimalMin.class, + (annotation, actual) -> { + DecimalMin maxAnnotation = (DecimalMin) annotation; + BigDecimal limitValue = new BigDecimal(maxAnnotation.value()); + BigDecimal actualValue = new BigDecimal(actual.toString()); + if (maxAnnotation.inclusive()) { + return limitValue.compareTo(actualValue) <= 0; + } else { + return limitValue.compareTo(actualValue) < 0; + } + }, + "This field (%s) is less than (%s)"); - private static final List VALIDATIONS = Arrays.asList( - VALIDATION_MAX, - VALIDATION_MIN, - VALIDATION_DECIMAL_MAX, - VALIDATION_DECIMAL_MIN); - } + private static final List VALIDATIONS = + Arrays.asList( + VALIDATION_MAX, VALIDATION_MIN, VALIDATION_DECIMAL_MAX, VALIDATION_DECIMAL_MIN); + } - default void validate() - { - Class klass = getClass(); - while (klass != Object.class) { - for (Field field : klass.getDeclaredFields()) { - for (ValidationList.Validation validation : ValidationList.VALIDATIONS) { - Class annotationClass = validation.annotationClass; - if (field.isAnnotationPresent(annotationClass)) { - Annotation annotation = field.getAnnotation(annotationClass); - Object value; - try { - field.setAccessible(true); - value = field.get(this); - } - catch (IllegalAccessException e) { - throw new RuntimeException( - String.format("Failed to get a value from field (%s)", field), e); - } + default void validate() { + Class klass = getClass(); + while (klass != Object.class) { + for (Field field : klass.getDeclaredFields()) { + for (ValidationList.Validation validation : ValidationList.VALIDATIONS) { + Class annotationClass = validation.annotationClass; + if (field.isAnnotationPresent(annotationClass)) { + Annotation annotation = field.getAnnotation(annotationClass); + Object value; + try { + field.setAccessible(true); + value = field.get(this); + } catch (IllegalAccessException e) { + throw new RuntimeException( + String.format("Failed to get a value from field (%s)", field), e); + } - if (value == null) { - break; - } + if (value == null) { + break; + } - if (!(value instanceof Number)) { - throw new IllegalArgumentException( - String.format("This field has (%s), but actual field is (%s)", annotation, value.getClass())); - } + if (!(value instanceof Number)) { + throw new IllegalArgumentException( + String.format( + "This field has (%s), but actual field is (%s)", + annotation, value.getClass())); + } - if (!validation.isValid.apply(annotation, (Number) value)) { - throw new IllegalArgumentException( - String.format(validation.messageTemplate, field, value)); - } - } - } + if (!validation.isValid.apply(annotation, (Number) value)) { + throw new IllegalArgumentException( + String.format(validation.messageTemplate, field, value)); } - klass = klass.getSuperclass(); + } } + } + klass = klass.getSuperclass(); } + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/DecimalMax.java b/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/DecimalMax.java index 932f39bc..134e7a3b 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/DecimalMax.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/DecimalMax.java @@ -25,8 +25,8 @@ @Target(ElementType.FIELD) @Inherited @Retention(RetentionPolicy.RUNTIME) -public @interface DecimalMax -{ - String value(); - boolean inclusive() default true; +public @interface DecimalMax { + String value(); + + boolean inclusive() default true; } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/DecimalMin.java b/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/DecimalMin.java index fb0dfb11..9c1d23d5 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/DecimalMin.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/DecimalMin.java @@ -25,8 +25,8 @@ @Target(ElementType.FIELD) @Inherited @Retention(RetentionPolicy.RUNTIME) -public @interface DecimalMin -{ - String value(); - boolean inclusive() default true; +public @interface DecimalMin { + String value(); + + boolean inclusive() default true; } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/Max.java b/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/Max.java index 828339b2..173b14f3 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/Max.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/Max.java @@ -25,8 +25,8 @@ @Target(ElementType.FIELD) @Inherited @Retention(RetentionPolicy.RUNTIME) -public @interface Max -{ - long value(); - boolean inclusive() default true; +public @interface Max { + long value(); + + boolean inclusive() default true; } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/Min.java b/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/Min.java index dc078327..bdaba7c2 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/Min.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/Min.java @@ -25,8 +25,8 @@ @Target(ElementType.FIELD) @Inherited @Retention(RetentionPolicy.RUNTIME) -public @interface Min -{ - long value(); - boolean inclusive() default true; +public @interface Min { + long value(); + + boolean inclusive() default true; } diff --git a/fluency-core/src/test/java/org/komamitsu/fluency/FluencyTest.java b/fluency-core/src/test/java/org/komamitsu/fluency/FluencyTest.java index bebfe9d0..65666d9d 100644 --- a/fluency-core/src/test/java/org/komamitsu/fluency/FluencyTest.java +++ b/fluency-core/src/test/java/org/komamitsu/fluency/FluencyTest.java @@ -16,7 +16,21 @@ package org.komamitsu.fluency; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; + import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -28,185 +42,151 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; - -class FluencyTest -{ - private static final Logger LOG = LoggerFactory.getLogger(FluencyTest.class); - private Ingester ingester; - private Buffer.Config bufferConfig; - private Flusher.Config flusherConfig; - - @BeforeEach - void setUp() - { - ingester = mock(Ingester.class); - - bufferConfig = new Buffer.Config(); - flusherConfig = new Flusher.Config(); +class FluencyTest { + private static final Logger LOG = LoggerFactory.getLogger(FluencyTest.class); + private Ingester ingester; + private Buffer.Config bufferConfig; + private Flusher.Config flusherConfig; + + @BeforeEach + void setUp() { + ingester = mock(Ingester.class); + + bufferConfig = new Buffer.Config(); + flusherConfig = new Flusher.Config(); + } + + @Test + void testIsTerminated() throws IOException, InterruptedException { + Buffer buffer = new Buffer(bufferConfig, new JsonRecordFormatter()); + Flusher flusher = new Flusher(flusherConfig, buffer, ingester); + try (Fluency fluency = new Fluency(buffer, flusher)) { + assertFalse(fluency.isTerminated()); + fluency.close(); + TimeUnit.SECONDS.sleep(1); + assertTrue(fluency.isTerminated()); } - - @Test - void testIsTerminated() - throws IOException, InterruptedException - { - Buffer buffer = new Buffer(bufferConfig, new JsonRecordFormatter()); - Flusher flusher = new Flusher(flusherConfig, buffer, ingester); - try (Fluency fluency = new Fluency(buffer, flusher)) { - assertFalse(fluency.isTerminated()); - fluency.close(); - TimeUnit.SECONDS.sleep(1); - assertTrue(fluency.isTerminated()); - } - } - - @Test - void testGetAllocatedBufferSize() - throws IOException - { - Buffer.Config bufferConfig = new Buffer.Config(); - bufferConfig.setChunkInitialSize(1024); - Buffer buffer = new Buffer(bufferConfig, new JsonRecordFormatter()); - Flusher flusher = new Flusher(flusherConfig, buffer, ingester); - try (Fluency fluency = new Fluency(buffer, flusher)) { - assertThat(fluency.getAllocatedBufferSize()).isEqualTo(0L); - fluency.emit("foodb.bartbl", ImmutableMap.of("comment", "hello, world")); - assertThat(fluency.getAllocatedBufferSize()).isEqualTo(1024L); - } + } + + @Test + void testGetAllocatedBufferSize() throws IOException { + Buffer.Config bufferConfig = new Buffer.Config(); + bufferConfig.setChunkInitialSize(1024); + Buffer buffer = new Buffer(bufferConfig, new JsonRecordFormatter()); + Flusher flusher = new Flusher(flusherConfig, buffer, ingester); + try (Fluency fluency = new Fluency(buffer, flusher)) { + assertThat(fluency.getAllocatedBufferSize()).isEqualTo(0L); + fluency.emit("foodb.bartbl", ImmutableMap.of("comment", "hello, world")); + assertThat(fluency.getAllocatedBufferSize()).isEqualTo(1024L); } - - @ParameterizedTest - @CsvSource({"1, false", "3, true"}) - void testWaitUntilFlusherTerminated(int waitUntilFlusherTerm, boolean expected) - throws IOException, InterruptedException - { - flusherConfig.setWaitUntilTerminated(1); - - // Wait before actually closing in Buffer - int waitBeforeCloseMillis = 2000; - Buffer buffer = spy(new Buffer(bufferConfig, new JsonRecordFormatter())); - doAnswer((invocation) -> { - long start = System.currentTimeMillis(); - try { + } + + @ParameterizedTest + @CsvSource({"1, false", "3, true"}) + void testWaitUntilFlusherTerminated(int waitUntilFlusherTerm, boolean expected) + throws IOException, InterruptedException { + flusherConfig.setWaitUntilTerminated(1); + + // Wait before actually closing in Buffer + int waitBeforeCloseMillis = 2000; + Buffer buffer = spy(new Buffer(bufferConfig, new JsonRecordFormatter())); + doAnswer( + (invocation) -> { + long start = System.currentTimeMillis(); + try { TimeUnit.MILLISECONDS.sleep(waitBeforeCloseMillis); - } - catch (InterruptedException e) { + } catch (InterruptedException e) { long rest = waitBeforeCloseMillis - (System.currentTimeMillis() - start); if (rest > 0) { - try { - TimeUnit.MILLISECONDS.sleep(rest); - } - catch (InterruptedException e1) { - } + try { + TimeUnit.MILLISECONDS.sleep(rest); + } catch (InterruptedException e1) { + } } - } - return null; - }).doCallRealMethod().when(buffer).close(); + } + return null; + }) + .doCallRealMethod() + .when(buffer) + .close(); + + Flusher flusher = new Flusher(flusherConfig, buffer, ingester); + Fluency fluency = new Fluency(buffer, flusher); + + fluency.emit("foo.bar", new HashMap<>()); + fluency.close(); + assertThat(fluency.waitUntilFlusherTerminated(waitUntilFlusherTerm)).isEqualTo(expected); + } + + @ParameterizedTest + @CsvSource({"1, false", "3, true"}) + void testWaitUntilFlushingAllBuffer(int waitUntilFlusherTerm, boolean expected) + throws IOException, InterruptedException { + flusherConfig.setFlushAttemptIntervalMillis(2000); + + Buffer buffer = new Buffer(bufferConfig, new JsonRecordFormatter()); + Flusher flusher = new Flusher(flusherConfig, buffer, ingester); + try (Fluency fluency = new Fluency(buffer, flusher)) { + fluency.emit("foo.bar", new HashMap<>()); + assertThat(fluency.waitUntilAllBufferFlushed(waitUntilFlusherTerm)).isEqualTo(expected); + } + } + + @Test + public void testBufferFullException() throws IOException { + final CountDownLatch latch = new CountDownLatch(1); + StuckIngester stuckIngester = new StuckIngester(latch); + + bufferConfig.setChunkInitialSize(64); + bufferConfig.setChunkExpandRatio(2); + bufferConfig.setMaxBufferSize(256); + bufferConfig.setChunkRetentionSize(196); + flusherConfig.setFlushAttemptIntervalMillis(1000); + + Buffer buffer = new Buffer(bufferConfig, new JsonRecordFormatter()); + Flusher flusher = new Flusher(flusherConfig, buffer, stuckIngester); + try (Fluency fluency = new Fluency(buffer, flusher)) { + Map event = new HashMap<>(); + event.put("name", "xxxx"); // '{"name":"xxxx"}' (length: 15 bytes) + // Buffers: 64 + 128 = 192 + // 64 + 128 + 256 = 448 > 256 + // 15 * (8 + 1) = 135 + for (int i = 0; i < 8; i++) { + fluency.emit("tag", event); + } + try { + fluency.emit("tag", event); + fail(); + } catch (BufferFullException e) { + assertTrue(true); + } finally { + latch.countDown(); + } + } + } - Flusher flusher = new Flusher(flusherConfig, buffer, ingester); - Fluency fluency = new Fluency(buffer, flusher); + static class StuckIngester implements Ingester { + private final CountDownLatch latch; - fluency.emit("foo.bar", new HashMap<>()); - fluency.close(); - assertThat(fluency.waitUntilFlusherTerminated(waitUntilFlusherTerm)).isEqualTo(expected); + StuckIngester(CountDownLatch latch) { + this.latch = latch; } - @ParameterizedTest - @CsvSource({"1, false", "3, true"}) - void testWaitUntilFlushingAllBuffer(int waitUntilFlusherTerm, boolean expected) - throws IOException, InterruptedException - { - flusherConfig.setFlushAttemptIntervalMillis(2000); - - Buffer buffer = new Buffer(bufferConfig, new JsonRecordFormatter()); - Flusher flusher = new Flusher(flusherConfig, buffer, ingester); - try (Fluency fluency = new Fluency(buffer, flusher)) { - fluency.emit("foo.bar", new HashMap<>()); - assertThat(fluency.waitUntilAllBufferFlushed(waitUntilFlusherTerm)).isEqualTo(expected); - } + @Override + public void ingest(String tag, ByteBuffer dataBuffer) { + try { + latch.await(); + } catch (InterruptedException e) { + FluencyTest.LOG.warn("Interrupted in send()", e); + } } - @Test - public void testBufferFullException() - throws IOException - { - final CountDownLatch latch = new CountDownLatch(1); - StuckIngester stuckIngester = new StuckIngester(latch); - - bufferConfig.setChunkInitialSize(64); - bufferConfig.setChunkExpandRatio(2); - bufferConfig.setMaxBufferSize(256); - bufferConfig.setChunkRetentionSize(196); - flusherConfig.setFlushAttemptIntervalMillis(1000); - - Buffer buffer = new Buffer(bufferConfig, new JsonRecordFormatter()); - Flusher flusher = new Flusher(flusherConfig, buffer, stuckIngester); - try (Fluency fluency = new Fluency(buffer, flusher)) { - Map event = new HashMap<>(); - event.put("name", "xxxx"); // '{"name":"xxxx"}' (length: 15 bytes) - // Buffers: 64 + 128 = 192 - // 64 + 128 + 256 = 448 > 256 - // 15 * (8 + 1) = 135 - for (int i = 0; i < 8; i++) { - fluency.emit("tag", event); - } - try { - fluency.emit("tag", event); - fail(); - } - catch (BufferFullException e) { - assertTrue(true); - } - finally { - latch.countDown(); - } - } + @Override + public Sender getSender() { + return null; } - static class StuckIngester - implements Ingester - { - private final CountDownLatch latch; - - StuckIngester(CountDownLatch latch) - { - this.latch = latch; - } - - @Override - public void ingest(String tag, ByteBuffer dataBuffer) - { - try { - latch.await(); - } - catch (InterruptedException e) { - FluencyTest.LOG.warn("Interrupted in send()", e); - } - } - - @Override - public Sender getSender() - { - return null; - } - - @Override - public void close() - throws IOException - { - } - } + @Override + public void close() throws IOException {} + } } diff --git a/fluency-core/src/test/java/org/komamitsu/fluency/JsonRecordFormatter.java b/fluency-core/src/test/java/org/komamitsu/fluency/JsonRecordFormatter.java index a99047f9..64e77d17 100644 --- a/fluency-core/src/test/java/org/komamitsu/fluency/JsonRecordFormatter.java +++ b/fluency-core/src/test/java/org/komamitsu/fluency/JsonRecordFormatter.java @@ -18,47 +18,39 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import org.komamitsu.fluency.recordformat.AbstractRecordFormatter; - import java.nio.ByteBuffer; import java.util.Map; +import org.komamitsu.fluency.recordformat.AbstractRecordFormatter; -public class JsonRecordFormatter - extends AbstractRecordFormatter -{ - private ObjectMapper objectMapper = new ObjectMapper(); - - public JsonRecordFormatter() - { - super(new Config()); - } - - @Override - public byte[] format(String tag, Object timestamp, Map data) - { - try { - return objectMapper.writeValueAsBytes(data); - } - catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } +public class JsonRecordFormatter extends AbstractRecordFormatter { + private ObjectMapper objectMapper = new ObjectMapper(); - @Override - public byte[] formatFromMessagePack(String tag, Object timestamp, byte[] mapValue, int offset, int len) - { - throw new RuntimeException("Shouldn't be called"); - } - - @Override - public byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue) - { - throw new RuntimeException("Shouldn't be called"); - } + public JsonRecordFormatter() { + super(new Config()); + } - @Override - public String formatName() - { - return "json"; + @Override + public byte[] format(String tag, Object timestamp, Map data) { + try { + return objectMapper.writeValueAsBytes(data); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); } + } + + @Override + public byte[] formatFromMessagePack( + String tag, Object timestamp, byte[] mapValue, int offset, int len) { + throw new RuntimeException("Shouldn't be called"); + } + + @Override + public byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue) { + throw new RuntimeException("Shouldn't be called"); + } + + @Override + public String formatName() { + return "json"; + } } diff --git a/fluency-core/src/test/java/org/komamitsu/fluency/buffer/BufferPoolTest.java b/fluency-core/src/test/java/org/komamitsu/fluency/buffer/BufferPoolTest.java index 6dbd44b8..40207744 100644 --- a/fluency-core/src/test/java/org/komamitsu/fluency/buffer/BufferPoolTest.java +++ b/fluency-core/src/test/java/org/komamitsu/fluency/buffer/BufferPoolTest.java @@ -16,9 +16,11 @@ package org.komamitsu.fluency.buffer; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.nio.ByteBuffer; @@ -29,154 +31,144 @@ import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class BufferPoolTest -{ - private static final Logger LOG = LoggerFactory.getLogger(BufferPoolTest.class); - - @Test - void testAcquireAndRelease() - throws IOException - { - BufferPool bufferPool = new BufferPool(8 * 1024, 256 * 1024); - ByteBuffer buffer0 = bufferPool.acquireBuffer(100 * 1024); - assertEquals(128 * 1024, buffer0.capacity()); - assertEquals(0, buffer0.position()); - assertEquals(128 * 1024, bufferPool.getAllocatedSize()); - - ByteBuffer buffer1 = bufferPool.acquireBuffer(40 * 1024); - assertEquals(64 * 1024, buffer1.capacity()); - assertEquals(0, buffer1.position()); - assertEquals(192 * 1024, bufferPool.getAllocatedSize()); - - bufferPool.returnBuffer(buffer0); - assertEquals(192 * 1024, bufferPool.getAllocatedSize()); - - ByteBuffer buffer2 = bufferPool.acquireBuffer(20 * 1024); - assertEquals(32 * 1024, buffer2.capacity()); - assertEquals(0, buffer2.position()); - assertEquals(224 * 1024, bufferPool.getAllocatedSize()); - - bufferPool.returnBuffer(buffer2); - assertEquals(224 * 1024, bufferPool.getAllocatedSize()); - - bufferPool.returnBuffer(buffer1); - assertEquals(224 * 1024, bufferPool.getAllocatedSize()); - - long totalBufferSize = getActualTotalBufferSize(bufferPool); - assertEquals(224 * 1024, totalBufferSize); - - bufferPool.releaseBuffers(); - assertEquals(0, bufferPool.getAllocatedSize()); - totalBufferSize = getActualTotalBufferSize(bufferPool); - assertEquals(0, totalBufferSize); - } - - @Test - void testAcquireWithBufferFull() - { - BufferPool bufferPool = new BufferPool(8 * 1024, 256 * 1024); - ByteBuffer buffer = bufferPool.acquireBuffer(64 * 1024); - assertEquals(64 * 1024, buffer.capacity()); - assertEquals(0, buffer.position()); - assertEquals(64 * 1024, bufferPool.getAllocatedSize()); - - buffer = bufferPool.acquireBuffer(64 * 1024); - assertEquals(64 * 1024, buffer.capacity()); - assertEquals(0, buffer.position()); - assertEquals(128 * 1024, bufferPool.getAllocatedSize()); - - buffer = bufferPool.acquireBuffer(64 * 1024); - assertEquals(64 * 1024, buffer.capacity()); - assertEquals(0, buffer.position()); - assertEquals(192 * 1024, bufferPool.getAllocatedSize()); - - buffer = bufferPool.acquireBuffer(64 * 1024); - assertEquals(64 * 1024, buffer.capacity()); - assertEquals(0, buffer.position()); - assertEquals(256 * 1024, bufferPool.getAllocatedSize()); - - buffer = bufferPool.acquireBuffer(64 * 1024); - assertNull(buffer); - } - - @Test - void testAcquireAndReleaseWithMultiThread() - throws InterruptedException - { - final int concurrency = 8; - final BufferPool bufferPool = new BufferPool(1024, 512 * 1024); - final CountDownLatch countDownLatch = new CountDownLatch(concurrency); - Runnable task = () -> { - LinkedList buffers = new LinkedList<>(); - for (int i = 0; i < 1000; i++) { - if ((i / 10) % 2 == 0) { - int size = ((i % 2) + 1) * 1024; - ByteBuffer buffer = bufferPool.acquireBuffer(size); - assertEquals(size, buffer.capacity()); - assertEquals(0, buffer.position()); - buffers.add(buffer); - } - else { - ByteBuffer buffer = buffers.pollFirst(); - assertNotNull(buffer); - bufferPool.returnBuffer(buffer); - } +class BufferPoolTest { + private static final Logger LOG = LoggerFactory.getLogger(BufferPoolTest.class); + + @Test + void testAcquireAndRelease() throws IOException { + BufferPool bufferPool = new BufferPool(8 * 1024, 256 * 1024); + ByteBuffer buffer0 = bufferPool.acquireBuffer(100 * 1024); + assertEquals(128 * 1024, buffer0.capacity()); + assertEquals(0, buffer0.position()); + assertEquals(128 * 1024, bufferPool.getAllocatedSize()); + + ByteBuffer buffer1 = bufferPool.acquireBuffer(40 * 1024); + assertEquals(64 * 1024, buffer1.capacity()); + assertEquals(0, buffer1.position()); + assertEquals(192 * 1024, bufferPool.getAllocatedSize()); + + bufferPool.returnBuffer(buffer0); + assertEquals(192 * 1024, bufferPool.getAllocatedSize()); + + ByteBuffer buffer2 = bufferPool.acquireBuffer(20 * 1024); + assertEquals(32 * 1024, buffer2.capacity()); + assertEquals(0, buffer2.position()); + assertEquals(224 * 1024, bufferPool.getAllocatedSize()); + + bufferPool.returnBuffer(buffer2); + assertEquals(224 * 1024, bufferPool.getAllocatedSize()); + + bufferPool.returnBuffer(buffer1); + assertEquals(224 * 1024, bufferPool.getAllocatedSize()); + + long totalBufferSize = getActualTotalBufferSize(bufferPool); + assertEquals(224 * 1024, totalBufferSize); + + bufferPool.releaseBuffers(); + assertEquals(0, bufferPool.getAllocatedSize()); + totalBufferSize = getActualTotalBufferSize(bufferPool); + assertEquals(0, totalBufferSize); + } + + @Test + void testAcquireWithBufferFull() { + BufferPool bufferPool = new BufferPool(8 * 1024, 256 * 1024); + ByteBuffer buffer = bufferPool.acquireBuffer(64 * 1024); + assertEquals(64 * 1024, buffer.capacity()); + assertEquals(0, buffer.position()); + assertEquals(64 * 1024, bufferPool.getAllocatedSize()); + + buffer = bufferPool.acquireBuffer(64 * 1024); + assertEquals(64 * 1024, buffer.capacity()); + assertEquals(0, buffer.position()); + assertEquals(128 * 1024, bufferPool.getAllocatedSize()); + + buffer = bufferPool.acquireBuffer(64 * 1024); + assertEquals(64 * 1024, buffer.capacity()); + assertEquals(0, buffer.position()); + assertEquals(192 * 1024, bufferPool.getAllocatedSize()); + + buffer = bufferPool.acquireBuffer(64 * 1024); + assertEquals(64 * 1024, buffer.capacity()); + assertEquals(0, buffer.position()); + assertEquals(256 * 1024, bufferPool.getAllocatedSize()); + + buffer = bufferPool.acquireBuffer(64 * 1024); + assertNull(buffer); + } + + @Test + void testAcquireAndReleaseWithMultiThread() throws InterruptedException { + final int concurrency = 8; + final BufferPool bufferPool = new BufferPool(1024, 512 * 1024); + final CountDownLatch countDownLatch = new CountDownLatch(concurrency); + Runnable task = + () -> { + LinkedList buffers = new LinkedList<>(); + for (int i = 0; i < 1000; i++) { + if ((i / 10) % 2 == 0) { + int size = ((i % 2) + 1) * 1024; + ByteBuffer buffer = bufferPool.acquireBuffer(size); + assertEquals(size, buffer.capacity()); + assertEquals(0, buffer.position()); + buffers.add(buffer); + } else { + ByteBuffer buffer = buffers.pollFirst(); + assertNotNull(buffer); + bufferPool.returnBuffer(buffer); } - assertEquals(0, buffers.size()); - countDownLatch.countDown(); + } + assertEquals(0, buffers.size()); + countDownLatch.countDown(); }; - ExecutorService executorService = Executors.newCachedThreadPool(); - for (int i = 0; i < concurrency; i++) { - executorService.execute(task); - } - assertTrue(countDownLatch.await(10, TimeUnit.SECONDS)); - - long totalBufferSize = getActualTotalBufferSize(bufferPool); - LOG.debug("bufferPool.getAllocatedSize() = {}, actual buffered total size={}", bufferPool.getAllocatedSize(), totalBufferSize); - assertEquals(bufferPool.getAllocatedSize(), totalBufferSize); + ExecutorService executorService = Executors.newCachedThreadPool(); + for (int i = 0; i < concurrency; i++) { + executorService.execute(task); } - - private long getActualTotalBufferSize(BufferPool bufferPool) - { - long totalBufferedSize = 0; - for (Map.Entry> entry : bufferPool.bufferPool.entrySet()) { - for (ByteBuffer buffer : entry.getValue()) { - totalBufferedSize += buffer.capacity(); - } - } - return totalBufferedSize; - } - - @Test - void useOffHeap() - throws IOException - { - BufferPool bufferPool = new BufferPool(8 * 1024, 256 * 1024); - assertFalse(bufferPool.getJvmHeapBufferMode()); - ByteBuffer buffer0 = bufferPool.acquireBuffer(100 * 1024); - assertTrue(buffer0.isDirect()); - assertEquals(128 * 1024, buffer0.capacity()); - assertEquals(0, buffer0.position()); - assertEquals(128 * 1024, bufferPool.getAllocatedSize()); - } - - @Test - void useOnHeap() - throws IOException - { - BufferPool bufferPool = new BufferPool(8 * 1024, 256 * 1024, true); - assertTrue(bufferPool.getJvmHeapBufferMode()); - ByteBuffer buffer0 = bufferPool.acquireBuffer(100 * 1024); - assertFalse(buffer0.isDirect()); - assertEquals(128 * 1024, buffer0.capacity()); - assertEquals(0, buffer0.position()); - assertEquals(128 * 1024, bufferPool.getAllocatedSize()); + assertTrue(countDownLatch.await(10, TimeUnit.SECONDS)); + + long totalBufferSize = getActualTotalBufferSize(bufferPool); + LOG.debug( + "bufferPool.getAllocatedSize() = {}, actual buffered total size={}", + bufferPool.getAllocatedSize(), + totalBufferSize); + assertEquals(bufferPool.getAllocatedSize(), totalBufferSize); + } + + private long getActualTotalBufferSize(BufferPool bufferPool) { + long totalBufferedSize = 0; + for (Map.Entry> entry : + bufferPool.bufferPool.entrySet()) { + for (ByteBuffer buffer : entry.getValue()) { + totalBufferedSize += buffer.capacity(); + } } + return totalBufferedSize; + } + + @Test + void useOffHeap() throws IOException { + BufferPool bufferPool = new BufferPool(8 * 1024, 256 * 1024); + assertFalse(bufferPool.getJvmHeapBufferMode()); + ByteBuffer buffer0 = bufferPool.acquireBuffer(100 * 1024); + assertTrue(buffer0.isDirect()); + assertEquals(128 * 1024, buffer0.capacity()); + assertEquals(0, buffer0.position()); + assertEquals(128 * 1024, bufferPool.getAllocatedSize()); + } + + @Test + void useOnHeap() throws IOException { + BufferPool bufferPool = new BufferPool(8 * 1024, 256 * 1024, true); + assertTrue(bufferPool.getJvmHeapBufferMode()); + ByteBuffer buffer0 = bufferPool.acquireBuffer(100 * 1024); + assertFalse(buffer0.isDirect()); + assertEquals(128 * 1024, buffer0.capacity()); + assertEquals(0, buffer0.position()); + assertEquals(128 * 1024, bufferPool.getAllocatedSize()); + } } diff --git a/fluency-core/src/test/java/org/komamitsu/fluency/buffer/BufferTest.java b/fluency-core/src/test/java/org/komamitsu/fluency/buffer/BufferTest.java index ec02867c..72bab9fb 100644 --- a/fluency-core/src/test/java/org/komamitsu/fluency/buffer/BufferTest.java +++ b/fluency-core/src/test/java/org/komamitsu/fluency/buffer/BufferTest.java @@ -16,10 +16,26 @@ package org.komamitsu.fluency.buffer; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.komamitsu.fluency.JsonRecordFormatter; @@ -31,400 +47,374 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicReference; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.*; - -class BufferTest -{ - private static final Logger LOG = LoggerFactory.getLogger(BufferTest.class); - private Ingester ingester; - private RecordFormatter recordFormatter; - private Buffer.Config bufferConfig; - - @BeforeEach - void setUp() - { - recordFormatter = new JsonRecordFormatter(); - ingester = mock(Ingester.class); - bufferConfig = new Buffer.Config(); +class BufferTest { + private static final Logger LOG = LoggerFactory.getLogger(BufferTest.class); + private Ingester ingester; + private RecordFormatter recordFormatter; + private Buffer.Config bufferConfig; + + @BeforeEach + void setUp() { + recordFormatter = new JsonRecordFormatter(); + ingester = mock(Ingester.class); + bufferConfig = new Buffer.Config(); + } + + @Test + void testBuffer() throws IOException { + int recordSize = 20; // '{"name":"komamitsu"}' + int tags = 10; + float allocRatio = 2f; + int maxAllocSize = (recordSize * 4) * tags; + + bufferConfig.setChunkInitialSize(recordSize); + bufferConfig.setChunkExpandRatio(allocRatio); + bufferConfig.setMaxBufferSize(maxAllocSize); + bufferConfig.setChunkRetentionSize(maxAllocSize / 2); + Buffer buffer = new Buffer(bufferConfig, recordFormatter); + + assertEquals(0, buffer.getAllocatedSize()); + assertEquals(0, buffer.getBufferedDataSize()); + assertEquals(0f, buffer.getBufferUsage(), 0.001); + + Map data = new HashMap<>(); + data.put("name", "komamitsu"); + for (int i = 0; i < tags; i++) { + buffer.append("foodb.bartbl" + i, 1420070400, data); } - - @Test - void testBuffer() - throws IOException - { - int recordSize = 20; // '{"name":"komamitsu"}' - int tags = 10; - float allocRatio = 2f; - int maxAllocSize = (recordSize * 4) * tags; - - bufferConfig.setChunkInitialSize(recordSize); - bufferConfig.setChunkExpandRatio(allocRatio); - bufferConfig.setMaxBufferSize(maxAllocSize); - bufferConfig.setChunkRetentionSize(maxAllocSize / 2); - Buffer buffer = new Buffer(bufferConfig, recordFormatter); - - assertEquals(0, buffer.getAllocatedSize()); - assertEquals(0, buffer.getBufferedDataSize()); - assertEquals(0f, buffer.getBufferUsage(), 0.001); - - Map data = new HashMap<>(); - data.put("name", "komamitsu"); - for (int i = 0; i < tags; i++) { - buffer.append("foodb.bartbl" + i, 1420070400, data); - } - assertEquals(recordSize * tags, buffer.getAllocatedSize()); - assertEquals(recordSize * tags, buffer.getBufferedDataSize()); - assertEquals(((float) recordSize) * tags / maxAllocSize, buffer.getBufferUsage(), 0.001); - for (int i = 0; i < tags; i++) { - buffer.append("foodb.bartbl" + i, 1420070400, data); - } - float totalAllocatedSize = recordSize * (1 + allocRatio) * tags; - assertEquals(totalAllocatedSize, buffer.getAllocatedSize(), 0.001); - assertEquals(2L * recordSize * tags, buffer.getBufferedDataSize()); - assertEquals(totalAllocatedSize / maxAllocSize, buffer.getBufferUsage(), 0.001); - - buffer.flush(ingester, true); - assertEquals(totalAllocatedSize, buffer.getAllocatedSize(), 0.001); - assertEquals(0, buffer.getBufferedDataSize()); - assertEquals(totalAllocatedSize / maxAllocSize, buffer.getBufferUsage(), 0.001); - - buffer.close(); - assertEquals(0, buffer.getAllocatedSize()); - assertEquals(0, buffer.getBufferedDataSize()); - assertEquals(0, buffer.getBufferUsage(), 0.001); + assertEquals(recordSize * tags, buffer.getAllocatedSize()); + assertEquals(recordSize * tags, buffer.getBufferedDataSize()); + assertEquals(((float) recordSize) * tags / maxAllocSize, buffer.getBufferUsage(), 0.001); + for (int i = 0; i < tags; i++) { + buffer.append("foodb.bartbl" + i, 1420070400, data); + } + float totalAllocatedSize = recordSize * (1 + allocRatio) * tags; + assertEquals(totalAllocatedSize, buffer.getAllocatedSize(), 0.001); + assertEquals(2L * recordSize * tags, buffer.getBufferedDataSize()); + assertEquals(totalAllocatedSize / maxAllocSize, buffer.getBufferUsage(), 0.001); + + buffer.flush(ingester, true); + assertEquals(totalAllocatedSize, buffer.getAllocatedSize(), 0.001); + assertEquals(0, buffer.getBufferedDataSize()); + assertEquals(totalAllocatedSize / maxAllocSize, buffer.getBufferUsage(), 0.001); + + buffer.close(); + assertEquals(0, buffer.getAllocatedSize()); + assertEquals(0, buffer.getBufferedDataSize()); + assertEquals(0, buffer.getBufferUsage(), 0.001); + } + + @Test + void testStrictBufferRetentionSizeManagement() throws IOException { + int recordSize = 20; // '{"name":"komamitsu"}' + int initSize = 15; + float allocRatio = 2f; + // 15 -> 30 -> 60 -> 120 -> 240 + int maxAllocSize = (int) (initSize * allocRatio * allocRatio * allocRatio * allocRatio); + String tag = "foodb.bartbl"; + + int retentionSize = 60; + + bufferConfig.setChunkInitialSize(initSize); + bufferConfig.setChunkExpandRatio(allocRatio); + bufferConfig.setMaxBufferSize(maxAllocSize); + bufferConfig.setChunkRetentionSize(retentionSize); + Buffer buffer = new Buffer(bufferConfig, recordFormatter); + + Map data = new HashMap<>(); + data.put("name", "komamitsu"); + + // Buffered data size is 0 and additional 20 bytes data will be buffered (total size: 20 bytes). + buffer.append(tag, 1420070400, data); + buffer.flush(ingester, false); + verify(ingester, times(0)).ingest(anyString(), any(ByteBuffer.class)); + assertEquals(20, buffer.getBufferedDataSize()); + + // Buffered data size is 20 and additional 20 bytes data will be buffered (total size: 40 + // bytes). + buffer.append(tag, 1420070400, data); + buffer.flush(ingester, false); + verify(ingester, times(0)).ingest(anyString(), any(ByteBuffer.class)); + assertEquals(40, buffer.getBufferedDataSize()); + + // Buffered data size is 40 and additional 20 bytes data will be buffered (total size: 60 + // bytes). + buffer.append(tag, 1420070400, data); + buffer.flush(ingester, false); + verify(ingester, times(0)).ingest(anyString(), any(ByteBuffer.class)); + assertEquals(60, buffer.getBufferedDataSize()); + + // Buffered data size is 60 and additional 20 bytes exceeds the retention size. + // The buffered 60 bytes data will be flushed and the additional data will be buffered (total + // size: 20 bytes). + doAnswer( + invocation -> { + String receivedTag = invocation.getArgument(0); + ByteBuffer receivedBuffer = invocation.getArgument(1); + assertEquals(tag, receivedTag); + assertEquals(60, receivedBuffer.remaining()); + return null; + }) + .when(ingester) + .ingest(anyString(), any(ByteBuffer.class)); + + buffer.append(tag, 1420070400, data); + buffer.flush(ingester, false); + verify(ingester, times(1)).ingest(anyString(), any(ByteBuffer.class)); + assertEquals(20, buffer.getBufferedDataSize()); + } + + @Test + void flush() throws IOException { + Buffer buffer = new Buffer(bufferConfig, recordFormatter); + + Ingester ingester = mock(Ingester.class); + buffer.flush(ingester, false); + + verify(ingester, times(0)).ingest(anyString(), any(ByteBuffer.class)); + + Map data = new HashMap<>(); + data.put("name", "komamitsu"); + buffer.append("foodb.bartbl", 1420070400, data); + + buffer.flush(ingester, false); + + verify(ingester, times(0)).ingest(anyString(), any(ByteBuffer.class)); + + buffer.flush(ingester, true); + + verify(ingester, times(1)).ingest(eq("foodb.bartbl"), any(ByteBuffer.class)); + } + + @Test + void flushWithInterrupt() throws IOException { + Buffer buffer = new Buffer(bufferConfig, recordFormatter); + + Ingester ingester = mock(Ingester.class); + AtomicReference receivedBuffer = new AtomicReference<>(); + doAnswer( + (Answer) + invocation -> { + receivedBuffer.set(((ByteBuffer) invocation.getArgument(1)).duplicate()); + return null; + }) + .when(ingester) + .ingest(anyString(), any(ByteBuffer.class)); + + Map data = new HashMap<>(); + data.put("name", "komamitsu"); + buffer.append("foodb.bartbl", 1420070400, data); + + try { + Thread.currentThread().interrupt(); + buffer.flush(ingester, true); + fail(); + } catch (IOException e) { + // Caused by interruption. This is expected. } - @Test - void testStrictBufferRetentionSizeManagement() - throws IOException - { - int recordSize = 20; // '{"name":"komamitsu"}' - int initSize = 15; - float allocRatio = 2f; - // 15 -> 30 -> 60 -> 120 -> 240 - int maxAllocSize = (int) (initSize * allocRatio * allocRatio * allocRatio * allocRatio); - String tag = "foodb.bartbl"; - - int retentionSize = 60; - - bufferConfig.setChunkInitialSize(initSize); - bufferConfig.setChunkExpandRatio(allocRatio); - bufferConfig.setMaxBufferSize(maxAllocSize); - bufferConfig.setChunkRetentionSize(retentionSize); - Buffer buffer = new Buffer(bufferConfig, recordFormatter); - - Map data = new HashMap<>(); - data.put("name", "komamitsu"); - - // Buffered data size is 0 and additional 20 bytes data will be buffered (total size: 20 bytes). - buffer.append(tag, 1420070400, data); - buffer.flush(ingester, false); - verify(ingester, times(0)).ingest(anyString(), any(ByteBuffer.class)); - assertEquals(20, buffer.getBufferedDataSize()); - - // Buffered data size is 20 and additional 20 bytes data will be buffered (total size: 40 bytes). - buffer.append(tag, 1420070400, data); - buffer.flush(ingester, false); - verify(ingester, times(0)).ingest(anyString(), any(ByteBuffer.class)); - assertEquals(40, buffer.getBufferedDataSize()); - - // Buffered data size is 40 and additional 20 bytes data will be buffered (total size: 60 bytes). - buffer.append(tag, 1420070400, data); - buffer.flush(ingester, false); - verify(ingester, times(0)).ingest(anyString(), any(ByteBuffer.class)); - assertEquals(60, buffer.getBufferedDataSize()); - - // Buffered data size is 60 and additional 20 bytes exceeds the retention size. - // The buffered 60 bytes data will be flushed and the additional data will be buffered (total size: 20 bytes). - doAnswer(invocation -> { - String receivedTag = invocation.getArgument(0); - ByteBuffer receivedBuffer = invocation.getArgument(1); - assertEquals(tag, receivedTag); - assertEquals(60, receivedBuffer.remaining()); - return null; - }).when(ingester).ingest(anyString(), any(ByteBuffer.class)); - - buffer.append(tag, 1420070400, data); - buffer.flush(ingester, false); - verify(ingester, times(1)).ingest(anyString(), any(ByteBuffer.class)); - assertEquals(20, buffer.getBufferedDataSize()); + // Retry + buffer.flush(ingester, true); + + verify(ingester, times(1)).ingest(eq("foodb.bartbl"), any(ByteBuffer.class)); + ObjectMapper objectMapper = new ObjectMapper(); + byte[] receivedData = new byte[receivedBuffer.get().remaining()]; + receivedBuffer.get().get(receivedData); + Map deserialized = + objectMapper.readValue(receivedData, new TypeReference>() {}); + System.out.println(deserialized); + } + + @Test + void testFileBackup() throws IOException { + bufferConfig.setFileBackupDir(System.getProperty("java.io.tmpdir")); + bufferConfig.setFileBackupPrefix("FileBackupTest"); + + // Just for cleaning backup files + try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { + buffer.clearBackupFiles(); + } + try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { + assertEquals(0, buffer.getBufferedDataSize()); } - @Test - void flush() - throws IOException - { - Buffer buffer = new Buffer(bufferConfig, recordFormatter); + long currentTime = System.currentTimeMillis() / 1000; + Map event0 = ImmutableMap.of("name", "a", "age", 42); + Map event1 = ImmutableMap.of("name", "b", "age", 99); + try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { + buffer.append("foo", currentTime, event0); + buffer.append("bar", currentTime, event1); + } - Ingester ingester = mock(Ingester.class); - buffer.flush(ingester, false); + Ingester ingester = mock(Ingester.class); + try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { + buffer.flushInternal(ingester, true); + } - verify(ingester, times(0)).ingest(anyString(), any(ByteBuffer.class)); + ObjectMapper objectMapper = new ObjectMapper(); + for (Tuple> tagAndEvent : + ImmutableList.of(new Tuple<>("foo", event0), new Tuple<>("bar", event1))) { + ArgumentCaptor byteBufferArgumentCaptor = + ArgumentCaptor.forClass(ByteBuffer.class); + verify(ingester, times(1)) + .ingest(eq(tagAndEvent.getFirst()), byteBufferArgumentCaptor.capture()); + ByteBuffer byteBuffer = byteBufferArgumentCaptor.getValue(); + byte[] bytes = new byte[byteBuffer.remaining()]; + byteBuffer.get(bytes); + Map map = + objectMapper.readValue(bytes, new TypeReference>() {}); + assertEquals(tagAndEvent.getSecond(), map); + } + } - Map data = new HashMap<>(); - data.put("name", "komamitsu"); - buffer.append("foodb.bartbl", 1420070400, data); + @Test + void testFileBackupThatIsNotDirectory() throws IOException { + File backupDirFile = File.createTempFile("testFileBackupWithInvalidDir", ".tmp"); + backupDirFile.deleteOnExit(); - buffer.flush(ingester, false); + bufferConfig.setFileBackupDir(backupDirFile.getAbsolutePath()); - verify(ingester, times(0)).ingest(anyString(), any(ByteBuffer.class)); + try (Buffer ignored = new Buffer(bufferConfig, recordFormatter)) { + fail(); + } catch (IllegalArgumentException e) { + assertTrue(true); + } + } + + @Test + void testFileBackupThatIsWritable() { + File backupDir = + new File( + System.getProperties().getProperty("java.io.tmpdir"), UUID.randomUUID().toString()); + if (backupDir.mkdir()) { + LOG.info("Created directory: {}", backupDir); + } + if (!backupDir.setWritable(false)) { + throw new RuntimeException("Failed to revoke writable permission"); + } + backupDir.deleteOnExit(); - buffer.flush(ingester, true); + bufferConfig.setFileBackupDir(backupDir.getAbsolutePath()); - verify(ingester, times(1)).ingest(eq("foodb.bartbl"), any(ByteBuffer.class)); + try (Buffer ignored = new Buffer(bufferConfig, recordFormatter)) { + fail(); + } catch (IllegalArgumentException e) { + assertTrue(true); } - - @Test - void flushWithInterrupt() - throws IOException - { - Buffer buffer = new Buffer(bufferConfig, recordFormatter); - - Ingester ingester = mock(Ingester.class); - AtomicReference receivedBuffer = new AtomicReference<>(); - doAnswer((Answer) invocation -> { - receivedBuffer.set(((ByteBuffer)invocation.getArgument(1)).duplicate()); - return null; - }).when(ingester).ingest(anyString(), any(ByteBuffer.class)); - - Map data = new HashMap<>(); - data.put("name", "komamitsu"); - buffer.append("foodb.bartbl", 1420070400, data); - - try { - Thread.currentThread().interrupt(); - buffer.flush(ingester, true); - fail(); - } - catch (IOException e) { - // Caused by interruption. This is expected. - } - - // Retry - buffer.flush(ingester, true); - - verify(ingester, times(1)).ingest(eq("foodb.bartbl"), any(ByteBuffer.class)); - ObjectMapper objectMapper = new ObjectMapper(); - byte[] receivedData = new byte[receivedBuffer.get().remaining()]; - receivedBuffer.get().get(receivedData); - Map deserialized = objectMapper.readValue(receivedData, new TypeReference>() {}); - System.out.println(deserialized); + } + + @Test + void testFileBackupThatIsReadable() { + File backupDir = + new File( + System.getProperties().getProperty("java.io.tmpdir"), UUID.randomUUID().toString()); + if (backupDir.mkdir()) { + LOG.info("Created directory: {}", backupDir); } - - @Test - void testFileBackup() - throws IOException - { - bufferConfig.setFileBackupDir(System.getProperty("java.io.tmpdir")); - bufferConfig.setFileBackupPrefix("FileBackupTest"); - - // Just for cleaning backup files - try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { - buffer.clearBackupFiles(); - } - try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { - assertEquals(0, buffer.getBufferedDataSize()); - } - - long currentTime = System.currentTimeMillis() / 1000; - Map event0 = ImmutableMap.of("name", "a", "age", 42); - Map event1 = ImmutableMap.of("name", "b", "age", 99); - try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { - buffer.append("foo", currentTime, event0); - buffer.append("bar", currentTime, event1); - } - - Ingester ingester = mock(Ingester.class); - try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { - buffer.flushInternal(ingester, true); - } - - ObjectMapper objectMapper = new ObjectMapper(); - for (Tuple> tagAndEvent : - ImmutableList.of(new Tuple<>("foo", event0), new Tuple<>("bar", event1))) { - ArgumentCaptor byteBufferArgumentCaptor = ArgumentCaptor.forClass(ByteBuffer.class); - verify(ingester, times(1)).ingest(eq(tagAndEvent.getFirst()), byteBufferArgumentCaptor.capture()); - ByteBuffer byteBuffer = byteBufferArgumentCaptor.getValue(); - byte[] bytes = new byte[byteBuffer.remaining()]; - byteBuffer.get(bytes); - Map map = objectMapper.readValue(bytes, new TypeReference>() {}); - assertEquals(tagAndEvent.getSecond(), map); - } + if (!backupDir.setReadable(false)) { + throw new RuntimeException("Failed to revoke readable permission"); } + backupDir.deleteOnExit(); - @Test - void testFileBackupThatIsNotDirectory() - throws IOException - { - File backupDirFile = File.createTempFile("testFileBackupWithInvalidDir", ".tmp"); - backupDirFile.deleteOnExit(); - - bufferConfig.setFileBackupDir(backupDirFile.getAbsolutePath()); + bufferConfig.setFileBackupDir(backupDir.getAbsolutePath()); - try (Buffer ignored = new Buffer(bufferConfig, recordFormatter)) { - fail(); - } - catch (IllegalArgumentException e) { - assertTrue(true); - } + try (Buffer ignored = new Buffer(bufferConfig, recordFormatter)) { + fail(); + } catch (IllegalArgumentException e) { + assertTrue(true); } - - @Test - void testFileBackupThatIsWritable() - { - File backupDir = new File(System.getProperties().getProperty("java.io.tmpdir"), UUID.randomUUID().toString()); - if (backupDir.mkdir()) { - LOG.info("Created directory: {}", backupDir); - } - if (!backupDir.setWritable(false)) { - throw new RuntimeException("Failed to revoke writable permission"); - } - backupDir.deleteOnExit(); - - bufferConfig.setFileBackupDir(backupDir.getAbsolutePath()); - - try (Buffer ignored = new Buffer(bufferConfig, recordFormatter)) { - fail(); - } - catch (IllegalArgumentException e) { - assertTrue(true); - } + } + + @Test + void testGetAllocatedSize() throws IOException { + bufferConfig.setChunkInitialSize(256 * 1024); + try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { + assertThat(buffer.getAllocatedSize()).isEqualTo(0L); + Map map = new HashMap<>(); + map.put("name", "komamitsu"); + for (int i = 0; i < 10; i++) { + buffer.append("foo.bar", new Date().getTime(), map); + } + assertThat(buffer.getAllocatedSize()).isEqualTo(256 * 1024L); } - - @Test - void testFileBackupThatIsReadable() - { - File backupDir = new File(System.getProperties().getProperty("java.io.tmpdir"), UUID.randomUUID().toString()); - if (backupDir.mkdir()) { - LOG.info("Created directory: {}", backupDir); - } - if (!backupDir.setReadable(false)) { - throw new RuntimeException("Failed to revoke readable permission"); - } - backupDir.deleteOnExit(); - - bufferConfig.setFileBackupDir(backupDir.getAbsolutePath()); - - try (Buffer ignored = new Buffer(bufferConfig, recordFormatter)) { - fail(); - } - catch (IllegalArgumentException e) { - assertTrue(true); - } + } + + @Test + void testGetBufferedDataSize() throws IOException { + bufferConfig.setChunkInitialSize(256 * 1024); + try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { + assertThat(buffer.getBufferedDataSize()).isEqualTo(0L); + + Map map = new HashMap<>(); + map.put("name", "komamitsu"); + for (int i = 0; i < 10; i++) { + buffer.append("foo.bar", new Date().getTime(), map); + } + assertThat(buffer.getBufferedDataSize()).isGreaterThan(0L); + assertThat(buffer.getBufferedDataSize()).isLessThan(512L); + + buffer.flush(ingester, true); + assertThat(buffer.getBufferedDataSize()).isEqualTo(0L); + } + } + + @Test + void testAppendIfItDoesNotThrowBufferOverflow() throws IOException { + bufferConfig.setChunkInitialSize(64 * 1024); + try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { + + StringBuilder buf = new StringBuilder(); + + for (int i = 0; i < 1024 * 60; i++) { + buf.append('x'); + } + String str60kb = buf.toString(); + + for (int i = 0; i < 1024 * 40; i++) { + buf.append('x'); + } + String str100kb = buf.toString(); + + { + Map map = new HashMap<>(); + map.put("k", str60kb); + buffer.append("tag0", new Date().getTime(), map); + } + + { + Map map = new HashMap<>(); + map.put("k", str100kb); + buffer.append("tag0", new Date().getTime(), map); + } } + } - @Test - void testGetAllocatedSize() - throws IOException + @Test + void validateConfig() { { - bufferConfig.setChunkInitialSize(256 * 1024); - try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { - assertThat(buffer.getAllocatedSize()).isEqualTo(0L); - Map map = new HashMap<>(); - map.put("name", "komamitsu"); - for (int i = 0; i < 10; i++) { - buffer.append("foo.bar", new Date().getTime(), map); - } - assertThat(buffer.getAllocatedSize()).isEqualTo(256 * 1024L); - } + Buffer.Config config = new Buffer.Config(); + config.setChunkInitialSize(4 * 1024 * 1024); + config.setChunkRetentionSize(4 * 1024 * 1024); + assertThrows(IllegalArgumentException.class, () -> new Buffer(config, recordFormatter)); } - @Test - void testGetBufferedDataSize() - throws IOException { - bufferConfig.setChunkInitialSize(256 * 1024); - try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { - assertThat(buffer.getBufferedDataSize()).isEqualTo(0L); - - Map map = new HashMap<>(); - map.put("name", "komamitsu"); - for (int i = 0; i < 10; i++) { - buffer.append("foo.bar", new Date().getTime(), map); - } - assertThat(buffer.getBufferedDataSize()).isGreaterThan(0L); - assertThat(buffer.getBufferedDataSize()).isLessThan(512L); - - buffer.flush(ingester, true); - assertThat(buffer.getBufferedDataSize()).isEqualTo(0L); - } + Buffer.Config config = new Buffer.Config(); + config.setChunkRetentionSize(64 * 1024 * 1024); + config.setMaxBufferSize(64 * 1024 * 1024); + assertThrows(IllegalArgumentException.class, () -> new Buffer(config, recordFormatter)); } - @Test - void testAppendIfItDoesNotThrowBufferOverflow() - throws IOException { - bufferConfig.setChunkInitialSize(64 * 1024); - try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { - - StringBuilder buf = new StringBuilder(); - - for (int i = 0; i < 1024 * 60; i++) { - buf.append('x'); - } - String str60kb = buf.toString(); - - for (int i = 0; i < 1024 * 40; i++) { - buf.append('x'); - } - String str100kb = buf.toString(); - - { - Map map = new HashMap<>(); - map.put("k", str60kb); - buffer.append("tag0", new Date().getTime(), map); - } - - { - Map map = new HashMap<>(); - map.put("k", str100kb); - buffer.append("tag0", new Date().getTime(), map); - } - } + Buffer.Config config = new Buffer.Config(); + config.setChunkExpandRatio(1.19f); + assertThrows(IllegalArgumentException.class, () -> new Buffer(config, recordFormatter)); } - @Test - void validateConfig() { - { - Buffer.Config config = new Buffer.Config(); - config.setChunkInitialSize(4 * 1024 * 1024); - config.setChunkRetentionSize(4 * 1024 * 1024); - assertThrows(IllegalArgumentException.class, () -> new Buffer(config, recordFormatter)); - } - - { - Buffer.Config config = new Buffer.Config(); - config.setChunkRetentionSize(64 * 1024 * 1024); - config.setMaxBufferSize(64 * 1024 * 1024); - assertThrows(IllegalArgumentException.class, () -> new Buffer(config, recordFormatter)); - } - - { - Buffer.Config config = new Buffer.Config(); - config.setChunkExpandRatio(1.19f); - assertThrows(IllegalArgumentException.class, () -> new Buffer(config, recordFormatter)); - } - - { - Buffer.Config config = new Buffer.Config(); - config.setChunkRetentionTimeMillis(49); - assertThrows(IllegalArgumentException.class, () -> new Buffer(config, recordFormatter)); - } + Buffer.Config config = new Buffer.Config(); + config.setChunkRetentionTimeMillis(49); + assertThrows(IllegalArgumentException.class, () -> new Buffer(config, recordFormatter)); } + } } diff --git a/fluency-core/src/test/java/org/komamitsu/fluency/buffer/FileBackupTest.java b/fluency-core/src/test/java/org/komamitsu/fluency/buffer/FileBackupTest.java index 02261888..f4b88381 100644 --- a/fluency-core/src/test/java/org/komamitsu/fluency/buffer/FileBackupTest.java +++ b/fluency-core/src/test/java/org/komamitsu/fluency/buffer/FileBackupTest.java @@ -1,6 +1,8 @@ package org.komamitsu.fluency.buffer; -import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; import java.io.File; import java.io.IOException; @@ -13,10 +15,7 @@ import java.util.List; import java.util.Objects; import java.util.stream.Collectors; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; +import org.junit.jupiter.api.Test; class FileBackupTest { @@ -26,29 +25,36 @@ private void createTempFile(File dir, String filename, String content) throws IO tempfilePath.toFile().deleteOnExit(); } - private void assertSavedBuffer(FileBackup.SavedBuffer savedBuffer, Path expectedPath, byte[] expectedContent, String... expectedParams) { + private void assertSavedBuffer( + FileBackup.SavedBuffer savedBuffer, + Path expectedPath, + byte[] expectedContent, + String... expectedParams) { assertThat(savedBuffer.getPath()).isEqualTo(expectedPath); - savedBuffer.open((params, channel) -> { - assertThat(params.toArray()).isEqualTo(expectedParams); - try { - long size = channel.size(); - ByteBuffer buf = ByteBuffer.allocate((int) size); - channel.read(buf); - assertThat(buf.array()).isEqualTo(expectedContent); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + savedBuffer.open( + (params, channel) -> { + assertThat(params.toArray()).isEqualTo(expectedParams); + try { + long size = channel.size(); + ByteBuffer buf = ByteBuffer.allocate((int) size); + channel.read(buf); + assertThat(buf.array()).isEqualTo(expectedContent); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); } - private void assertSavedFile(File savedFile, - String bufferFormatType, - String prefix, - long startNanos, - long endNanos, - String param1, - String param2, - byte[] expectedContent) throws IOException { + private void assertSavedFile( + File savedFile, + String bufferFormatType, + String prefix, + long startNanos, + long endNanos, + String param1, + String param2, + byte[] expectedContent) + throws IOException { String fileName = savedFile.toPath().getFileName().toString(); assertThat(fileName).endsWith(".buf"); @@ -78,37 +84,48 @@ void getSavedFiles_GivenSomeFiles_ShouldReturnThem() throws IOException { long nanoSeconds3 = System.nanoTime(); File backupDir = Files.createTempDirectory("test").toFile(); backupDir.deleteOnExit(); - createTempFile(backupDir, + createTempFile( + backupDir, String.format("xmy_buf_type_my_prefix#param_a#param_b#%d.buf", System.nanoTime()), "ignored"); - createTempFile(backupDir, + createTempFile( + backupDir, String.format("xmy_buf_type_my_prefix#param_a#param_b#%d.buf", System.nanoTime()), "ignored"); - createTempFile(backupDir, + createTempFile( + backupDir, String.format("y_buf_type_my_prefix#param_a#param_b#%d.buf", System.nanoTime()), "ignored"); - createTempFile(backupDir, + createTempFile( + backupDir, String.format("my_buf_type_my_prefix#1paramA#1paramB#%d.buf", nanoSeconds1), "content1"); - createTempFile(backupDir, + createTempFile( + backupDir, String.format("my_buf_type_my_prefix#2param-a#2param-b#%d.buf", nanoSeconds2), "content2"); - createTempFile(backupDir, + createTempFile( + backupDir, String.format("my_buf_type_my_prefix#3param_a#3param_b#%d.buf", nanoSeconds3), "content3"); - createTempFile(backupDir, + createTempFile( + backupDir, String.format("my_buf_type_my_prefixz#param_a#param_b#%d.buf", System.nanoTime()), "ignored"); - createTempFile(backupDir, + createTempFile( + backupDir, String.format("my_buf_type_my_prefi#param_a#param_b#%d.buf", System.nanoTime()), "ignored"); - createTempFile(backupDir, + createTempFile( + backupDir, String.format("my_buf_type_my_prefix#param:a#param:b#%d.buf", System.nanoTime()), "ignored"); - createTempFile(backupDir, + createTempFile( + backupDir, String.format("my_buf_type_my_prefix#param_a#param_b#%d", System.nanoTime()), "ignored"); - createTempFile(backupDir, + createTempFile( + backupDir, String.format("my_buf_type_my_prefix#param_a#param_b#%d.buff", System.nanoTime()), "ignored"); Buffer buffer = mock(Buffer.class); @@ -116,22 +133,33 @@ void getSavedFiles_GivenSomeFiles_ShouldReturnThem() throws IOException { String prefix = "my_prefix"; FileBackup fileBackup = new FileBackup(backupDir, buffer, prefix); - List savedFiles = fileBackup.getSavedFiles().stream().sorted( - Comparator.comparing(FileBackup.SavedBuffer::getPath)).collect(Collectors.toList()); + List savedFiles = + fileBackup.getSavedFiles().stream() + .sorted(Comparator.comparing(FileBackup.SavedBuffer::getPath)) + .collect(Collectors.toList()); System.out.println(savedFiles); assertThat(savedFiles).size().isEqualTo(3); - assertSavedBuffer(savedFiles.get(0), - backupDir.toPath().resolve(String.format("my_buf_type_my_prefix#1paramA#1paramB#%d.buf", nanoSeconds1)), + assertSavedBuffer( + savedFiles.get(0), + backupDir + .toPath() + .resolve(String.format("my_buf_type_my_prefix#1paramA#1paramB#%d.buf", nanoSeconds1)), "content1".getBytes(StandardCharsets.UTF_8), "1paramA", "1paramB"); - assertSavedBuffer(savedFiles.get(1), - backupDir.toPath().resolve(String.format("my_buf_type_my_prefix#2param-a#2param-b#%d.buf", nanoSeconds2)), + assertSavedBuffer( + savedFiles.get(1), + backupDir + .toPath() + .resolve(String.format("my_buf_type_my_prefix#2param-a#2param-b#%d.buf", nanoSeconds2)), "content2".getBytes(StandardCharsets.UTF_8), "2param-a", "2param-b"); - assertSavedBuffer(savedFiles.get(2), - backupDir.toPath().resolve(String.format("my_buf_type_my_prefix#3param_a#3param_b#%d.buf", nanoSeconds3)), + assertSavedBuffer( + savedFiles.get(2), + backupDir + .toPath() + .resolve(String.format("my_buf_type_my_prefix#3param_a#3param_b#%d.buf", nanoSeconds3)), "content3".getBytes(StandardCharsets.UTF_8), "3param_a", "3param_b"); @@ -157,11 +185,13 @@ void saveBuffer() throws IOException { ByteBuffer.wrap("content3".getBytes(StandardCharsets.UTF_8))); long endNanos = System.nanoTime(); - List savedFiles = Arrays.stream(Objects.requireNonNull(backupDir.listFiles())) - .sorted(Comparator.comparing(File::toString)) - .collect(Collectors.toList()); + List savedFiles = + Arrays.stream(Objects.requireNonNull(backupDir.listFiles())) + .sorted(Comparator.comparing(File::toString)) + .collect(Collectors.toList()); assertThat(savedFiles).size().isEqualTo(3); - assertSavedFile(savedFiles.get(0), + assertSavedFile( + savedFiles.get(0), "my_buf_type", "my_prefix", startNanos, @@ -169,7 +199,8 @@ void saveBuffer() throws IOException { "1paramA", "1paramB", "content1".getBytes(StandardCharsets.UTF_8)); - assertSavedFile(savedFiles.get(1), + assertSavedFile( + savedFiles.get(1), "my_buf_type", "my_prefix", startNanos, @@ -177,7 +208,8 @@ void saveBuffer() throws IOException { "2param-a", "2param-b", "content2".getBytes(StandardCharsets.UTF_8)); - assertSavedFile(savedFiles.get(2), + assertSavedFile( + savedFiles.get(2), "my_buf_type", "my_prefix", startNanos, @@ -186,4 +218,4 @@ void saveBuffer() throws IOException { "3param_b", "content3".getBytes(StandardCharsets.UTF_8)); } -} \ No newline at end of file +} diff --git a/fluency-core/src/test/java/org/komamitsu/fluency/flusher/FlusherTest.java b/fluency-core/src/test/java/org/komamitsu/fluency/flusher/FlusherTest.java index d51dff21..85cca975 100644 --- a/fluency-core/src/test/java/org/komamitsu/fluency/flusher/FlusherTest.java +++ b/fluency-core/src/test/java/org/komamitsu/fluency/flusher/FlusherTest.java @@ -16,17 +16,6 @@ package org.komamitsu.fluency.flusher; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.JsonRecordFormatter; -import org.komamitsu.fluency.buffer.Buffer; -import org.komamitsu.fluency.ingester.Ingester; - -import java.io.IOException; -import java.lang.reflect.Field; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.TimeUnit; - import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.eq; @@ -37,117 +26,122 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -class FlusherTest -{ - private Ingester ingester; - private Buffer.Config bufferConfig; - private Flusher.Config flusherConfig; +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.JsonRecordFormatter; +import org.komamitsu.fluency.buffer.Buffer; +import org.komamitsu.fluency.ingester.Ingester; + +class FlusherTest { + private Ingester ingester; + private Buffer.Config bufferConfig; + private Flusher.Config flusherConfig; - @BeforeEach - void setUp() - { - ingester = mock(Ingester.class); + @BeforeEach + void setUp() { + ingester = mock(Ingester.class); - bufferConfig = new Buffer.Config(); - flusherConfig = new Flusher.Config(); - } + bufferConfig = new Buffer.Config(); + flusherConfig = new Flusher.Config(); + } - @Test - void testAsyncFlusher() - throws IOException, InterruptedException - { - flusherConfig.setFlushAttemptIntervalMillis(500); + @Test + void testAsyncFlusher() throws IOException, InterruptedException { + flusherConfig.setFlushAttemptIntervalMillis(500); + + Buffer buffer = spy(new Buffer(bufferConfig, new JsonRecordFormatter())); + Flusher flusher = new Flusher(flusherConfig, buffer, ingester); - Buffer buffer = spy(new Buffer(bufferConfig, new JsonRecordFormatter())); - Flusher flusher = new Flusher(flusherConfig, buffer, ingester); + verify(buffer, times(0)).flush(eq(ingester), anyBoolean()); - verify(buffer, times(0)).flush(eq(ingester), anyBoolean()); + verify(buffer, times(0)).flush(eq(ingester), anyBoolean()); - verify(buffer, times(0)).flush(eq(ingester), anyBoolean()); + flusher.flush(); + TimeUnit.MILLISECONDS.sleep(50); + verify(buffer, times(0)).flush(eq(ingester), eq(false)); + verify(buffer, times(1)).flush(eq(ingester), eq(true)); - flusher.flush(); - TimeUnit.MILLISECONDS.sleep(50); - verify(buffer, times(0)).flush(eq(ingester), eq(false)); - verify(buffer, times(1)).flush(eq(ingester), eq(true)); + TimeUnit.SECONDS.sleep(1); + verify(buffer, atLeast(1)).flush(eq(ingester), eq(false)); + verify(buffer, atMost(3)).flush(eq(ingester), eq(false)); + verify(buffer, times(1)).flush(eq(ingester), eq(true)); - TimeUnit.SECONDS.sleep(1); - verify(buffer, atLeast(1)).flush(eq(ingester), eq(false)); - verify(buffer, atMost(3)).flush(eq(ingester), eq(false)); - verify(buffer, times(1)).flush(eq(ingester), eq(true)); + verify(buffer, times(0)).close(); - verify(buffer, times(0)).close(); + flusher.close(); + verify(buffer, times(1)).close(); - flusher.close(); - verify(buffer, times(1)).close(); + verify(buffer, atLeast(2)).flush(eq(ingester), eq(false)); + verify(buffer, atMost(3)).flush(eq(ingester), eq(false)); - verify(buffer, atLeast(2)).flush(eq(ingester), eq(false)); - verify(buffer, atMost(3)).flush(eq(ingester), eq(false)); + verify(buffer, atLeast(2)).flush(eq(ingester), eq(true)); + verify(buffer, atMost(3)).flush(eq(ingester), eq(true)); + } - verify(buffer, atLeast(2)).flush(eq(ingester), eq(true)); - verify(buffer, atMost(3)).flush(eq(ingester), eq(true)); + @Test + void validateConfig() { + Buffer buffer = spy(new Buffer(bufferConfig, new JsonRecordFormatter())); + + { + Flusher.Config config = new Flusher.Config(); + config.setFlushAttemptIntervalMillis(19); + assertThrows(IllegalArgumentException.class, () -> new Flusher(config, buffer, ingester)); } - @Test - void validateConfig() { - Buffer buffer = spy(new Buffer(bufferConfig, new JsonRecordFormatter())); - - { - Flusher.Config config = new Flusher.Config(); - config.setFlushAttemptIntervalMillis(19); - assertThrows(IllegalArgumentException.class, () -> new Flusher(config, buffer, ingester)); - } - - { - Flusher.Config config = new Flusher.Config(); - config.setFlushAttemptIntervalMillis(2001); - assertThrows(IllegalArgumentException.class, () -> new Flusher(config, buffer, ingester)); - } - - { - Flusher.Config config = new Flusher.Config(); - config.setWaitUntilBufferFlushed(0); - assertThrows(IllegalArgumentException.class, () -> new Flusher(config, buffer, ingester)); - } - - { - Flusher.Config config = new Flusher.Config(); - config.setWaitUntilTerminated(0); - assertThrows(IllegalArgumentException.class, () -> new Flusher(config, buffer, ingester)); - } + Flusher.Config config = new Flusher.Config(); + config.setFlushAttemptIntervalMillis(2001); + assertThrows(IllegalArgumentException.class, () -> new Flusher(config, buffer, ingester)); } - @Test - void queueSizeShouldNotExceedLimit() throws Exception { - flusherConfig.setFlushAttemptIntervalMillis(200); - - Buffer buffer = new Buffer(bufferConfig, new JsonRecordFormatter()); - Flusher flusher = new Flusher(flusherConfig, buffer, ingester); - - BlockingQueue eventQueue = (BlockingQueue) extractValue(flusher, "eventQueue"); - - int nonZeroCounts = 0; - for (int index = 0; index < 10_000; index++) { - flusher.flush(); - if(!eventQueue.isEmpty()) { - nonZeroCounts++; - } - // The eventQueue will always have less that 16 elements (the default max size) - assertTrue(eventQueue.size() <= 16); - } - // The eventQueue will be non empty at least a couple of times - assertTrue(nonZeroCounts > 0); - - // Wait for sufficiently long (amount of time > queue poll wait) and the queue should be polled completely to become empty - Thread.sleep(1000); - assertEquals(0, eventQueue.size()); + Flusher.Config config = new Flusher.Config(); + config.setWaitUntilBufferFlushed(0); + assertThrows(IllegalArgumentException.class, () -> new Flusher(config, buffer, ingester)); } - private Object extractValue(Object object, String fieldName) throws Exception { - Field field = object.getClass().getDeclaredField(fieldName); - field.setAccessible(true); - return field.get(object); + Flusher.Config config = new Flusher.Config(); + config.setWaitUntilTerminated(0); + assertThrows(IllegalArgumentException.class, () -> new Flusher(config, buffer, ingester)); + } + } + + @Test + void queueSizeShouldNotExceedLimit() throws Exception { + flusherConfig.setFlushAttemptIntervalMillis(200); + + Buffer buffer = new Buffer(bufferConfig, new JsonRecordFormatter()); + Flusher flusher = new Flusher(flusherConfig, buffer, ingester); + + BlockingQueue eventQueue = + (BlockingQueue) extractValue(flusher, "eventQueue"); + + int nonZeroCounts = 0; + for (int index = 0; index < 10_000; index++) { + flusher.flush(); + if (!eventQueue.isEmpty()) { + nonZeroCounts++; + } + // The eventQueue will always have less that 16 elements (the default max size) + assertTrue(eventQueue.size() <= 16); } + // The eventQueue will be non empty at least a couple of times + assertTrue(nonZeroCounts > 0); + + // Wait for sufficiently long (amount of time > queue poll wait) and the queue should be polled + // completely to become empty + Thread.sleep(1000); + assertEquals(0, eventQueue.size()); + } + + private Object extractValue(Object object, String fieldName) throws Exception { + Field field = object.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + return field.get(object); + } } diff --git a/fluency-core/src/test/java/org/komamitsu/fluency/recordformat/MessagePackRecordFormatterTest.java b/fluency-core/src/test/java/org/komamitsu/fluency/recordformat/MessagePackRecordFormatterTest.java index b3065f65..0b6b76da 100644 --- a/fluency-core/src/test/java/org/komamitsu/fluency/recordformat/MessagePackRecordFormatterTest.java +++ b/fluency-core/src/test/java/org/komamitsu/fluency/recordformat/MessagePackRecordFormatterTest.java @@ -16,9 +16,14 @@ package org.komamitsu.fluency.recordformat; +import static org.junit.jupiter.api.Assertions.assertEquals; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Map; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.msgpack.core.MessagePack; @@ -28,126 +33,117 @@ import org.msgpack.value.Value; import org.msgpack.value.ValueFactory; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; +class MessagePackRecordFormatterTest { + private static final String TAG = "foodb.bartbl"; + private static final long FUTURE_EPOCH = 4294967296L; + private static final Map RECORD_0 = + ImmutableMap.of("name", "first", "age", 42, "email", "hello@world.com"); + private static final Map RECORD_1 = + ImmutableMap.of("name", "second", "age", 55, "time", FUTURE_EPOCH, "comment", "zzzzzz"); + private static final Map RECORD_2 = + ImmutableMap.of("job", "knight", "name", "third", "age", 99); + private static final StringValue KEY_TIME = ValueFactory.newString("time"); + private static final StringValue KEY_NAME = ValueFactory.newString("name"); + private static final StringValue KEY_AGE = ValueFactory.newString("age"); + private static final StringValue KEY_EMAIL = ValueFactory.newString("email"); + private static final StringValue KEY_COMMENT = ValueFactory.newString("comment"); + private static final StringValue KEY_JOB = ValueFactory.newString("job"); + private MessagePackRecordFormatter recordFormatter; -class MessagePackRecordFormatterTest -{ - private static final String TAG = "foodb.bartbl"; - private static final long FUTURE_EPOCH = 4294967296L; - private static final Map RECORD_0 = ImmutableMap.of("name", "first", "age", 42, "email", "hello@world.com"); - private static final Map RECORD_1 = ImmutableMap.of("name", "second", "age", 55, "time", FUTURE_EPOCH, "comment", "zzzzzz"); - private static final Map RECORD_2 = ImmutableMap.of("job", "knight", "name", "third", "age", 99); - private static final StringValue KEY_TIME = ValueFactory.newString("time"); - private static final StringValue KEY_NAME = ValueFactory.newString("name"); - private static final StringValue KEY_AGE = ValueFactory.newString("age"); - private static final StringValue KEY_EMAIL = ValueFactory.newString("email"); - private static final StringValue KEY_COMMENT = ValueFactory.newString("comment"); - private static final StringValue KEY_JOB = ValueFactory.newString("job"); - private MessagePackRecordFormatter recordFormatter; + @BeforeEach + void setUp() { + recordFormatter = new MessagePackRecordFormatter(new MessagePackRecordFormatter.Config()); + } - @BeforeEach - void setUp() - { - recordFormatter = new MessagePackRecordFormatter(new MessagePackRecordFormatter.Config()); + private void assertRecord0(byte[] formatted, long expectedTime) throws IOException { + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(formatted)) { + Map map = unpacker.unpackValue().asMapValue().map(); + assertEquals(4, map.size()); + assertEquals(expectedTime, map.get(KEY_TIME).asNumberValue().toLong()); + assertEquals("first", map.get(KEY_NAME).asStringValue().asString()); + assertEquals(42, map.get(KEY_AGE).asNumberValue().toInt()); + assertEquals("hello@world.com", map.get(KEY_EMAIL).asStringValue().asString()); } + } - private void assertRecord0(byte[] formatted, long expectedTime) - throws IOException - { - try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(formatted)) { - Map map = unpacker.unpackValue().asMapValue().map(); - assertEquals(4, map.size()); - assertEquals(expectedTime, map.get(KEY_TIME).asNumberValue().toLong()); - assertEquals("first", map.get(KEY_NAME).asStringValue().asString()); - assertEquals(42, map.get(KEY_AGE).asNumberValue().toInt()); - assertEquals("hello@world.com", map.get(KEY_EMAIL).asStringValue().asString()); - } + private void assertRecord1(byte[] formatted, long expectedTime) throws IOException { + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(formatted)) { + Map map = unpacker.unpackValue().asMapValue().map(); + assertEquals(4, map.size()); + assertEquals(expectedTime, map.get(KEY_TIME).asNumberValue().toLong()); + assertEquals("second", map.get(KEY_NAME).asStringValue().asString()); + assertEquals(55, map.get(KEY_AGE).asNumberValue().toInt()); + assertEquals("zzzzzz", map.get(KEY_COMMENT).asStringValue().asString()); } + } - private void assertRecord1(byte[] formatted, long expectedTime) - throws IOException - { - try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(formatted)) { - Map map = unpacker.unpackValue().asMapValue().map(); - assertEquals(4, map.size()); - assertEquals(expectedTime, map.get(KEY_TIME).asNumberValue().toLong()); - assertEquals("second", map.get(KEY_NAME).asStringValue().asString()); - assertEquals(55, map.get(KEY_AGE).asNumberValue().toInt()); - assertEquals("zzzzzz", map.get(KEY_COMMENT).asStringValue().asString()); - } + private void assertRecord2(byte[] formatted, long expectedTime) throws IOException { + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(formatted)) { + Map map = unpacker.unpackValue().asMapValue().map(); + assertEquals(4, map.size()); + assertEquals(expectedTime, map.get(KEY_TIME).asNumberValue().toLong()); + assertEquals("third", map.get(KEY_NAME).asStringValue().asString()); + assertEquals(99, map.get(KEY_AGE).asNumberValue().toInt()); + assertEquals("knight", map.get(KEY_JOB).asStringValue().asString()); } + } - private void assertRecord2(byte[] formatted, long expectedTime) - throws IOException - { - try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(formatted)) { - Map map = unpacker.unpackValue().asMapValue().map(); - assertEquals(4, map.size()); - assertEquals(expectedTime, map.get(KEY_TIME).asNumberValue().toLong()); - assertEquals("third", map.get(KEY_NAME).asStringValue().asString()); - assertEquals(99, map.get(KEY_AGE).asNumberValue().toInt()); - assertEquals("knight", map.get(KEY_JOB).asStringValue().asString()); - } - } + @Test + void format() throws IOException { + long now = System.currentTimeMillis() / 1000; + assertRecord0(recordFormatter.format(TAG, now, RECORD_0), now); + assertRecord1(recordFormatter.format(TAG, now, RECORD_1), FUTURE_EPOCH); + assertRecord2(recordFormatter.format(TAG, now, RECORD_2), now); + } - @Test - void format() - throws IOException + @Test + void formatFromMessagePackBytes() throws IOException { + long now = System.currentTimeMillis() / 1000; + ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); { - long now = System.currentTimeMillis() / 1000; - assertRecord0(recordFormatter.format(TAG, now, RECORD_0), now); - assertRecord1(recordFormatter.format(TAG, now, RECORD_1), FUTURE_EPOCH); - assertRecord2(recordFormatter.format(TAG, now, RECORD_2), now); + byte[] bytes = objectMapper.writeValueAsBytes(RECORD_0); + assertRecord0(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), now); } - - @Test - void formatFromMessagePackBytes() - throws IOException { - long now = System.currentTimeMillis() / 1000; - ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); - { - byte[] bytes = objectMapper.writeValueAsBytes(RECORD_0); - assertRecord0(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), now); - } - { - byte[] bytes = objectMapper.writeValueAsBytes(RECORD_1); - assertRecord1(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), FUTURE_EPOCH); - } - { - byte[] bytes = objectMapper.writeValueAsBytes(RECORD_2); - assertRecord2(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), now); - } + byte[] bytes = objectMapper.writeValueAsBytes(RECORD_1); + assertRecord1( + recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), FUTURE_EPOCH); } - - private ByteBuffer convertMapToMessagePackByteBuffer(Map record, boolean isDirect) - throws JsonProcessingException { - ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); - byte[] bytes = objectMapper.writeValueAsBytes(record); - ByteBuffer byteBuffer; - if (isDirect) { - byteBuffer = ByteBuffer.allocateDirect(bytes.length); - } - else { - byteBuffer = ByteBuffer.allocate(bytes.length); - } - byteBuffer.put(bytes); - byteBuffer.flip(); - return byteBuffer; + byte[] bytes = objectMapper.writeValueAsBytes(RECORD_2); + assertRecord2(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), now); } + } - @Test - void formatFromMessagePackByteBuffer() - throws IOException - { - long now = System.currentTimeMillis() / 1000; - assertRecord0(recordFormatter.formatFromMessagePack(TAG, now, convertMapToMessagePackByteBuffer(RECORD_0, false)), now); - assertRecord1(recordFormatter.formatFromMessagePack(TAG, now, convertMapToMessagePackByteBuffer(RECORD_1, true)), FUTURE_EPOCH); - assertRecord2(recordFormatter.formatFromMessagePack(TAG, now, convertMapToMessagePackByteBuffer(RECORD_2, true)), now); + private ByteBuffer convertMapToMessagePackByteBuffer(Map record, boolean isDirect) + throws JsonProcessingException { + ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); + byte[] bytes = objectMapper.writeValueAsBytes(record); + ByteBuffer byteBuffer; + if (isDirect) { + byteBuffer = ByteBuffer.allocateDirect(bytes.length); + } else { + byteBuffer = ByteBuffer.allocate(bytes.length); } + byteBuffer.put(bytes); + byteBuffer.flip(); + return byteBuffer; + } + + @Test + void formatFromMessagePackByteBuffer() throws IOException { + long now = System.currentTimeMillis() / 1000; + assertRecord0( + recordFormatter.formatFromMessagePack( + TAG, now, convertMapToMessagePackByteBuffer(RECORD_0, false)), + now); + assertRecord1( + recordFormatter.formatFromMessagePack( + TAG, now, convertMapToMessagePackByteBuffer(RECORD_1, true)), + FUTURE_EPOCH); + assertRecord2( + recordFormatter.formatFromMessagePack( + TAG, now, convertMapToMessagePackByteBuffer(RECORD_2, true)), + now); + } } diff --git a/fluency-core/src/test/java/org/komamitsu/fluency/validation/ValidatableTest.java b/fluency-core/src/test/java/org/komamitsu/fluency/validation/ValidatableTest.java index 3d210189..881dd3e7 100644 --- a/fluency-core/src/test/java/org/komamitsu/fluency/validation/ValidatableTest.java +++ b/fluency-core/src/test/java/org/komamitsu/fluency/validation/ValidatableTest.java @@ -16,147 +16,132 @@ package org.komamitsu.fluency.validation; +import static org.junit.jupiter.api.Assertions.assertThrows; + import org.junit.jupiter.api.Test; import org.komamitsu.fluency.validation.annotation.DecimalMax; import org.komamitsu.fluency.validation.annotation.DecimalMin; import org.komamitsu.fluency.validation.annotation.Max; import org.komamitsu.fluency.validation.annotation.Min; -import static org.junit.jupiter.api.Assertions.assertThrows; +class ValidatableTest { + private static class MaxTest implements Validatable { + @Max(42) + private final int i; + + @Max(Integer.MAX_VALUE) + private final Long l; + + @Max(value = 42, inclusive = false) + private final int exclusive; -class ValidatableTest -{ - private static class MaxTest - implements Validatable - { - @Max(42) - private final int i; - - @Max(Integer.MAX_VALUE) - private final Long l; - - @Max(value = 42, inclusive = false) - private final int exclusive; - - public MaxTest(int i, Long l, int exclusive) - { - this.i = i; - this.l = l; - this.exclusive = exclusive; - } + public MaxTest(int i, Long l, int exclusive) { + this.i = i; + this.l = l; + this.exclusive = exclusive; } + } - private static class MinTest - implements Validatable - { - @Min(42) - private final int i; + private static class MinTest implements Validatable { + @Min(42) + private final int i; - @Min(Integer.MIN_VALUE) - private final Long l; + @Min(Integer.MIN_VALUE) + private final Long l; - @Min(value = 42, inclusive = false) - private final int exclusive; + @Min(value = 42, inclusive = false) + private final int exclusive; - public MinTest(int i, Long l, int exclusive) - { - this.i = i; - this.l = l; - this.exclusive = exclusive; - } + public MinTest(int i, Long l, int exclusive) { + this.i = i; + this.l = l; + this.exclusive = exclusive; } + } - private static class DecimalMaxTest - implements Validatable - { - @DecimalMax("3.14") - private final float f; + private static class DecimalMaxTest implements Validatable { + @DecimalMax("3.14") + private final float f; - @DecimalMax(value = "3.14", inclusive = false) - private final Double exclusive; + @DecimalMax(value = "3.14", inclusive = false) + private final Double exclusive; - public DecimalMaxTest(float f, Double exclusive) - { - this.f = f; - this.exclusive = exclusive; - } + public DecimalMaxTest(float f, Double exclusive) { + this.f = f; + this.exclusive = exclusive; } + } - private static class DecimalMinTest - implements Validatable - { - @DecimalMin("3.14") - private final float f; + private static class DecimalMinTest implements Validatable { + @DecimalMin("3.14") + private final float f; - @DecimalMin(value = "3.14", inclusive = false) - private final Double exclusive; + @DecimalMin(value = "3.14", inclusive = false) + private final Double exclusive; - public DecimalMinTest(float f, Double exclusive) - { - this.f = f; - this.exclusive = exclusive; - } + public DecimalMinTest(float f, Double exclusive) { + this.f = f; + this.exclusive = exclusive; } + } - @Test - void validateMax() - { - new MaxTest(42, (long) Integer.MAX_VALUE, 41).validate(); + @Test + void validateMax() { + new MaxTest(42, (long) Integer.MAX_VALUE, 41).validate(); - assertThrows(IllegalArgumentException.class, - () -> new MaxTest(43, (long) Integer.MAX_VALUE, 41).validate()); + assertThrows( + IllegalArgumentException.class, + () -> new MaxTest(43, (long) Integer.MAX_VALUE, 41).validate()); - assertThrows(IllegalArgumentException.class, - () -> new MaxTest(42, (long) Integer.MAX_VALUE + 1, 41).validate()); + assertThrows( + IllegalArgumentException.class, + () -> new MaxTest(42, (long) Integer.MAX_VALUE + 1, 41).validate()); - assertThrows(IllegalArgumentException.class, - () -> new MaxTest(42, (long) Integer.MAX_VALUE, 42).validate()); + assertThrows( + IllegalArgumentException.class, + () -> new MaxTest(42, (long) Integer.MAX_VALUE, 42).validate()); - new MaxTest(42, null, 41).validate(); - } + new MaxTest(42, null, 41).validate(); + } - @Test - void validateMin() - { - new MinTest(42, (long) Integer.MIN_VALUE, 43).validate(); + @Test + void validateMin() { + new MinTest(42, (long) Integer.MIN_VALUE, 43).validate(); - assertThrows(IllegalArgumentException.class, - () -> new MinTest(41, (long) Integer.MIN_VALUE, 43).validate()); + assertThrows( + IllegalArgumentException.class, + () -> new MinTest(41, (long) Integer.MIN_VALUE, 43).validate()); - assertThrows(IllegalArgumentException.class, - () -> new MinTest(42, (long) Integer.MIN_VALUE - 1, 43).validate()); + assertThrows( + IllegalArgumentException.class, + () -> new MinTest(42, (long) Integer.MIN_VALUE - 1, 43).validate()); - assertThrows(IllegalArgumentException.class, - () -> new MinTest(42, (long) Integer.MIN_VALUE, 42).validate()); + assertThrows( + IllegalArgumentException.class, + () -> new MinTest(42, (long) Integer.MIN_VALUE, 42).validate()); - new MinTest(42, null, 43).validate(); - } + new MinTest(42, null, 43).validate(); + } - @Test - void validateDecimalMax() - { - new DecimalMaxTest(3.14f, 3.13).validate(); + @Test + void validateDecimalMax() { + new DecimalMaxTest(3.14f, 3.13).validate(); - assertThrows(IllegalArgumentException.class, - () -> new DecimalMaxTest(3.15f, 3.13).validate()); + assertThrows(IllegalArgumentException.class, () -> new DecimalMaxTest(3.15f, 3.13).validate()); - assertThrows(IllegalArgumentException.class, - () -> new DecimalMaxTest(3.14f, 3.14).validate()); + assertThrows(IllegalArgumentException.class, () -> new DecimalMaxTest(3.14f, 3.14).validate()); - new DecimalMaxTest(3.14f, null).validate(); - } + new DecimalMaxTest(3.14f, null).validate(); + } - @Test - void validateDecimalMin() - { - new DecimalMinTest(3.14f, 3.15).validate(); + @Test + void validateDecimalMin() { + new DecimalMinTest(3.14f, 3.15).validate(); - assertThrows(IllegalArgumentException.class, - () -> new DecimalMinTest(3.13f, 3.15).validate()); + assertThrows(IllegalArgumentException.class, () -> new DecimalMinTest(3.13f, 3.15).validate()); - assertThrows(IllegalArgumentException.class, - () -> new DecimalMinTest(3.14f, 3.14).validate()); + assertThrows(IllegalArgumentException.class, () -> new DecimalMinTest(3.14f, 3.14).validate()); - new DecimalMinTest(3.14f, null).validate(); - } -} \ No newline at end of file + new DecimalMinTest(3.14f, null).validate(); + } +} diff --git a/fluency-fluentd-ext/src/main/java/org/komamitsu/fluency/fluentd/FluencyExtBuilderForFluentd.java b/fluency-fluentd-ext/src/main/java/org/komamitsu/fluency/fluentd/FluencyExtBuilderForFluentd.java index f74beb84..e622fd6f 100644 --- a/fluency-fluentd-ext/src/main/java/org/komamitsu/fluency/fluentd/FluencyExtBuilderForFluentd.java +++ b/fluency-fluentd-ext/src/main/java/org/komamitsu/fluency/fluentd/FluencyExtBuilderForFluentd.java @@ -16,48 +16,37 @@ package org.komamitsu.fluency.fluentd; +import java.nio.file.Path; import org.komamitsu.fluency.Fluency; import org.komamitsu.fluency.fluentd.ingester.sender.FluentdSender; -import org.komamitsu.fluency.fluentd.ingester.sender.MultiSender; import org.komamitsu.fluency.fluentd.ingester.sender.UnixSocketSender; import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.PhiAccrualFailureDetectStrategy; import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.UnixSocketHeartbeater; -import java.net.UnixDomainSocketAddress; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; +public class FluencyExtBuilderForFluentd extends FluencyBuilderForFluentd { + public Fluency build(Path socketPath) { + return buildFromIngester(recordFormatter, buildIngester(createBaseSender(socketPath, false))); + } -public class FluencyExtBuilderForFluentd - extends FluencyBuilderForFluentd -{ - public Fluency build(Path socketPath) - { - return buildFromIngester( - recordFormatter, - buildIngester(createBaseSender(socketPath, false))); + private FluentdSender createBaseSender(Path path, boolean withHeartBeater) { + UnixSocketSender.Config senderConfig = new UnixSocketSender.Config(); + FailureDetector failureDetector = null; + if (path != null) { + senderConfig.setPath(path.toAbsolutePath()); } - - private FluentdSender createBaseSender(Path path, boolean withHeartBeater) - { - UnixSocketSender.Config senderConfig = new UnixSocketSender.Config(); - FailureDetector failureDetector = null; - if (path != null) { - senderConfig.setPath(path.toAbsolutePath()); - } - if (withHeartBeater) { - UnixSocketHeartbeater.Config hbConfig = new UnixSocketHeartbeater.Config(); - hbConfig.setPath(path.toAbsolutePath()); - UnixSocketHeartbeater heartbeater = new UnixSocketHeartbeater(hbConfig); - failureDetector = new FailureDetector(new PhiAccrualFailureDetectStrategy(), heartbeater); - } - if (connectionTimeoutMilli != null) { - senderConfig.setConnectionTimeoutMilli(connectionTimeoutMilli); - } - if (readTimeoutMilli != null) { - senderConfig.setReadTimeoutMilli(readTimeoutMilli); - } - return new UnixSocketSender(senderConfig, failureDetector); + if (withHeartBeater) { + UnixSocketHeartbeater.Config hbConfig = new UnixSocketHeartbeater.Config(); + hbConfig.setPath(path.toAbsolutePath()); + UnixSocketHeartbeater heartbeater = new UnixSocketHeartbeater(hbConfig); + failureDetector = new FailureDetector(new PhiAccrualFailureDetectStrategy(), heartbeater); + } + if (connectionTimeoutMilli != null) { + senderConfig.setConnectionTimeoutMilli(connectionTimeoutMilli); + } + if (readTimeoutMilli != null) { + senderConfig.setReadTimeoutMilli(readTimeoutMilli); } + return new UnixSocketSender(senderConfig, failureDetector); + } } diff --git a/fluency-fluentd-ext/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/UnixSocketSender.java b/fluency-fluentd-ext/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/UnixSocketSender.java index 30bb817f..e8696f53 100644 --- a/fluency-fluentd-ext/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/UnixSocketSender.java +++ b/fluency-fluentd-ext/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/UnixSocketSender.java @@ -16,11 +16,6 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; -import org.komamitsu.fluency.validation.Validatable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.net.SocketException; import java.net.UnixDomainSocketAddress; @@ -29,113 +24,92 @@ import java.nio.file.Path; import java.util.List; import java.util.concurrent.atomic.AtomicReference; +import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; +import org.komamitsu.fluency.validation.Validatable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class UnixSocketSender - extends NetworkSender -{ - private static final Logger LOG = LoggerFactory.getLogger(UnixSocketSender.class); - private final AtomicReference channel = new AtomicReference<>(); - private final Config config; - - public UnixSocketSender() - { - this(new Config()); - } - - public UnixSocketSender(Config config) - { - this(config, null); +public class UnixSocketSender extends NetworkSender { + private static final Logger LOG = LoggerFactory.getLogger(UnixSocketSender.class); + private final AtomicReference channel = new AtomicReference<>(); + private final Config config; + + public UnixSocketSender() { + this(new Config()); + } + + public UnixSocketSender(Config config) { + this(config, null); + } + + public UnixSocketSender(FailureDetector failureDetector) { + this(new Config(), failureDetector); + } + + public UnixSocketSender(Config config, FailureDetector failureDetector) { + super(config, failureDetector); + config.validateValues(); + this.config = config; + } + + @Override + protected SocketChannel getOrCreateSocketInternal() throws IOException { + if (channel.get() == null) { + UnixDomainSocketAddress socketAddress = UnixDomainSocketAddress.of(config.getPath()); + channel.set(SocketChannel.open(socketAddress)); } - - public UnixSocketSender(FailureDetector failureDetector) - { - this(new Config(), failureDetector); + return channel.get(); + } + + public Path getPath() { + return config.getPath(); + } + + @Override + protected void sendBuffers(SocketChannel socketChannel, List buffers) + throws IOException { + socketChannel.write(buffers.toArray(new ByteBuffer[0])); + } + + @Override + protected void recvResponse(SocketChannel socketChannel, ByteBuffer buffer) throws IOException { + int read = socketChannel.read(buffer); + if (read < 0) { + throw new SocketException("The connection is disconnected by the peer"); } + } - public UnixSocketSender(Config config, FailureDetector failureDetector) - { - super(config, failureDetector); - config.validateValues(); - this.config = config; + @Override + protected void closeSocket() throws IOException { + SocketChannel existingSocketChannel; + if ((existingSocketChannel = channel.getAndSet(null)) != null) { + existingSocketChannel.close(); } + } - @Override - protected SocketChannel getOrCreateSocketInternal() - throws IOException - { - if (channel.get() == null) { - UnixDomainSocketAddress socketAddress = UnixDomainSocketAddress.of(config.getPath()); - channel.set(SocketChannel.open(socketAddress)); - } - return channel.get(); - } + @Override + public String toString() { + return "UnixSocketSender{" + "config=" + config + "} " + super.toString(); + } - public Path getPath() - { - return config.getPath(); - } + public static class Config extends NetworkSender.Config implements Validatable { + private Path path; - @Override - protected void sendBuffers(SocketChannel socketChannel, List buffers) - throws IOException - { - socketChannel.write(buffers.toArray(new ByteBuffer[0])); + void validateValues() { + validate(); } - @Override - protected void recvResponse(SocketChannel socketChannel, ByteBuffer buffer) - throws IOException - { - int read = socketChannel.read(buffer); - if (read < 0) { - throw new SocketException("The connection is disconnected by the peer"); - } + public Path getPath() { + return path; } - @Override - protected void closeSocket() - throws IOException - { - SocketChannel existingSocketChannel; - if ((existingSocketChannel = channel.getAndSet(null)) != null) { - existingSocketChannel.close(); - } + public void setPath(Path path) { + this.path = path; } @Override - public String toString() - { - return "UnixSocketSender{" + - "config=" + config + - "} " + super.toString(); - } - - public static class Config - extends NetworkSender.Config - implements Validatable - { - private Path path; - - void validateValues() - { - validate(); - } - - public Path getPath() - { - return path; - } - - public void setPath(Path path) - { - this.path = path; - } - - @Override - public String toString() { - return "Config{" + - "path=" + path + - "} " + super.toString(); - } + public String toString() { + return "Config{" + "path=" + path + "} " + super.toString(); } + } } diff --git a/fluency-fluentd-ext/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UnixSocketHeartbeater.java b/fluency-fluentd-ext/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UnixSocketHeartbeater.java index 66dddbb8..5efdfe1b 100644 --- a/fluency-fluentd-ext/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UnixSocketHeartbeater.java +++ b/fluency-fluentd-ext/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UnixSocketHeartbeater.java @@ -16,68 +16,53 @@ package org.komamitsu.fluency.fluentd.ingester.sender.heartbeat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.net.UnixDomainSocketAddress; import java.nio.channels.SocketChannel; import java.nio.file.Path; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class UnixSocketHeartbeater - extends Heartbeater -{ - private static final Logger LOG = LoggerFactory.getLogger(UnixSocketHeartbeater.class); - private final Config config; +public class UnixSocketHeartbeater extends Heartbeater { + private static final Logger LOG = LoggerFactory.getLogger(UnixSocketHeartbeater.class); + private final Config config; - public UnixSocketHeartbeater() - { - this(new Config()); - } + public UnixSocketHeartbeater() { + this(new Config()); + } - public UnixSocketHeartbeater(final Config config) - { - super(config); - this.config = config; - } + public UnixSocketHeartbeater(final Config config) { + super(config); + this.config = config; + } - @Override - protected void invoke() - throws IOException - { - try (SocketChannel socketChannel = SocketChannel.open(UnixDomainSocketAddress.of(config.getPath()))) { - LOG.trace("UnixSocketHeartbeat: {}", socketChannel); - pong(); - } + @Override + protected void invoke() throws IOException { + try (SocketChannel socketChannel = + SocketChannel.open(UnixDomainSocketAddress.of(config.getPath()))) { + LOG.trace("UnixSocketHeartbeat: {}", socketChannel); + pong(); } + } - public Path getPath() - { - return config.getPath(); - } + public Path getPath() { + return config.getPath(); + } - @Override - public String toString() - { - return "UnixSocketHeartbeater{" + - "config=" + config + - "} " + super.toString(); - } + @Override + public String toString() { + return "UnixSocketHeartbeater{" + "config=" + config + "} " + super.toString(); + } - public static class Config - extends Heartbeater.Config - { - private Path path; + public static class Config extends Heartbeater.Config { + private Path path; - public Path getPath() - { - return path; - } + public Path getPath() { + return path; + } - public void setPath(Path path) - { - this.path = path; - } + public void setPath(Path path) { + this.path = path; } + } } - diff --git a/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/FluencyExtBuilderForFluentdTest.java b/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/FluencyExtBuilderForFluentdTest.java index 715062a4..7807d008 100644 --- a/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/FluencyExtBuilderForFluentdTest.java +++ b/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/FluencyExtBuilderForFluentdTest.java @@ -1,5 +1,10 @@ package org.komamitsu.fluency.fluentd; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import org.junit.jupiter.api.Test; import org.komamitsu.fluency.Fluency; import org.komamitsu.fluency.buffer.Buffer; @@ -12,83 +17,73 @@ import org.komamitsu.fluency.fluentd.ingester.sender.retry.ExponentialBackOffRetryStrategy; import org.komamitsu.fluency.flusher.Flusher; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; +class FluencyExtBuilderForFluentdTest { + // These assertMethods are copied from FluencyBuilderForFluentdTest + private void assertBuffer(Buffer buffer) { + assertThat(buffer.getMaxBufferSize()).isEqualTo(512 * 1024 * 1024L); + assertThat(buffer.getFileBackupDir()).isNull(); + assertThat(buffer.bufferFormatType()).isEqualTo("packed_forward"); + assertThat(buffer.getChunkExpandRatio()).isEqualTo(2f); + assertThat(buffer.getChunkRetentionSize()).isEqualTo(4 * 1024 * 1024); + assertThat(buffer.getChunkInitialSize()).isEqualTo(1 * 1024 * 1024); + assertThat(buffer.getChunkRetentionTimeMillis()).isEqualTo(1000); + assertThat(buffer.getJvmHeapBufferMode()).isFalse(); + } -import static org.assertj.core.api.Assertions.assertThat; + private void assertFlusher(Flusher flusher) { + assertThat(flusher.isTerminated()).isFalse(); + assertThat(flusher.getFlushAttemptIntervalMillis()).isEqualTo(600); + assertThat(flusher.getWaitUntilBufferFlushed()).isEqualTo(60); + assertThat(flusher.getWaitUntilTerminated()).isEqualTo(60); + } -class FluencyExtBuilderForFluentdTest -{ - // These assertMethods are copied from FluencyBuilderForFluentdTest - private void assertBuffer(Buffer buffer) - { - assertThat(buffer.getMaxBufferSize()).isEqualTo(512 * 1024 * 1024L); - assertThat(buffer.getFileBackupDir()).isNull(); - assertThat(buffer.bufferFormatType()).isEqualTo("packed_forward"); - assertThat(buffer.getChunkExpandRatio()).isEqualTo(2f); - assertThat(buffer.getChunkRetentionSize()).isEqualTo(4 * 1024 * 1024); - assertThat(buffer.getChunkInitialSize()).isEqualTo(1 * 1024 * 1024); - assertThat(buffer.getChunkRetentionTimeMillis()).isEqualTo(1000); - assertThat(buffer.getJvmHeapBufferMode()).isFalse(); - } + private void assertDefaultRetryableSender( + RetryableSender sender, Class expectedBaseClass) { + assertThat(sender.getRetryStrategy()).isInstanceOf(ExponentialBackOffRetryStrategy.class); + ExponentialBackOffRetryStrategy retryStrategy = + (ExponentialBackOffRetryStrategy) sender.getRetryStrategy(); + assertThat(retryStrategy.getMaxRetryCount()).isEqualTo(7); + assertThat(retryStrategy.getBaseIntervalMillis()).isEqualTo(400); + assertThat(sender.getBaseSender()).isInstanceOf(expectedBaseClass); + } - private void assertFlusher(Flusher flusher) - { - assertThat(flusher.isTerminated()).isFalse(); - assertThat(flusher.getFlushAttemptIntervalMillis()).isEqualTo(600); - assertThat(flusher.getWaitUntilBufferFlushed()).isEqualTo(60); - assertThat(flusher.getWaitUntilTerminated()).isEqualTo(60); - } + private void assertUnixSocketSender( + UnixSocketSender sender, Path expectedPath, boolean shouldHaveFailureDetector) { + assertThat(sender.getPath()).isEqualTo(expectedPath); + assertThat(sender.getConnectionTimeoutMilli()).isEqualTo(5000); + assertThat(sender.getReadTimeoutMilli()).isEqualTo(5000); - private void assertDefaultRetryableSender(RetryableSender sender, Class expectedBaseClass) - { - assertThat(sender.getRetryStrategy()).isInstanceOf(ExponentialBackOffRetryStrategy.class); - ExponentialBackOffRetryStrategy retryStrategy = (ExponentialBackOffRetryStrategy) sender.getRetryStrategy(); - assertThat(retryStrategy.getMaxRetryCount()).isEqualTo(7); - assertThat(retryStrategy.getBaseIntervalMillis()).isEqualTo(400); - assertThat(sender.getBaseSender()).isInstanceOf(expectedBaseClass); + FailureDetector failureDetector = sender.getFailureDetector(); + if (shouldHaveFailureDetector) { + assertThat(failureDetector.getFailureIntervalMillis()).isEqualTo(3 * 1000); + assertThat(failureDetector.getFailureDetectStrategy()) + .isInstanceOf(PhiAccrualFailureDetectStrategy.class); + assertThat(failureDetector.getHeartbeater()).isInstanceOf(UnixSocketHeartbeater.class); + { + UnixSocketHeartbeater hb = (UnixSocketHeartbeater) failureDetector.getHeartbeater(); + assertThat(hb.getPath()).isEqualTo(expectedPath); + } + assertThat(failureDetector.getHeartbeater().getIntervalMillis()).isEqualTo(1000); + } else { + assertThat(failureDetector).isNull(); } + } - private void assertUnixSocketSender(UnixSocketSender sender, Path expectedPath, boolean shouldHaveFailureDetector) - { - assertThat(sender.getPath()).isEqualTo(expectedPath); - assertThat(sender.getConnectionTimeoutMilli()).isEqualTo(5000); - assertThat(sender.getReadTimeoutMilli()).isEqualTo(5000); - - FailureDetector failureDetector = sender.getFailureDetector(); - if (shouldHaveFailureDetector) { - assertThat(failureDetector.getFailureIntervalMillis()).isEqualTo(3 * 1000); - assertThat(failureDetector.getFailureDetectStrategy()).isInstanceOf(PhiAccrualFailureDetectStrategy.class); - assertThat(failureDetector.getHeartbeater()).isInstanceOf(UnixSocketHeartbeater.class); - { - UnixSocketHeartbeater hb = (UnixSocketHeartbeater) failureDetector.getHeartbeater(); - assertThat(hb.getPath()).isEqualTo(expectedPath); - } - assertThat(failureDetector.getHeartbeater().getIntervalMillis()).isEqualTo(1000); - } - else { - assertThat(failureDetector).isNull(); - } - } - - private void assertDefaultFluentdSender(FluentdSender sender, Path expectedPath) - { - assertThat(sender).isInstanceOf(RetryableSender.class); - RetryableSender retryableSender = (RetryableSender) sender; - assertDefaultRetryableSender(retryableSender, UnixSocketSender.class); - assertUnixSocketSender((UnixSocketSender) retryableSender.getBaseSender(), expectedPath, false); - } + private void assertDefaultFluentdSender(FluentdSender sender, Path expectedPath) { + assertThat(sender).isInstanceOf(RetryableSender.class); + RetryableSender retryableSender = (RetryableSender) sender; + assertDefaultRetryableSender(retryableSender, UnixSocketSender.class); + assertUnixSocketSender((UnixSocketSender) retryableSender.getBaseSender(), expectedPath, false); + } - @Test - void build() - throws IOException - { - Path socketPath = Paths.get(System.getProperty("java.io.tmpdir"), "foo/bar.socket"); - try (Fluency fluency = new FluencyExtBuilderForFluentd().build(socketPath)) { - assertBuffer(fluency.getBuffer()); - assertFlusher(fluency.getFlusher()); - assertDefaultFluentdSender((FluentdSender) fluency.getFlusher().getIngester().getSender(), socketPath); - } + @Test + void build() throws IOException { + Path socketPath = Paths.get(System.getProperty("java.io.tmpdir"), "foo/bar.socket"); + try (Fluency fluency = new FluencyExtBuilderForFluentd().build(socketPath)) { + assertBuffer(fluency.getBuffer()); + assertFlusher(fluency.getFlusher()); + assertDefaultFluentdSender( + (FluentdSender) fluency.getFlusher().getIngester().getSender(), socketPath); } + } } diff --git a/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/MockUnixSocketServer.java b/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/MockUnixSocketServer.java index 90345c3b..7b35962f 100644 --- a/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/MockUnixSocketServer.java +++ b/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/MockUnixSocketServer.java @@ -16,10 +16,6 @@ package org.komamitsu.fluency.fluentd; -import org.komamitsu.fluency.util.Tuple; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.Closeable; import java.io.IOException; import java.net.StandardProtocolFamily; @@ -37,300 +33,296 @@ import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import org.komamitsu.fluency.util.Tuple; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class MockUnixSocketServer - implements Closeable -{ - private static final Logger LOG = LoggerFactory.getLogger(MockUnixSocketServer.class); - private Path socketPath = Paths.get(System.getProperty("java.io.tmpdir"), - String.format("fluency-mock-unixsocket-server-%s", UUID.randomUUID())); - - private final AtomicLong lastEventTimeStampMilli = new AtomicLong(); - private final AtomicInteger threadSeqNum = new AtomicInteger(); - private ExecutorService executorService; - private ServerTask serverTask; - // TODO Make this class immutable - private final List tasks = new ArrayList<>(); - - private final List> events = new CopyOnWriteArrayList<>(); - - public List> getEvents() - { - return events; +public class MockUnixSocketServer implements Closeable { + private static final Logger LOG = LoggerFactory.getLogger(MockUnixSocketServer.class); + private Path socketPath = + Paths.get( + System.getProperty("java.io.tmpdir"), + String.format("fluency-mock-unixsocket-server-%s", UUID.randomUUID())); + + private final AtomicLong lastEventTimeStampMilli = new AtomicLong(); + private final AtomicInteger threadSeqNum = new AtomicInteger(); + private ExecutorService executorService; + private ServerTask serverTask; + // TODO Make this class immutable + private final List tasks = new ArrayList<>(); + + private final List> events = + new CopyOnWriteArrayList<>(); + + public List> getEvents() { + return events; + } + + @Override + public void close() throws IOException { + stop(true); + } + + public enum Type { + CONNECT, + RECEIVE, + CLOSE; + } + + protected EventHandler eventHandler() { + return new EventHandler() { + @Override + public void onConnect(SocketChannel acceptSocketChannel) { + events.add(new Tuple<>(Type.CONNECT, null)); + } + + @Override + public void onReceive(SocketChannel acceptSocketChannel, ByteBuffer byteBuffer) { + events.add(new Tuple<>(Type.RECEIVE, byteBuffer.flip().remaining())); + } + + @Override + public void onClose(SocketChannel acceptSocketChannel) { + events.add(new Tuple<>(Type.CLOSE, null)); + } + }; + } + + public Path getSocketPath() { + return socketPath; + } + + public synchronized void start() throws Exception { + if (executorService == null) { + this.executorService = + Executors.newCachedThreadPool( + r -> + new Thread( + r, String.format("accepted-socket-worker-%d", threadSeqNum.getAndAdd(1)))); } - @Override - public void close() - throws IOException - { - stop(true); + if (serverTask == null) { + UnixDomainSocketAddress socketAddress = UnixDomainSocketAddress.of(socketPath); + serverTask = + new ServerTask( + executorService, lastEventTimeStampMilli, eventHandler(), socketAddress, tasks); + executorService.execute(serverTask); + tasks.add(serverTask); } - - public enum Type - { - CONNECT, RECEIVE, CLOSE; + } + + public void waitUntilEventsStop() throws InterruptedException { + for (int i = 0; i < 20; i++) { + if (lastEventTimeStampMilli.get() + 2000 < System.currentTimeMillis()) { + return; + } + TimeUnit.MILLISECONDS.sleep(500); } + throw new IllegalStateException("Events didn't stop in the expected time"); + } - protected EventHandler eventHandler() - { - return new EventHandler() - { - @Override - public void onConnect(SocketChannel acceptSocketChannel) - { - events.add(new Tuple<>(Type.CONNECT, null)); - } - - @Override - public void onReceive(SocketChannel acceptSocketChannel, ByteBuffer byteBuffer) - { - events.add(new Tuple<>(Type.RECEIVE, byteBuffer.flip().remaining())); - } + public synchronized void stop() throws IOException { + stop(false); + } - @Override - public void onClose(SocketChannel acceptSocketChannel) - { - events.add(new Tuple<>(Type.CLOSE, null)); - } - }; + public synchronized void stop(boolean immediate) throws IOException { + LOG.debug("Stopping the mock server... {}", this); + if (executorService == null) { + return; } - - public Path getSocketPath() - { - return socketPath; + executorService.shutdown(); + try { + if (!executorService.awaitTermination(1000, TimeUnit.MILLISECONDS)) { + LOG.debug("Shutting down the mock server and child tasks... {}", this); + executorService.shutdownNow(); + } + } catch (InterruptedException e) { + LOG.warn("ExecutorService.shutdown() was failed: {}", this, e); + Thread.currentThread().interrupt(); } - public synchronized void start() - throws Exception - { - if (executorService == null) { - this.executorService = Executors.newCachedThreadPool(r -> - new Thread(r, String.format("accepted-socket-worker-%d", threadSeqNum.getAndAdd(1)))); - } - - if (serverTask == null) { - UnixDomainSocketAddress socketAddress = UnixDomainSocketAddress.of(socketPath); - serverTask = new ServerTask(executorService, lastEventTimeStampMilli, eventHandler(), socketAddress, tasks); - executorService.execute(serverTask); - tasks.add(serverTask); + if (immediate) { + LOG.debug("Closing related sockets {}", this); + for (Runnable runnable : tasks) { + if (runnable instanceof ServerTask) { + ((ServerTask) runnable).close(); + } else if (runnable instanceof ServerTask.AcceptTask) { + ((ServerTask.AcceptTask) runnable).close(); } + } } - public void waitUntilEventsStop() - throws InterruptedException - { - for (int i = 0; i < 20; i++) { - if (lastEventTimeStampMilli.get() + 2000 < System.currentTimeMillis()) { - return; - } - TimeUnit.MILLISECONDS.sleep(500); - } - throw new IllegalStateException("Events didn't stop in the expected time"); - } + executorService = null; + serverTask = null; + } - public synchronized void stop() - throws IOException - { - stop(false); - } - - public synchronized void stop(boolean immediate) - throws IOException - { - LOG.debug("Stopping the mock server... {}", this); - if (executorService == null) { - return; - } - executorService.shutdown(); - try { - if (!executorService.awaitTermination(1000, TimeUnit.MILLISECONDS)) { - LOG.debug("Shutting down the mock server and child tasks... {}", this); - executorService.shutdownNow(); - } - } - catch (InterruptedException e) { - LOG.warn("ExecutorService.shutdown() was failed: {}", this, e); - Thread.currentThread().interrupt(); - } + private interface EventHandler { + void onConnect(SocketChannel acceptSocket); - if (immediate) { - LOG.debug("Closing related sockets {}", this); - for (Runnable runnable : tasks) { - if (runnable instanceof ServerTask) { - ((ServerTask) runnable).close(); - } else if (runnable instanceof ServerTask.AcceptTask) { - ((ServerTask.AcceptTask) runnable).close(); - } - } - } + void onReceive(SocketChannel acceptSocket, ByteBuffer byteBuffer); - executorService = null; - serverTask = null; - } + void onClose(SocketChannel acceptSocket); + } - private interface EventHandler - { - void onConnect(SocketChannel acceptSocket); + private static class ServerTask implements Runnable { + private final ServerSocketChannel serverSocketChannel; + private final ExecutorService serverExecutorService; + private final EventHandler eventHandler; + private final AtomicLong lastEventTimeStampMilli; + private final List tasks; + private final UnixDomainSocketAddress socketAddress; - void onReceive(SocketChannel acceptSocket, ByteBuffer byteBuffer); + @Override + public String toString() { + return "ServerTask{" + + "serverSocketChannel=" + + serverSocketChannel + + ", lastEventTimeStampMilli=" + + lastEventTimeStampMilli + + ", socketAddress=" + + socketAddress + + '}'; + } - void onClose(SocketChannel acceptSocket); + private ServerTask( + ExecutorService executorService, + AtomicLong lastEventTimeStampMilli, + EventHandler eventHandler, + UnixDomainSocketAddress socketAddress, + List tasks) + throws IOException { + this.serverExecutorService = executorService; + this.lastEventTimeStampMilli = lastEventTimeStampMilli; + this.eventHandler = eventHandler; + this.socketAddress = socketAddress; + this.serverSocketChannel = + ServerSocketChannel.open(StandardProtocolFamily.UNIX).bind(socketAddress); + this.tasks = tasks; } - private static class ServerTask - implements Runnable - { - private final ServerSocketChannel serverSocketChannel; - private final ExecutorService serverExecutorService; - private final EventHandler eventHandler; - private final AtomicLong lastEventTimeStampMilli; - private final List tasks; - private final UnixDomainSocketAddress socketAddress; - - @Override - public String toString() { - return "ServerTask{" + - "serverSocketChannel=" + serverSocketChannel + - ", lastEventTimeStampMilli=" + lastEventTimeStampMilli + - ", socketAddress=" + socketAddress + - '}'; + @Override + public void run() { + try { + while (!serverExecutorService.isShutdown()) { + try { + LOG.debug("ServerTask: accepting... this={}", this); + SocketChannel acceptSocket = serverSocketChannel.accept(); + LOG.debug("ServerTask: accepted. this={}, remote={}", this, acceptSocket); + AcceptTask acceptTask = + new AcceptTask( + serverExecutorService, lastEventTimeStampMilli, eventHandler, acceptSocket); + serverExecutorService.execute(acceptTask); + tasks.add(acceptTask); + } catch (RejectedExecutionException | ClosedByInterruptException e) { + LOG.debug( + "ServerTask: ServerSocketChannel.accept() failed[{}]: this={}", + e.getMessage(), + this); + } catch (IOException e) { + LOG.warn( + "ServerTask: ServerSocketChannel.accept() failed[{}]: this={}", + e.getMessage(), + this); + } } - - private ServerTask( - ExecutorService executorService, - AtomicLong lastEventTimeStampMilli, - EventHandler eventHandler, - UnixDomainSocketAddress socketAddress, - List tasks - ) - throws IOException - { - this.serverExecutorService = executorService; - this.lastEventTimeStampMilli = lastEventTimeStampMilli; - this.eventHandler = eventHandler; - this.socketAddress = socketAddress; - this.serverSocketChannel = ServerSocketChannel.open(StandardProtocolFamily.UNIX).bind(socketAddress); - this.tasks = tasks; + } finally { + try { + LOG.debug("ServerTask: closing. this={}", this); + close(); + } catch (IOException e) { + LOG.warn("ServerTask: close() failed", e); } + } + LOG.info("ServerTask: Finishing ServerTask...: this={}", this); + } - @Override - public void run() - { - try { - while (!serverExecutorService.isShutdown()) { - try { - LOG.debug("ServerTask: accepting... this={}", this); - SocketChannel acceptSocket = serverSocketChannel.accept(); - LOG.debug("ServerTask: accepted. this={}, remote={}", this, acceptSocket); - AcceptTask acceptTask = new AcceptTask(serverExecutorService, lastEventTimeStampMilli, eventHandler, acceptSocket); - serverExecutorService.execute(acceptTask); - tasks.add(acceptTask); - } catch (RejectedExecutionException | ClosedByInterruptException e) { - LOG.debug("ServerTask: ServerSocketChannel.accept() failed[{}]: this={}", e.getMessage(), this); - } catch (IOException e) { - LOG.warn("ServerTask: ServerSocketChannel.accept() failed[{}]: this={}", e.getMessage(), this); - } - } - } - finally { - try { - LOG.debug("ServerTask: closing. this={}", this); - close(); - } catch (IOException e) { - LOG.warn("ServerTask: close() failed", e); - } - } - LOG.info("ServerTask: Finishing ServerTask...: this={}", this); - } + private void close() throws IOException { + try { + serverSocketChannel.close(); + } finally { + Files.deleteIfExists(socketAddress.getPath()); + } + } - private void close() - throws IOException - { + private static class AcceptTask implements Runnable { + private final SocketChannel acceptSocketChannel; + private final EventHandler eventHandler; + private final ExecutorService serverExecutorService; + private final AtomicLong lastEventTimeStampMilli; + + private AcceptTask( + ExecutorService serverExecutorService, + AtomicLong lastEventTimeStampMilli, + EventHandler eventHandler, + SocketChannel acceptSocketChannel) { + this.serverExecutorService = serverExecutorService; + this.lastEventTimeStampMilli = lastEventTimeStampMilli; + this.eventHandler = eventHandler; + this.acceptSocketChannel = acceptSocketChannel; + } + + @Override + public String toString() { + return "AcceptTask{" + + "acceptSocketChannel=" + + acceptSocketChannel + + ", lastEventTimeStampMilli=" + + lastEventTimeStampMilli + + '}'; + } + + private void close() throws IOException { + acceptSocketChannel.close(); + } + + @Override + public void run() { + LOG.debug("AcceptTask: connected. this={}", this); + try { + eventHandler.onConnect(acceptSocketChannel); + ByteBuffer byteBuffer = ByteBuffer.allocate(512 * 1024); + while (!serverExecutorService.isShutdown()) { try { - serverSocketChannel.close(); - } - finally { - Files.deleteIfExists(socketAddress.getPath()); - } - } - - private static class AcceptTask - implements Runnable - { - private final SocketChannel acceptSocketChannel; - private final EventHandler eventHandler; - private final ExecutorService serverExecutorService; - private final AtomicLong lastEventTimeStampMilli; - - private AcceptTask(ExecutorService serverExecutorService, AtomicLong lastEventTimeStampMilli, EventHandler eventHandler, SocketChannel acceptSocketChannel) - { - this.serverExecutorService = serverExecutorService; - this.lastEventTimeStampMilli = lastEventTimeStampMilli; - this.eventHandler = eventHandler; - this.acceptSocketChannel = acceptSocketChannel; - } - - @Override - public String toString() - { - return "AcceptTask{" + - "acceptSocketChannel=" + acceptSocketChannel + - ", lastEventTimeStampMilli=" + lastEventTimeStampMilli + - '}'; - } - - private void close() - throws IOException - { - acceptSocketChannel.close(); - } - - @Override - public void run() - { - LOG.debug("AcceptTask: connected. this={}", this); + int len = acceptSocketChannel.read(byteBuffer); + if (len <= 0) { + LOG.debug( + "AcceptTask: closed. len={}, this={}, remote={}", + len, + this, + acceptSocketChannel); + eventHandler.onClose(acceptSocketChannel); try { - eventHandler.onConnect(acceptSocketChannel); - ByteBuffer byteBuffer = ByteBuffer.allocate(512 * 1024); - while (!serverExecutorService.isShutdown()) { - try { - int len = acceptSocketChannel.read(byteBuffer); - if (len <= 0) { - LOG.debug("AcceptTask: closed. len={}, this={}, remote={}", len, this, acceptSocketChannel); - eventHandler.onClose(acceptSocketChannel); - try { - close(); - } - catch (IOException e) { - LOG.warn("AcceptTask: close() failed: this={}", this, e); - } - break; - } - else { - eventHandler.onReceive(acceptSocketChannel, byteBuffer); - lastEventTimeStampMilli.set(System.currentTimeMillis()); - } - } - catch (IOException e) { - LOG.warn("AcceptTask: recv() failed: this={}, message={}, cause={}", - this, e.getMessage(), e.getCause() == null ? "" : e.getCause().getMessage()); - if (!acceptSocketChannel.isOpen()) { - eventHandler.onClose(acceptSocketChannel); - throw new RuntimeException(e); - } - } - } - } - finally { - try { - LOG.debug("AcceptTask: Finished. Closing... this={}, remote={}", this, acceptSocketChannel); - close(); - } - catch (IOException e) { - LOG.warn("AcceptTask: close() failed", e); - } + close(); + } catch (IOException e) { + LOG.warn("AcceptTask: close() failed: this={}", this, e); } + break; + } else { + eventHandler.onReceive(acceptSocketChannel, byteBuffer); + lastEventTimeStampMilli.set(System.currentTimeMillis()); + } + } catch (IOException e) { + LOG.warn( + "AcceptTask: recv() failed: this={}, message={}, cause={}", + this, + e.getMessage(), + e.getCause() == null ? "" : e.getCause().getMessage()); + if (!acceptSocketChannel.isOpen()) { + eventHandler.onClose(acceptSocketChannel); + throw new RuntimeException(e); + } } + } + } finally { + try { + LOG.debug( + "AcceptTask: Finished. Closing... this={}, remote={}", this, acceptSocketChannel); + close(); + } catch (IOException e) { + LOG.warn("AcceptTask: close() failed", e); + } } + } } + } } diff --git a/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/UnixSocketSenderTest.java b/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/UnixSocketSenderTest.java index 4c5fa74c..1c61e62d 100644 --- a/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/UnixSocketSenderTest.java +++ b/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/UnixSocketSenderTest.java @@ -16,14 +16,8 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.fluentd.MockUnixSocketServer; -import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; -import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.PhiAccrualFailureDetectStrategy; -import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.UnixSocketHeartbeater; -import org.komamitsu.fluency.util.Tuple; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.*; import java.io.IOException; import java.net.SocketException; @@ -35,259 +29,259 @@ import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.fluentd.MockUnixSocketServer; +import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; +import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.PhiAccrualFailureDetectStrategy; +import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.UnixSocketHeartbeater; +import org.komamitsu.fluency.util.Tuple; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.*; - -class UnixSocketSenderTest -{ - private static final Logger LOG = LoggerFactory.getLogger(UnixSocketSenderTest.class); - - - @Test - void testSend() - throws Exception - { - testSendBase(socketPath -> { - UnixSocketSender.Config senderConfig = new UnixSocketSender.Config(); - senderConfig.setPath(socketPath); - return new UnixSocketSender(senderConfig); - }, - count -> assertThat(count).isEqualTo(1), - count -> assertThat(count).isEqualTo(1)); - } - - @Test - void testSendWithHeartbeart() - throws Exception - { - testSendBase(socketPath -> { - UnixSocketHeartbeater.Config hbConfig = new UnixSocketHeartbeater.Config(); - hbConfig.setPath(socketPath); - hbConfig.setIntervalMillis(400); - - UnixSocketSender.Config senderConfig = new UnixSocketSender.Config(); - senderConfig.setPath(socketPath); - - return new UnixSocketSender(senderConfig, - new FailureDetector( - new PhiAccrualFailureDetectStrategy(), - new UnixSocketHeartbeater(hbConfig))); - }, - count -> assertThat(count).isGreaterThan(1), - count -> assertThat(count).isGreaterThan(1)); - } - - private void testSendBase( - SenderCreator senderCreator, - Consumer connectCountAssertion, - Consumer closeCountAssertion) - throws Exception - { - try (MockUnixSocketServer server = new MockUnixSocketServer()) { - server.start(); - - int concurrency = 10; - final int reqNum = 5000; - final CountDownLatch latch = new CountDownLatch(concurrency); - UnixSocketSender sender = senderCreator.create(server.getSocketPath()); - - // To receive heartbeat at least once - TimeUnit.MILLISECONDS.sleep(500); - - final ExecutorService senderExecutorService = Executors.newCachedThreadPool(); - for (int i = 0; i < concurrency; i++) { - senderExecutorService.execute(() -> { - try { - byte[] bytes = "0123456789".getBytes(); - - for (int j = 0; j < reqNum; j++) { - sender.send(ByteBuffer.wrap(bytes)); - } - latch.countDown(); - } catch (IOException e) { - LOG.error("Failed to send data", e); - } - }); - } - - assertTrue(latch.await(4, TimeUnit.SECONDS)); - sender.close(); - - server.waitUntilEventsStop(); - server.stop(); - - int connectCount = 0; - int closeCount = 0; - long recvCount = 0; - long recvLen = 0; - for (Tuple event : server.getEvents()) { - switch (event.getFirst()) { - case CONNECT: - connectCount++; - break; - case CLOSE: - closeCount++; - break; - case RECEIVE: - recvCount++; - recvLen += event.getSecond(); - break; +class UnixSocketSenderTest { + private static final Logger LOG = LoggerFactory.getLogger(UnixSocketSenderTest.class); + + @Test + void testSend() throws Exception { + testSendBase( + socketPath -> { + UnixSocketSender.Config senderConfig = new UnixSocketSender.Config(); + senderConfig.setPath(socketPath); + return new UnixSocketSender(senderConfig); + }, + count -> assertThat(count).isEqualTo(1), + count -> assertThat(count).isEqualTo(1)); + } + + @Test + void testSendWithHeartbeart() throws Exception { + testSendBase( + socketPath -> { + UnixSocketHeartbeater.Config hbConfig = new UnixSocketHeartbeater.Config(); + hbConfig.setPath(socketPath); + hbConfig.setIntervalMillis(400); + + UnixSocketSender.Config senderConfig = new UnixSocketSender.Config(); + senderConfig.setPath(socketPath); + + return new UnixSocketSender( + senderConfig, + new FailureDetector( + new PhiAccrualFailureDetectStrategy(), new UnixSocketHeartbeater(hbConfig))); + }, + count -> assertThat(count).isGreaterThan(1), + count -> assertThat(count).isGreaterThan(1)); + } + + private void testSendBase( + SenderCreator senderCreator, + Consumer connectCountAssertion, + Consumer closeCountAssertion) + throws Exception { + try (MockUnixSocketServer server = new MockUnixSocketServer()) { + server.start(); + + int concurrency = 10; + final int reqNum = 5000; + final CountDownLatch latch = new CountDownLatch(concurrency); + UnixSocketSender sender = senderCreator.create(server.getSocketPath()); + + // To receive heartbeat at least once + TimeUnit.MILLISECONDS.sleep(500); + + final ExecutorService senderExecutorService = Executors.newCachedThreadPool(); + for (int i = 0; i < concurrency; i++) { + senderExecutorService.execute( + () -> { + try { + byte[] bytes = "0123456789".getBytes(); + + for (int j = 0; j < reqNum; j++) { + sender.send(ByteBuffer.wrap(bytes)); } - } - LOG.debug("recvCount={}", recvCount); - - connectCountAssertion.accept(connectCount); - assertThat(recvLen).isEqualTo((long) concurrency * reqNum * 10); - closeCountAssertion.accept(closeCount); + latch.countDown(); + } catch (IOException e) { + LOG.error("Failed to send data", e); + } + }); + } + + assertTrue(latch.await(4, TimeUnit.SECONDS)); + sender.close(); + + server.waitUntilEventsStop(); + server.stop(); + + int connectCount = 0; + int closeCount = 0; + long recvCount = 0; + long recvLen = 0; + for (Tuple event : server.getEvents()) { + switch (event.getFirst()) { + case CONNECT: + connectCount++; + break; + case CLOSE: + closeCount++; + break; + case RECEIVE: + recvCount++; + recvLen += event.getSecond(); + break; } - } + } + LOG.debug("recvCount={}", recvCount); - @Test - void testReadTimeout() - throws Exception - { - try (MockUnixSocketServer server = new MockUnixSocketServer()) { - server.start(); - - try { - final CountDownLatch latch = new CountDownLatch(1); - ExecutorService executorService = Executors.newSingleThreadExecutor(); - executorService.execute(() -> { - UnixSocketSender.Config senderConfig = new UnixSocketSender.Config(); - senderConfig.setPath(server.getSocketPath()); - senderConfig.setReadTimeoutMilli(1000); - UnixSocketSender sender = new UnixSocketSender(senderConfig); - try { - sender.sendWithAck(Arrays.asList(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))), "Waiting ack forever"); - } catch (Throwable e) { - if (e instanceof SocketTimeoutException) { - latch.countDown(); - } else { - throw new RuntimeException(e); - } - } - }); - assertTrue(latch.await(2000, TimeUnit.MILLISECONDS)); - } finally { - server.stop(); - } - } + connectCountAssertion.accept(connectCount); + assertThat(recvLen).isEqualTo((long) concurrency * reqNum * 10); + closeCountAssertion.accept(closeCount); } - - private Throwable extractRootCause(Throwable exception) - { - Throwable e = exception; - while (e.getCause() != null) { - e = e.getCause(); - } - return e; + } + + @Test + void testReadTimeout() throws Exception { + try (MockUnixSocketServer server = new MockUnixSocketServer()) { + server.start(); + + try { + final CountDownLatch latch = new CountDownLatch(1); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + executorService.execute( + () -> { + UnixSocketSender.Config senderConfig = new UnixSocketSender.Config(); + senderConfig.setPath(server.getSocketPath()); + senderConfig.setReadTimeoutMilli(1000); + UnixSocketSender sender = new UnixSocketSender(senderConfig); + try { + sender.sendWithAck( + Arrays.asList(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))), + "Waiting ack forever"); + } catch (Throwable e) { + if (e instanceof SocketTimeoutException) { + latch.countDown(); + } else { + throw new RuntimeException(e); + } + } + }); + assertTrue(latch.await(2000, TimeUnit.MILLISECONDS)); + } finally { + server.stop(); + } } + } - @Test - void testDisconnBeforeRecv() - throws Exception - { - try (MockUnixSocketServer server = new MockUnixSocketServer()) { - server.start(); - - try { - final CountDownLatch latch = new CountDownLatch(1); - ExecutorService executorService = Executors.newSingleThreadExecutor(); - executorService.execute(() -> { - UnixSocketSender.Config senderConfig = new UnixSocketSender.Config(); - senderConfig.setPath(server.getSocketPath()); - senderConfig.setReadTimeoutMilli(4000); - UnixSocketSender sender = new UnixSocketSender(senderConfig); - try { - sender.sendWithAck(Arrays.asList(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))), "Waiting ack forever"); - } catch (Throwable e) { - Throwable rootCause = extractRootCause(e); - if (rootCause instanceof SocketException && rootCause.getMessage().toLowerCase().contains("disconnected")) { - latch.countDown(); - } else { - throw new RuntimeException(e); - } - } - }); + private Throwable extractRootCause(Throwable exception) { + Throwable e = exception; + while (e.getCause() != null) { + e = e.getCause(); + } + return e; + } + + @Test + void testDisconnBeforeRecv() throws Exception { + try (MockUnixSocketServer server = new MockUnixSocketServer()) { + server.start(); + + try { + final CountDownLatch latch = new CountDownLatch(1); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + executorService.execute( + () -> { + UnixSocketSender.Config senderConfig = new UnixSocketSender.Config(); + senderConfig.setPath(server.getSocketPath()); + senderConfig.setReadTimeoutMilli(4000); + UnixSocketSender sender = new UnixSocketSender(senderConfig); + try { + sender.sendWithAck( + Arrays.asList(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))), + "Waiting ack forever"); + } catch (Throwable e) { + Throwable rootCause = extractRootCause(e); + if (rootCause instanceof SocketException + && rootCause.getMessage().toLowerCase().contains("disconnected")) { + latch.countDown(); + } else { + throw new RuntimeException(e); + } + } + }); - TimeUnit.MILLISECONDS.sleep(1000); - server.stop(true); + TimeUnit.MILLISECONDS.sleep(1000); + server.stop(true); - assertTrue(latch.await(8000, TimeUnit.MILLISECONDS)); - } finally { - server.stop(); - } - } + assertTrue(latch.await(8000, TimeUnit.MILLISECONDS)); + } finally { + server.stop(); + } } - - @Test - void testClose() - throws Exception - { - try (MockUnixSocketServer server = new MockUnixSocketServer()) { - server.start(); - - try { - final AtomicLong duration = new AtomicLong(); - ExecutorService executorService = Executors.newSingleThreadExecutor(); - Future future = executorService.submit(() -> { - UnixSocketSender.Config senderConfig = new UnixSocketSender.Config(); - senderConfig.setPath(server.getSocketPath()); - senderConfig.setWaitBeforeCloseMilli(1500); - UnixSocketSender sender = new UnixSocketSender(senderConfig); - long start; - try { - sender.send(Arrays.asList(ByteBuffer.wrap("hello, world".getBytes("UTF-8")))); - start = System.currentTimeMillis(); - sender.close(); - duration.set(System.currentTimeMillis() - start); - } catch (Exception e) { - LOG.error("Unexpected exception", e); - } - - return null; + } + + @Test + void testClose() throws Exception { + try (MockUnixSocketServer server = new MockUnixSocketServer()) { + server.start(); + + try { + final AtomicLong duration = new AtomicLong(); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + Future future = + executorService.submit( + () -> { + UnixSocketSender.Config senderConfig = new UnixSocketSender.Config(); + senderConfig.setPath(server.getSocketPath()); + senderConfig.setWaitBeforeCloseMilli(1500); + UnixSocketSender sender = new UnixSocketSender(senderConfig); + long start; + try { + sender.send(Arrays.asList(ByteBuffer.wrap("hello, world".getBytes("UTF-8")))); + start = System.currentTimeMillis(); + sender.close(); + duration.set(System.currentTimeMillis() - start); + } catch (Exception e) { + LOG.error("Unexpected exception", e); + } + + return null; }); - future.get(3000, TimeUnit.MILLISECONDS); - assertTrue(duration.get() > 1000 && duration.get() < 2000); - } finally { - server.stop(); - } - } + future.get(3000, TimeUnit.MILLISECONDS); + assertTrue(duration.get() > 1000 && duration.get() < 2000); + } finally { + server.stop(); + } } + } + + @Test + void testConfig() { + UnixSocketSender.Config config = new UnixSocketSender.Config(); + assertEquals(1000, config.getWaitBeforeCloseMilli()); + // TODO: Add others later + } - @Test - void testConfig() + @Test + void validateConfig() { { - UnixSocketSender.Config config = new UnixSocketSender.Config(); - assertEquals(1000, config.getWaitBeforeCloseMilli()); - // TODO: Add others later + UnixSocketSender.Config config = new UnixSocketSender.Config(); + config.setConnectionTimeoutMilli(9); + assertThrows(IllegalArgumentException.class, () -> new UnixSocketSender(config)); } - @Test - void validateConfig() { - { - UnixSocketSender.Config config = new UnixSocketSender.Config(); - config.setConnectionTimeoutMilli(9); - assertThrows(IllegalArgumentException.class, () -> new UnixSocketSender(config)); - } - - { - UnixSocketSender.Config config = new UnixSocketSender.Config(); - config.setReadTimeoutMilli(9); - assertThrows(IllegalArgumentException.class, () -> new UnixSocketSender(config)); - } - - { - UnixSocketSender.Config config = new UnixSocketSender.Config(); - config.setWaitBeforeCloseMilli(-1); - assertThrows(IllegalArgumentException.class, () -> new UnixSocketSender(config)); - } + UnixSocketSender.Config config = new UnixSocketSender.Config(); + config.setReadTimeoutMilli(9); + assertThrows(IllegalArgumentException.class, () -> new UnixSocketSender(config)); } - interface SenderCreator { - UnixSocketSender create(Path socketPath); + UnixSocketSender.Config config = new UnixSocketSender.Config(); + config.setWaitBeforeCloseMilli(-1); + assertThrows(IllegalArgumentException.class, () -> new UnixSocketSender(config)); } + } + + interface SenderCreator { + UnixSocketSender create(Path socketPath); + } } diff --git a/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UnixSocketHeartbeaterTest.java b/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UnixSocketHeartbeaterTest.java index 3eb95a52..c10a529d 100644 --- a/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UnixSocketHeartbeaterTest.java +++ b/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UnixSocketHeartbeaterTest.java @@ -16,9 +16,8 @@ package org.komamitsu.fluency.fluentd.ingester.sender.heartbeat; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.net.StandardProtocolFamily; @@ -32,108 +31,97 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - +class UnixSocketHeartbeaterTest { + private Path socketPath; + private ServerSocketChannel serverSocketChannel; -class UnixSocketHeartbeaterTest -{ - private Path socketPath; - private ServerSocketChannel serverSocketChannel; + @BeforeEach + void setUp() throws IOException { + socketPath = + Paths.get( + System.getProperty("java.io.tmpdir"), + String.format("fluency-unixsocket-hb-test-%s", UUID.randomUUID())); + UnixDomainSocketAddress socketAddress = UnixDomainSocketAddress.of(socketPath); - @BeforeEach - void setUp() - throws IOException - { - socketPath = Paths.get(System.getProperty("java.io.tmpdir"), - String.format("fluency-unixsocket-hb-test-%s", UUID.randomUUID())); - UnixDomainSocketAddress socketAddress = UnixDomainSocketAddress.of(socketPath); + serverSocketChannel = ServerSocketChannel.open(StandardProtocolFamily.UNIX); + serverSocketChannel.bind(socketAddress); + } - serverSocketChannel = ServerSocketChannel.open(StandardProtocolFamily.UNIX); - serverSocketChannel.bind(socketAddress); - } + @AfterEach + void tearDown() throws IOException { + Files.deleteIfExists(socketPath); + } - @AfterEach - void tearDown() - throws IOException - { - Files.deleteIfExists(socketPath); - } - - @Test - void testHeartbeaterUp() - throws IOException, InterruptedException - { - final CountDownLatch latch = new CountDownLatch(2); - Executors.newSingleThreadExecutor().execute(() -> { - try { + @Test + void testHeartbeaterUp() throws IOException, InterruptedException { + final CountDownLatch latch = new CountDownLatch(2); + Executors.newSingleThreadExecutor() + .execute( + () -> { + try { serverSocketChannel.accept(); latch.countDown(); - } - catch (IOException e) { + } catch (IOException e) { e.printStackTrace(); - } - }); + } + }); - UnixSocketHeartbeater.Config config = new UnixSocketHeartbeater.Config(); - config.setPath(socketPath); - config.setIntervalMillis(500); - try (UnixSocketHeartbeater heartbeater = new UnixSocketHeartbeater(config)) { - final AtomicInteger pongCounter = new AtomicInteger(); - final AtomicInteger failureCounter = new AtomicInteger(); - heartbeater.setCallback(new Heartbeater.Callback() - { - @Override - public void onHeartbeat() - { - pongCounter.incrementAndGet(); - latch.countDown(); - } + UnixSocketHeartbeater.Config config = new UnixSocketHeartbeater.Config(); + config.setPath(socketPath); + config.setIntervalMillis(500); + try (UnixSocketHeartbeater heartbeater = new UnixSocketHeartbeater(config)) { + final AtomicInteger pongCounter = new AtomicInteger(); + final AtomicInteger failureCounter = new AtomicInteger(); + heartbeater.setCallback( + new Heartbeater.Callback() { + @Override + public void onHeartbeat() { + pongCounter.incrementAndGet(); + latch.countDown(); + } - @Override - public void onFailure(Throwable cause) - { - failureCounter.incrementAndGet(); - } - }); - heartbeater.start(); - assertTrue(latch.await(5, TimeUnit.SECONDS)); - assertTrue(0 < pongCounter.get() && pongCounter.get() < 3); - assertEquals(0, failureCounter.get()); - } + @Override + public void onFailure(Throwable cause) { + failureCounter.incrementAndGet(); + } + }); + heartbeater.start(); + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(0 < pongCounter.get() && pongCounter.get() < 3); + assertEquals(0, failureCounter.get()); } + } - @Test - void testHeartbeaterDown() - throws IOException, InterruptedException - { - serverSocketChannel.close(); + @Test + void testHeartbeaterDown() throws IOException, InterruptedException { + serverSocketChannel.close(); - UnixSocketHeartbeater.Config config = new UnixSocketHeartbeater.Config(); - config.setPath(socketPath); - config.setIntervalMillis(500); - try (UnixSocketHeartbeater heartbeater = new UnixSocketHeartbeater(config)) { - final AtomicInteger pongCounter = new AtomicInteger(); - final AtomicInteger failureCounter = new AtomicInteger(); - heartbeater.setCallback(new Heartbeater.Callback() - { - @Override - public void onHeartbeat() - { - pongCounter.incrementAndGet(); - } + UnixSocketHeartbeater.Config config = new UnixSocketHeartbeater.Config(); + config.setPath(socketPath); + config.setIntervalMillis(500); + try (UnixSocketHeartbeater heartbeater = new UnixSocketHeartbeater(config)) { + final AtomicInteger pongCounter = new AtomicInteger(); + final AtomicInteger failureCounter = new AtomicInteger(); + heartbeater.setCallback( + new Heartbeater.Callback() { + @Override + public void onHeartbeat() { + pongCounter.incrementAndGet(); + } - @Override - public void onFailure(Throwable cause) - { - failureCounter.incrementAndGet(); - } - }); - heartbeater.start(); - TimeUnit.SECONDS.sleep(1); - assertEquals(0, pongCounter.get()); - assertTrue(failureCounter.get() > 0); - } + @Override + public void onFailure(Throwable cause) { + failureCounter.incrementAndGet(); + } + }); + heartbeater.start(); + TimeUnit.SECONDS.sleep(1); + assertEquals(0, pongCounter.get()); + assertTrue(failureCounter.get() > 0); } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/FluencyBuilderForFluentd.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/FluencyBuilderForFluentd.java index eac64f4d..b746701d 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/FluencyBuilderForFluentd.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/FluencyBuilderForFluentd.java @@ -16,6 +16,10 @@ package org.komamitsu.fluency.fluentd; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.List; +import javax.net.ssl.SSLSocketFactory; import org.komamitsu.fluency.Fluency; import org.komamitsu.fluency.fluentd.ingester.FluentdIngester; import org.komamitsu.fluency.fluentd.ingester.sender.FluentdSender; @@ -31,249 +35,213 @@ import org.komamitsu.fluency.fluentd.recordformat.FluentdRecordFormatter; import org.komamitsu.fluency.ingester.Ingester; -import java.net.InetSocketAddress; -import javax.net.ssl.SSLSocketFactory; -import java.util.ArrayList; -import java.util.List; - -public class FluencyBuilderForFluentd - extends org.komamitsu.fluency.FluencyBuilder -{ - private Integer senderMaxRetryCount; - private Integer senderBaseRetryIntervalMillis; - private Integer senderMaxRetryIntervalMillis; - private boolean ackResponseMode; - private boolean sslEnabled; - private SSLSocketFactory sslSocketFactory; - protected Integer connectionTimeoutMilli; - protected Integer readTimeoutMilli; - protected FluentdRecordFormatter recordFormatter = new FluentdRecordFormatter(); - - public Integer getSenderMaxRetryCount() - { - return senderMaxRetryCount; - } - - public Integer getSenderBaseRetryIntervalMillis() - { - return senderBaseRetryIntervalMillis; - } - - public Integer getSenderMaxRetryIntervalMillis() - { - return senderMaxRetryIntervalMillis; - } - - public void setSenderBaseRetryIntervalMillis(Integer senderBaseRetryIntervalMillis) - { - this.senderBaseRetryIntervalMillis = senderBaseRetryIntervalMillis; - } - - public void setSenderMaxRetryIntervalMillis(Integer senderMaxRetryIntervalMillis) - { - this.senderMaxRetryIntervalMillis = senderMaxRetryIntervalMillis; - } - - public void setSenderMaxRetryCount(Integer senderMaxRetryCount) - { - this.senderMaxRetryCount = senderMaxRetryCount; - } - - public boolean isAckResponseMode() - { - return ackResponseMode; - } - - public void setAckResponseMode(boolean ackResponseMode) - { - this.ackResponseMode = ackResponseMode; - } - - public boolean isSslEnabled() - { - return sslEnabled; - } - - public void setSslEnabled(boolean sslEnabled) - { - this.sslEnabled = sslEnabled; - } - - public void setSslSocketFactory(SSLSocketFactory sslSocketFactory) - { - this.sslSocketFactory = sslSocketFactory; - } - - public Integer getConnectionTimeoutMilli() - { - return connectionTimeoutMilli; - } - - public void setConnectionTimeoutMilli(Integer connectionTimeoutMilli) - { - this.connectionTimeoutMilli = connectionTimeoutMilli; - } - - public Integer getReadTimeoutMilli() - { - return readTimeoutMilli; - } - - public void setReadTimeoutMilli(Integer readTimeoutMilli) - { - this.readTimeoutMilli = readTimeoutMilli; - } - - public FluentdRecordFormatter getRecordFormatter() - { - return recordFormatter; - } - - public void setRecordFormatter(FluentdRecordFormatter recordFormatter) - { - this.recordFormatter = recordFormatter; - } - - public Fluency build(String host, int port) - { - return buildFromIngester( - recordFormatter, - buildIngester(createBaseSender(host, port))); - } - - public Fluency build(int port) - { - return buildFromIngester( - recordFormatter, - buildIngester(createBaseSender(null, port))); - } - - public Fluency build() - { - return buildFromIngester( - recordFormatter, - buildIngester(createBaseSender(null, null))); - } - - public Fluency build(List servers) - { - List senders = new ArrayList<>(); - for (InetSocketAddress server : servers) { - senders.add(createBaseSender(server.getHostName(), server.getPort(), true)); - } - return buildFromIngester( - recordFormatter, - buildIngester(new MultiSender(senders))); - } - - private FluentdSender createBaseSender(String host, Integer port) - { - return createBaseSender(host, port, false); - } - - private FluentdSender createBaseSender(String host, Integer port, boolean withHeartBeater) - { - if (withHeartBeater && port == null) { - throw new IllegalArgumentException("`port` should be specified when using heartbeat"); - } - - if (isSslEnabled()) { - SSLSender.Config senderConfig = new SSLSender.Config(); - FailureDetector failureDetector = null; - if (host != null) { - senderConfig.setHost(host); - } - if (port != null) { - senderConfig.setPort(port); - } - if (sslSocketFactory != null) { - senderConfig.setSslSocketFactory(sslSocketFactory); - } - if (withHeartBeater) { - SSLHeartbeater.Config hbConfig = new SSLHeartbeater.Config(); - hbConfig.setHost(host); - hbConfig.setPort(port); - - if (sslSocketFactory != null) { - hbConfig.setSslSocketFactory(sslSocketFactory); - } - - SSLHeartbeater heartbeater = new SSLHeartbeater(hbConfig); - failureDetector = new FailureDetector(new PhiAccrualFailureDetectStrategy(), heartbeater); - } - if (connectionTimeoutMilli != null) { - senderConfig.setConnectionTimeoutMilli(connectionTimeoutMilli); - } - if (readTimeoutMilli != null) { - senderConfig.setReadTimeoutMilli(readTimeoutMilli); - } - return new SSLSender(senderConfig, failureDetector); - } - else { - TCPSender.Config senderConfig = new TCPSender.Config(); - FailureDetector failureDetector = null; - if (host != null) { - senderConfig.setHost(host); - } - if (port != null) { - senderConfig.setPort(port); - } - if (withHeartBeater) { - TCPHeartbeater.Config hbConfig = new TCPHeartbeater.Config(); - hbConfig.setHost(host); - hbConfig.setPort(port); - TCPHeartbeater heartbeater = new TCPHeartbeater(hbConfig); - failureDetector = new FailureDetector(new PhiAccrualFailureDetectStrategy(), heartbeater); - } - if (connectionTimeoutMilli != null) { - senderConfig.setConnectionTimeoutMilli(connectionTimeoutMilli); - } - if (readTimeoutMilli != null) { - senderConfig.setReadTimeoutMilli(readTimeoutMilli); - } - return new TCPSender(senderConfig, failureDetector); - } - } - - @Override - public String toString() - { - return "FluencyBuilder{" + - "senderMaxRetryCount=" + senderMaxRetryCount + - ", ackResponseMode=" + ackResponseMode + - ", sslEnabled=" + sslEnabled + - "} " + super.toString(); - } - - protected Ingester buildIngester(FluentdSender baseSender) - { - ExponentialBackOffRetryStrategy.Config retryStrategyConfig = - new ExponentialBackOffRetryStrategy.Config(); - - if (getSenderMaxRetryCount() != null) { - retryStrategyConfig.setMaxRetryCount(getSenderMaxRetryCount()); - } - - if (getSenderBaseRetryIntervalMillis() != null) { - retryStrategyConfig.setBaseIntervalMillis(getSenderBaseRetryIntervalMillis()); - } - - if (getSenderMaxRetryIntervalMillis() != null) { - retryStrategyConfig.setMaxIntervalMillis(getSenderMaxRetryIntervalMillis()); +public class FluencyBuilderForFluentd extends org.komamitsu.fluency.FluencyBuilder { + private Integer senderMaxRetryCount; + private Integer senderBaseRetryIntervalMillis; + private Integer senderMaxRetryIntervalMillis; + private boolean ackResponseMode; + private boolean sslEnabled; + private SSLSocketFactory sslSocketFactory; + protected Integer connectionTimeoutMilli; + protected Integer readTimeoutMilli; + protected FluentdRecordFormatter recordFormatter = new FluentdRecordFormatter(); + + public Integer getSenderMaxRetryCount() { + return senderMaxRetryCount; + } + + public Integer getSenderBaseRetryIntervalMillis() { + return senderBaseRetryIntervalMillis; + } + + public Integer getSenderMaxRetryIntervalMillis() { + return senderMaxRetryIntervalMillis; + } + + public void setSenderBaseRetryIntervalMillis(Integer senderBaseRetryIntervalMillis) { + this.senderBaseRetryIntervalMillis = senderBaseRetryIntervalMillis; + } + + public void setSenderMaxRetryIntervalMillis(Integer senderMaxRetryIntervalMillis) { + this.senderMaxRetryIntervalMillis = senderMaxRetryIntervalMillis; + } + + public void setSenderMaxRetryCount(Integer senderMaxRetryCount) { + this.senderMaxRetryCount = senderMaxRetryCount; + } + + public boolean isAckResponseMode() { + return ackResponseMode; + } + + public void setAckResponseMode(boolean ackResponseMode) { + this.ackResponseMode = ackResponseMode; + } + + public boolean isSslEnabled() { + return sslEnabled; + } + + public void setSslEnabled(boolean sslEnabled) { + this.sslEnabled = sslEnabled; + } + + public void setSslSocketFactory(SSLSocketFactory sslSocketFactory) { + this.sslSocketFactory = sslSocketFactory; + } + + public Integer getConnectionTimeoutMilli() { + return connectionTimeoutMilli; + } + + public void setConnectionTimeoutMilli(Integer connectionTimeoutMilli) { + this.connectionTimeoutMilli = connectionTimeoutMilli; + } + + public Integer getReadTimeoutMilli() { + return readTimeoutMilli; + } + + public void setReadTimeoutMilli(Integer readTimeoutMilli) { + this.readTimeoutMilli = readTimeoutMilli; + } + + public FluentdRecordFormatter getRecordFormatter() { + return recordFormatter; + } + + public void setRecordFormatter(FluentdRecordFormatter recordFormatter) { + this.recordFormatter = recordFormatter; + } + + public Fluency build(String host, int port) { + return buildFromIngester(recordFormatter, buildIngester(createBaseSender(host, port))); + } + + public Fluency build(int port) { + return buildFromIngester(recordFormatter, buildIngester(createBaseSender(null, port))); + } + + public Fluency build() { + return buildFromIngester(recordFormatter, buildIngester(createBaseSender(null, null))); + } + + public Fluency build(List servers) { + List senders = new ArrayList<>(); + for (InetSocketAddress server : servers) { + senders.add(createBaseSender(server.getHostName(), server.getPort(), true)); + } + return buildFromIngester(recordFormatter, buildIngester(new MultiSender(senders))); + } + + private FluentdSender createBaseSender(String host, Integer port) { + return createBaseSender(host, port, false); + } + + private FluentdSender createBaseSender(String host, Integer port, boolean withHeartBeater) { + if (withHeartBeater && port == null) { + throw new IllegalArgumentException("`port` should be specified when using heartbeat"); + } + + if (isSslEnabled()) { + SSLSender.Config senderConfig = new SSLSender.Config(); + FailureDetector failureDetector = null; + if (host != null) { + senderConfig.setHost(host); + } + if (port != null) { + senderConfig.setPort(port); + } + if (sslSocketFactory != null) { + senderConfig.setSslSocketFactory(sslSocketFactory); + } + if (withHeartBeater) { + SSLHeartbeater.Config hbConfig = new SSLHeartbeater.Config(); + hbConfig.setHost(host); + hbConfig.setPort(port); + + if (sslSocketFactory != null) { + hbConfig.setSslSocketFactory(sslSocketFactory); } - RetryableSender.Config senderConfig = new RetryableSender.Config(); - - if (getErrorHandler() != null) { - senderConfig.setErrorHandler(getErrorHandler()); - } - - RetryableSender retryableSender = new RetryableSender(senderConfig, baseSender, - new ExponentialBackOffRetryStrategy(retryStrategyConfig)); - - FluentdIngester.Config ingesterConfig = new FluentdIngester.Config(); - ingesterConfig.setAckResponseMode(isAckResponseMode()); - - return new FluentdIngester(ingesterConfig, retryableSender); - } + SSLHeartbeater heartbeater = new SSLHeartbeater(hbConfig); + failureDetector = new FailureDetector(new PhiAccrualFailureDetectStrategy(), heartbeater); + } + if (connectionTimeoutMilli != null) { + senderConfig.setConnectionTimeoutMilli(connectionTimeoutMilli); + } + if (readTimeoutMilli != null) { + senderConfig.setReadTimeoutMilli(readTimeoutMilli); + } + return new SSLSender(senderConfig, failureDetector); + } else { + TCPSender.Config senderConfig = new TCPSender.Config(); + FailureDetector failureDetector = null; + if (host != null) { + senderConfig.setHost(host); + } + if (port != null) { + senderConfig.setPort(port); + } + if (withHeartBeater) { + TCPHeartbeater.Config hbConfig = new TCPHeartbeater.Config(); + hbConfig.setHost(host); + hbConfig.setPort(port); + TCPHeartbeater heartbeater = new TCPHeartbeater(hbConfig); + failureDetector = new FailureDetector(new PhiAccrualFailureDetectStrategy(), heartbeater); + } + if (connectionTimeoutMilli != null) { + senderConfig.setConnectionTimeoutMilli(connectionTimeoutMilli); + } + if (readTimeoutMilli != null) { + senderConfig.setReadTimeoutMilli(readTimeoutMilli); + } + return new TCPSender(senderConfig, failureDetector); + } + } + + @Override + public String toString() { + return "FluencyBuilder{" + + "senderMaxRetryCount=" + + senderMaxRetryCount + + ", ackResponseMode=" + + ackResponseMode + + ", sslEnabled=" + + sslEnabled + + "} " + + super.toString(); + } + + protected Ingester buildIngester(FluentdSender baseSender) { + ExponentialBackOffRetryStrategy.Config retryStrategyConfig = + new ExponentialBackOffRetryStrategy.Config(); + + if (getSenderMaxRetryCount() != null) { + retryStrategyConfig.setMaxRetryCount(getSenderMaxRetryCount()); + } + + if (getSenderBaseRetryIntervalMillis() != null) { + retryStrategyConfig.setBaseIntervalMillis(getSenderBaseRetryIntervalMillis()); + } + + if (getSenderMaxRetryIntervalMillis() != null) { + retryStrategyConfig.setMaxIntervalMillis(getSenderMaxRetryIntervalMillis()); + } + + RetryableSender.Config senderConfig = new RetryableSender.Config(); + + if (getErrorHandler() != null) { + senderConfig.setErrorHandler(getErrorHandler()); + } + + RetryableSender retryableSender = + new RetryableSender( + senderConfig, baseSender, new ExponentialBackOffRetryStrategy(retryStrategyConfig)); + + FluentdIngester.Config ingesterConfig = new FluentdIngester.Config(); + ingesterConfig.setAckResponseMode(isAckResponseMode()); + + return new FluentdIngester(ingesterConfig, retryableSender); + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/FluentdIngester.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/FluentdIngester.java index 23a6f5a5..fb01185a 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/FluentdIngester.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/FluentdIngester.java @@ -17,6 +17,12 @@ package org.komamitsu.fluency.fluentd.ingester; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; import org.komamitsu.fluency.fluentd.ingester.sender.FluentdSender; import org.komamitsu.fluency.fluentd.ingester.sender.RequestOption; import org.komamitsu.fluency.ingester.Ingester; @@ -25,100 +31,81 @@ import org.msgpack.core.MessagePacker; import org.msgpack.jackson.dataformat.MessagePackFactory; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.List; -import java.util.UUID; - -public class FluentdIngester - implements Ingester -{ - private final Config config; - private final FluentdSender sender; - private final ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); - - public FluentdIngester(FluentdSender sender) - { - this(new Config(), sender); +public class FluentdIngester implements Ingester { + private final Config config; + private final FluentdSender sender; + private final ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); + + public FluentdIngester(FluentdSender sender) { + this(new Config(), sender); + } + + public FluentdIngester(Config config, FluentdSender sender) { + this.config = config; + this.sender = sender; + } + + @Override + public void ingest(String tag, ByteBuffer dataBuffer) throws IOException { + ByteArrayOutputStream header = new ByteArrayOutputStream(); + MessagePacker messagePacker = MessagePack.newDefaultPacker(header); + + int dataLength = dataBuffer.limit(); + messagePacker.packArrayHeader(3); + messagePacker.packString(tag); + messagePacker.packRawStringHeader(dataLength); + messagePacker.flush(); + + ByteBuffer headerBuffer = ByteBuffer.wrap(header.toByteArray()); + + if (config.isAckResponseMode()) { + // The spec (https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#entry) + // says to encode a 128 bit value as base64, but fluent-bit currently does something different + // and in fluent-bit and fluentd there is no validation. So for now we keep it simple, see + // discussion on issue #181. + String token = UUID.randomUUID().toString(); + + ByteBuffer optionBuffer = + ByteBuffer.wrap(objectMapper.writeValueAsBytes(new RequestOption(dataLength, token))); + List buffers = Arrays.asList(headerBuffer, dataBuffer, optionBuffer); + + synchronized (sender) { + sender.sendWithAck(buffers, token); + } + } else { + ByteBuffer optionBuffer = + ByteBuffer.wrap(objectMapper.writeValueAsBytes(new RequestOption(dataLength, null))); + List buffers = Arrays.asList(headerBuffer, dataBuffer, optionBuffer); + + synchronized (sender) { + sender.send(buffers); + } } + } - public FluentdIngester(Config config, FluentdSender sender) - { - this.config = config; - this.sender = sender; - } + @Override + public Sender getSender() { + return sender; + } - @Override - public void ingest(String tag, ByteBuffer dataBuffer) - throws IOException - { - ByteArrayOutputStream header = new ByteArrayOutputStream(); - MessagePacker messagePacker = MessagePack.newDefaultPacker(header); - - int dataLength = dataBuffer.limit(); - messagePacker.packArrayHeader(3); - messagePacker.packString(tag); - messagePacker.packRawStringHeader(dataLength); - messagePacker.flush(); - - ByteBuffer headerBuffer = ByteBuffer.wrap(header.toByteArray()); - - if (config.isAckResponseMode()) { - // The spec (https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#entry) - // says to encode a 128 bit value as base64, but fluent-bit currently does something different - // and in fluent-bit and fluentd there is no validation. So for now we keep it simple, see - // discussion on issue #181. - String token = UUID.randomUUID().toString(); - - ByteBuffer optionBuffer = ByteBuffer.wrap(objectMapper.writeValueAsBytes(new RequestOption(dataLength, token))); - List buffers = Arrays.asList(headerBuffer, dataBuffer, optionBuffer); - - synchronized (sender) { - sender.sendWithAck(buffers, token); - } - } - else { - ByteBuffer optionBuffer = ByteBuffer.wrap(objectMapper.writeValueAsBytes(new RequestOption(dataLength, null))); - List buffers = Arrays.asList(headerBuffer, dataBuffer, optionBuffer); - - synchronized (sender) { - sender.send(buffers); - } - } - } + public boolean isAckResponseMode() { + return config.isAckResponseMode(); + } - @Override - public Sender getSender() - { - return sender; - } + @Override + public void close() throws IOException { + sender.close(); + } - public boolean isAckResponseMode() - { - return config.isAckResponseMode(); - } + public static class Config { + private boolean ackResponseMode = false; - @Override - public void close() - throws IOException - { - sender.close(); + public boolean isAckResponseMode() { + return ackResponseMode; } - public static class Config - { - private boolean ackResponseMode = false; - - public boolean isAckResponseMode() - { - return ackResponseMode; - } - - public void setAckResponseMode(boolean ackResponseMode) - { - this.ackResponseMode = ackResponseMode; - } + public void setAckResponseMode(boolean ackResponseMode) { + this.ackResponseMode = ackResponseMode; } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/Response.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/Response.java index 9be0b116..434e4a50 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/Response.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/Response.java @@ -18,26 +18,20 @@ import com.fasterxml.jackson.annotation.JsonProperty; -public class Response -{ - private final String ack; +public class Response { + private final String ack; - public Response(@JsonProperty("ack") String ack) - { - this.ack = ack; - } + public Response(@JsonProperty("ack") String ack) { + this.ack = ack; + } - @JsonProperty("ack") - public String getAck() - { - return ack; - } + @JsonProperty("ack") + public String getAck() { + return ack; + } - @Override - public String toString() - { - return "ResponseOption{" + - "ack=" + ack + - '}'; - } + @Override + public String toString() { + return "ResponseOption{" + "ack=" + ack + '}'; + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/FluentdSender.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/FluentdSender.java index 136de816..b2d3d438 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/FluentdSender.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/FluentdSender.java @@ -16,94 +16,76 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.komamitsu.fluency.ingester.sender.ErrorHandler; -import org.komamitsu.fluency.ingester.sender.Sender; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import org.komamitsu.fluency.ingester.sender.ErrorHandler; +import org.komamitsu.fluency.ingester.sender.Sender; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public abstract class FluentdSender - implements Closeable, Sender -{ - private static final Logger LOG = LoggerFactory.getLogger(FluentdSender.class); - private final Config config; +public abstract class FluentdSender implements Closeable, Sender { + private static final Logger LOG = LoggerFactory.getLogger(FluentdSender.class); + private final Config config; - protected FluentdSender(Config config) - { - this.config = config; - } + protected FluentdSender(Config config) { + this.config = config; + } - protected FluentdSender() - { - this(new FluentdSender.Config()); - } + protected FluentdSender() { + this(new FluentdSender.Config()); + } - public synchronized void send(ByteBuffer buffer) - throws IOException - { - sendInternalWithRestoreBufferPositions(Arrays.asList(buffer), null); - } + public synchronized void send(ByteBuffer buffer) throws IOException { + sendInternalWithRestoreBufferPositions(Arrays.asList(buffer), null); + } - public synchronized void send(List buffers) - throws IOException - { - sendInternalWithRestoreBufferPositions(buffers, null); - } + public synchronized void send(List buffers) throws IOException { + sendInternalWithRestoreBufferPositions(buffers, null); + } + + public void sendWithAck(List buffers, String ackToken) throws IOException { + sendInternalWithRestoreBufferPositions(buffers, ackToken); + } - public void sendWithAck(List buffers, String ackToken) - throws IOException - { - sendInternalWithRestoreBufferPositions(buffers, ackToken); + private void sendInternalWithRestoreBufferPositions(List buffers, String ackToken) + throws IOException { + List positions = new ArrayList<>(buffers.size()); + for (ByteBuffer data : buffers) { + positions.add(data.position()); } - private void sendInternalWithRestoreBufferPositions(List buffers, String ackToken) - throws IOException - { - List positions = new ArrayList<>(buffers.size()); - for (ByteBuffer data : buffers) { - positions.add(data.position()); - } + try { + sendInternal(buffers, ackToken); + } catch (Exception e) { + for (int i = 0; i < buffers.size(); i++) { + buffers.get(i).position(positions.get(i)); + } + ErrorHandler errorHandler = config.getErrorHandler(); + if (errorHandler != null) { try { - sendInternal(buffers, ackToken); + errorHandler.handle(e); + } catch (Exception ex) { + LOG.warn("Failed to handle an error in the error handler {}", errorHandler, ex); } - catch (Exception e) { - for (int i = 0; i < buffers.size(); i++) { - buffers.get(i).position(positions.get(i)); - } - - ErrorHandler errorHandler = config.getErrorHandler(); - if (errorHandler != null) { - try { - errorHandler.handle(e); - } - catch (Exception ex) { - LOG.warn("Failed to handle an error in the error handler {}", errorHandler, ex); - } - } + } - if (e instanceof IOException) { - throw (IOException) e; - } - else { - throw new IOException(e); - } - } + if (e instanceof IOException) { + throw (IOException) e; + } else { + throw new IOException(e); + } } + } - public abstract boolean isAvailable(); + public abstract boolean isAvailable(); - abstract protected void sendInternal(List buffers, String ackToken) - throws IOException; + protected abstract void sendInternal(List buffers, String ackToken) + throws IOException; - public static class Config - extends Sender.Config - { - } + public static class Config extends Sender.Config {} } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/InetSocketSender.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/InetSocketSender.java index 95a869fb..98f5c911 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/InetSocketSender.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/InetSocketSender.java @@ -20,69 +20,51 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class InetSocketSender - extends NetworkSender -{ - private static final Logger LOG = LoggerFactory.getLogger(InetSocketSender.class); - private final Config config; +public abstract class InetSocketSender extends NetworkSender { + private static final Logger LOG = LoggerFactory.getLogger(InetSocketSender.class); + private final Config config; - public InetSocketSender(Config config, FailureDetector failureDetector) - { - super(config, failureDetector); - this.config = config; - } + public InetSocketSender(Config config, FailureDetector failureDetector) { + super(config, failureDetector); + this.config = config; + } - public String getHost() - { - return config.getHost(); - } + public String getHost() { + return config.getHost(); + } - public int getPort() - { - return config.getPort(); - } + public int getPort() { + return config.getPort(); + } - @Override - public String toString() - { - return "NetworkSender{" + - "config=" + config + - "} " + super.toString(); - } + @Override + public String toString() { + return "NetworkSender{" + "config=" + config + "} " + super.toString(); + } - public static class Config - extends NetworkSender.Config - { - private String host = "127.0.0.1"; - private int port = 24224; + public static class Config extends NetworkSender.Config { + private String host = "127.0.0.1"; + private int port = 24224; - public String getHost() - { - return host; - } + public String getHost() { + return host; + } - public void setHost(String host) - { - this.host = host; - } + public void setHost(String host) { + this.host = host; + } - public int getPort() - { - return port; - } + public int getPort() { + return port; + } - public void setPort(int port) - { - this.port = port; - } + public void setPort(int port) { + this.port = port; + } - @Override - public String toString() - { - return "Config{" + - "host='" + host + '\'' + - ", port=" + port + - "} " + super.toString(); - } + @Override + public String toString() { + return "Config{" + "host='" + host + '\'' + ", port=" + port + "} " + super.toString(); } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/MultiSender.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/MultiSender.java index 278429d8..f53abc31 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/MultiSender.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/MultiSender.java @@ -16,107 +16,84 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class MultiSender - extends FluentdSender -{ - private static final Logger LOG = LoggerFactory.getLogger(MultiSender.class); - private final List senders; +public class MultiSender extends FluentdSender { + private static final Logger LOG = LoggerFactory.getLogger(MultiSender.class); + private final List senders; - public MultiSender(List senders) - { - this(new Config(), senders); - } + public MultiSender(List senders) { + this(new Config(), senders); + } - public MultiSender(Config config, List senders) - { - super(config); - this.senders = senders; - } + public MultiSender(Config config, List senders) { + super(config); + this.senders = senders; + } - @Override - public boolean isAvailable() - { - return true; - } + @Override + public boolean isAvailable() { + return true; + } - @Override - protected synchronized void sendInternal(List buffers, String ackToken) - throws AllNodesUnavailableException - { - for (FluentdSender sender : senders) { - boolean isAvailable = sender.isAvailable(); - LOG.trace("send(): sender={}, isAvailable={}", sender, isAvailable); - if (isAvailable) { - try { - if (ackToken == null) { - sender.send(buffers); - } - else { - sender.sendWithAck(buffers, ackToken); - } - return; - } - catch (IOException e) { - LOG.warn("Failed to send: sender=" + sender + ". Trying to use next sender...", e); - } - } + @Override + protected synchronized void sendInternal(List buffers, String ackToken) + throws AllNodesUnavailableException { + for (FluentdSender sender : senders) { + boolean isAvailable = sender.isAvailable(); + LOG.trace("send(): sender={}, isAvailable={}", sender, isAvailable); + if (isAvailable) { + try { + if (ackToken == null) { + sender.send(buffers); + } else { + sender.sendWithAck(buffers, ackToken); + } + return; + } catch (IOException e) { + LOG.warn("Failed to send: sender=" + sender + ". Trying to use next sender...", e); } - throw new AllNodesUnavailableException("All nodes are unavailable"); + } } + throw new AllNodesUnavailableException("All nodes are unavailable"); + } - @Override - public void close() - throws IOException - { - IOException firstException = null; - for (FluentdSender sender : senders) { - try { - sender.close(); - } - catch (IOException e) { - if (firstException == null) { - firstException = e; - } - } - } - if (firstException != null) { - throw firstException; + @Override + public void close() throws IOException { + IOException firstException = null; + for (FluentdSender sender : senders) { + try { + sender.close(); + } catch (IOException e) { + if (firstException == null) { + firstException = e; } + } } - - public List getSenders() - { - return Collections.unmodifiableList(senders); + if (firstException != null) { + throw firstException; } + } - @Override - public String toString() - { - return "MultiSender{" + - "senders=" + senders + - "} " + super.toString(); - } + public List getSenders() { + return Collections.unmodifiableList(senders); + } - public static class AllNodesUnavailableException - extends IOException - { - public AllNodesUnavailableException(String s) - { - super(s); - } - } + @Override + public String toString() { + return "MultiSender{" + "senders=" + senders + "} " + super.toString(); + } - public static class Config - extends FluentdSender.Config - { + public static class AllNodesUnavailableException extends IOException { + public AllNodesUnavailableException(String s) { + super(s); } -} + } + public static class Config extends FluentdSender.Config {} +} diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/NetworkSender.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/NetworkSender.java index 8be5d26a..79916098 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/NetworkSender.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/NetworkSender.java @@ -17,15 +17,6 @@ package org.komamitsu.fluency.fluentd.ingester.sender; import com.fasterxml.jackson.databind.ObjectMapper; -import org.komamitsu.fluency.fluentd.ingester.Response; -import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; -import org.komamitsu.fluency.util.ExecutorServiceUtils; -import org.komamitsu.fluency.validation.Validatable; -import org.komamitsu.fluency.validation.annotation.Min; -import org.msgpack.jackson.dataformat.MessagePackFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.net.SocketTimeoutException; import java.nio.ByteBuffer; @@ -35,220 +26,202 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import org.komamitsu.fluency.fluentd.ingester.Response; +import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; +import org.komamitsu.fluency.util.ExecutorServiceUtils; +import org.komamitsu.fluency.validation.annotation.Min; +import org.msgpack.jackson.dataformat.MessagePackFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public abstract class NetworkSender - extends FluentdSender -{ - private static final Logger LOG = LoggerFactory.getLogger(NetworkSender.class); - private final byte[] optionBuffer = new byte[256]; - private final ExecutorService executorService = ExecutorServiceUtils.newSingleThreadDaemonExecutor(); +public abstract class NetworkSender extends FluentdSender { + private static final Logger LOG = LoggerFactory.getLogger(NetworkSender.class); + private final byte[] optionBuffer = new byte[256]; + private final ExecutorService executorService = + ExecutorServiceUtils.newSingleThreadDaemonExecutor(); - private final Config config; - private final FailureDetector failureDetector; - private final ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); + private final Config config; + private final FailureDetector failureDetector; + private final ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); - NetworkSender(FailureDetector failureDetector) - { - this(new Config(), failureDetector); - } + NetworkSender(FailureDetector failureDetector) { + this(new Config(), failureDetector); + } - NetworkSender(Config config, FailureDetector failureDetector) - { - super(config); - this.config = config; - this.failureDetector = failureDetector; - } + NetworkSender(Config config, FailureDetector failureDetector) { + super(config); + this.config = config; + this.failureDetector = failureDetector; + } - @Override - public boolean isAvailable() - { - return failureDetector == null || failureDetector.isAvailable(); - } + @Override + public boolean isAvailable() { + return failureDetector == null || failureDetector.isAvailable(); + } - abstract T getOrCreateSocketInternal() - throws IOException; + abstract T getOrCreateSocketInternal() throws IOException; - private synchronized T getOrCreateSocket() - throws IOException - { - return getOrCreateSocketInternal(); - } + private synchronized T getOrCreateSocket() throws IOException { + return getOrCreateSocketInternal(); + } - abstract void sendBuffers(T socket, List buffers) - throws IOException; + abstract void sendBuffers(T socket, List buffers) throws IOException; - abstract void recvResponse(T socket, ByteBuffer buffer) - throws IOException; + abstract void recvResponse(T socket, ByteBuffer buffer) throws IOException; - private void propagateFailure(Throwable e) - { - if (failureDetector != null) { - failureDetector.onFailure(e); - } + private void propagateFailure(Throwable e) { + if (failureDetector != null) { + failureDetector.onFailure(e); } + } - @Override - protected synchronized void sendInternal(List buffers, String ackToken) - throws IOException - { - long totalDataSize = buffers.stream().mapToInt(ByteBuffer::remaining).sum(); + @Override + protected synchronized void sendInternal(List buffers, String ackToken) + throws IOException { + long totalDataSize = buffers.stream().mapToInt(ByteBuffer::remaining).sum(); - try { - LOG.trace("send(): sender={}, totalDataSize={}", this, totalDataSize); - final T socket = getOrCreateSocket(); - sendBuffers(socket, buffers); + try { + LOG.trace("send(): sender={}, totalDataSize={}", this, totalDataSize); + final T socket = getOrCreateSocket(); + sendBuffers(socket, buffers); - if (ackToken == null) { - return; - } + if (ackToken == null) { + return; + } - // For ACK response mode - final ByteBuffer byteBuffer = ByteBuffer.wrap(optionBuffer); + // For ACK response mode + final ByteBuffer byteBuffer = ByteBuffer.wrap(optionBuffer); - Future future = executorService.submit(() -> { + Future future = + executorService.submit( + () -> { LOG.trace("recv(): sender={}", this); recvResponse(socket, byteBuffer); return null; - }); - - try { - future.get(config.getReadTimeoutMilli(), TimeUnit.MILLISECONDS); - } - catch (InterruptedException e) { - throw new IOException("InterruptedException occurred", e); - } - catch (ExecutionException e) { - throw new IOException("ExecutionException occurred", e); - } - catch (TimeoutException e) { - throw new SocketTimeoutException("Socket read timeout"); - } - - Response response = objectMapper.readValue(optionBuffer, Response.class); - if (!ackToken.equals(response.getAck())) { - throw new UnmatchedAckException("Ack tokens don't matched: expected=" + ", got=" + response.getAck()); - } - } - catch (IOException e) { - LOG.error("Failed to send {} bytes data", totalDataSize); - closeSocket(); - propagateFailure(e); - throw e; + }); + + try { + future.get(config.getReadTimeoutMilli(), TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + throw new IOException("InterruptedException occurred", e); + } catch (ExecutionException e) { + throw new IOException("ExecutionException occurred", e); + } catch (TimeoutException e) { + throw new SocketTimeoutException("Socket read timeout"); + } + + Response response = objectMapper.readValue(optionBuffer, Response.class); + if (!ackToken.equals(response.getAck())) { + throw new UnmatchedAckException( + "Ack tokens don't matched: expected=" + ", got=" + response.getAck()); + } + } catch (IOException e) { + LOG.error("Failed to send {} bytes data", totalDataSize); + closeSocket(); + propagateFailure(e); + throw e; + } + } + + abstract void closeSocket() throws IOException; + + @Override + public synchronized void close() throws IOException { + try { + // Wait to confirm unsent request is flushed + try { + TimeUnit.MILLISECONDS.sleep(config.getWaitBeforeCloseMilli()); + } catch (InterruptedException e) { + LOG.warn("Interrupted", e); + Thread.currentThread().interrupt(); + } + + closeSocket(); + } finally { + try { + if (failureDetector != null) { + failureDetector.close(); } + } finally { + ExecutorServiceUtils.finishExecutorService(executorService); + } + } + } + + public int getConnectionTimeoutMilli() { + return config.getConnectionTimeoutMilli(); + } + + public int getReadTimeoutMilli() { + return config.getReadTimeoutMilli(); + } + + public FailureDetector getFailureDetector() { + return failureDetector; + } + + @Override + public String toString() { + return "NetworkSender{" + + "config=" + + config + + ", failureDetector=" + + failureDetector + + "} " + + super.toString(); + } + + public static class UnmatchedAckException extends IOException { + public UnmatchedAckException(String message) { + super(message); } + } - abstract void closeSocket() - throws IOException; + public static class Config extends FluentdSender.Config { + @Min(10) + private int connectionTimeoutMilli = 5000; - @Override - public synchronized void close() - throws IOException - { - try { - // Wait to confirm unsent request is flushed - try { - TimeUnit.MILLISECONDS.sleep(config.getWaitBeforeCloseMilli()); - } - catch (InterruptedException e) { - LOG.warn("Interrupted", e); - Thread.currentThread().interrupt(); - } - - closeSocket(); - } - finally { - try { - if (failureDetector != null) { - failureDetector.close(); - } - } - finally { - ExecutorServiceUtils.finishExecutorService(executorService); - } - } - } + @Min(10) + private int readTimeoutMilli = 5000; - public int getConnectionTimeoutMilli() - { - return config.getConnectionTimeoutMilli(); - } + @Min(0) + private int waitBeforeCloseMilli = 1000; - public int getReadTimeoutMilli() - { - return config.getReadTimeoutMilli(); + public int getConnectionTimeoutMilli() { + return connectionTimeoutMilli; } - public FailureDetector getFailureDetector() - { - return failureDetector; + public void setConnectionTimeoutMilli(int connectionTimeoutMilli) { + this.connectionTimeoutMilli = connectionTimeoutMilli; } - @Override - public String toString() - { - return "NetworkSender{" + - "config=" + config + - ", failureDetector=" + failureDetector + - "} " + super.toString(); + public int getReadTimeoutMilli() { + return readTimeoutMilli; } - public static class UnmatchedAckException - extends IOException - { - public UnmatchedAckException(String message) - { - super(message); - } + public void setReadTimeoutMilli(int readTimeoutMilli) { + this.readTimeoutMilli = readTimeoutMilli; } - public static class Config - extends FluentdSender.Config - { - @Min(10) - private int connectionTimeoutMilli = 5000; - @Min(10) - private int readTimeoutMilli = 5000; - @Min(0) - private int waitBeforeCloseMilli = 1000; - - public int getConnectionTimeoutMilli() - { - return connectionTimeoutMilli; - } - - public void setConnectionTimeoutMilli(int connectionTimeoutMilli) - { - this.connectionTimeoutMilli = connectionTimeoutMilli; - } - - public int getReadTimeoutMilli() - { - return readTimeoutMilli; - } - - public void setReadTimeoutMilli(int readTimeoutMilli) - { - this.readTimeoutMilli = readTimeoutMilli; - } - - public int getWaitBeforeCloseMilli() - { - return waitBeforeCloseMilli; - } + public int getWaitBeforeCloseMilli() { + return waitBeforeCloseMilli; + } - public void setWaitBeforeCloseMilli(int waitBeforeCloseMilli) - { - this.waitBeforeCloseMilli = waitBeforeCloseMilli; - } + public void setWaitBeforeCloseMilli(int waitBeforeCloseMilli) { + this.waitBeforeCloseMilli = waitBeforeCloseMilli; + } - @Override - public String toString() - { - return "Config{" + - "connectionTimeoutMilli=" + connectionTimeoutMilli + - ", readTimeoutMilli=" + readTimeoutMilli + - ", waitBeforeCloseMilli=" + waitBeforeCloseMilli + - "} " + super.toString(); - } + @Override + public String toString() { + return "Config{" + + "connectionTimeoutMilli=" + + connectionTimeoutMilli + + ", readTimeoutMilli=" + + readTimeoutMilli + + ", waitBeforeCloseMilli=" + + waitBeforeCloseMilli + + "} " + + super.toString(); } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/RequestOption.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/RequestOption.java index 3855a259..4ca1dd99 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/RequestOption.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/RequestOption.java @@ -20,37 +20,27 @@ import com.fasterxml.jackson.annotation.JsonProperty; @JsonInclude(JsonInclude.Include.NON_NULL) -public class RequestOption -{ - private final int size; - private final String chunk; +public class RequestOption { + private final int size; + private final String chunk; - public RequestOption( - @JsonProperty("size") int size, - @JsonProperty("chunk") String chunk) - { - this.size = size; - this.chunk = chunk; - } + public RequestOption(@JsonProperty("size") int size, @JsonProperty("chunk") String chunk) { + this.size = size; + this.chunk = chunk; + } - @JsonProperty("size") - public int getSize() - { - return size; - } + @JsonProperty("size") + public int getSize() { + return size; + } - @JsonProperty("chunk") - public String getChunk() - { - return chunk; - } + @JsonProperty("chunk") + public String getChunk() { + return chunk; + } - @Override - public String toString() - { - return "RequestOption{" + - "size=" + size + - ", chunk=" + chunk + - '}'; - } + @Override + public String toString() { + return "RequestOption{" + "size=" + size + ", chunk=" + chunk + '}'; + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/RetryableSender.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/RetryableSender.java index 68596e2f..654e4f26 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/RetryableSender.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/RetryableSender.java @@ -16,125 +16,107 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.komamitsu.fluency.fluentd.ingester.sender.retry.RetryStrategy; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import org.komamitsu.fluency.fluentd.ingester.sender.retry.RetryStrategy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class RetryableSender - extends FluentdSender -{ - private static final Logger LOG = LoggerFactory.getLogger(RetryableSender.class); - private final FluentdSender baseSender; - private final AtomicBoolean isClosed = new AtomicBoolean(); - private RetryStrategy retryStrategy; - - public RetryableSender(FluentdSender baseSender, RetryStrategy retryStrategy) - { - this(new Config(), baseSender, retryStrategy); - } - - public RetryableSender(Config config, FluentdSender baseSender, RetryStrategy retryStrategy) - { - super(config); - this.baseSender = baseSender; - this.retryStrategy = retryStrategy; - } - - @Override - public void close() - throws IOException - { - baseSender.close(); - isClosed.set(true); - } - - @Override - public boolean isAvailable() - { - return true; - } - - @Override - protected synchronized void sendInternal(List buffers, String ackToken) - throws IOException - { - IOException firstException = null; - - int retry = 0; - while (!retryStrategy.isRetriedOver(retry)) { - if (isClosed.get()) { - throw new RetryOverException("This sender is already closed", firstException); - } - - try { - if (ackToken == null) { - baseSender.send(buffers); - } - else { - baseSender.sendWithAck(buffers, ackToken); - } - return; - } - catch (IOException e) { - firstException = e; - LOG.warn("Sender failed to send data. sender=" + this + ", retry=" + retry, e); - } - - try { - TimeUnit.MILLISECONDS.sleep(retryStrategy.getNextIntervalMillis(retry)); - } - catch (InterruptedException e) { - LOG.debug("Interrupted while waiting", e); - Thread.currentThread().interrupt(); - } - retry++; +public class RetryableSender extends FluentdSender { + private static final Logger LOG = LoggerFactory.getLogger(RetryableSender.class); + private final FluentdSender baseSender; + private final AtomicBoolean isClosed = new AtomicBoolean(); + private RetryStrategy retryStrategy; + + public RetryableSender(FluentdSender baseSender, RetryStrategy retryStrategy) { + this(new Config(), baseSender, retryStrategy); + } + + public RetryableSender(Config config, FluentdSender baseSender, RetryStrategy retryStrategy) { + super(config); + this.baseSender = baseSender; + this.retryStrategy = retryStrategy; + } + + @Override + public void close() throws IOException { + baseSender.close(); + isClosed.set(true); + } + + @Override + public boolean isAvailable() { + return true; + } + + @Override + protected synchronized void sendInternal(List buffers, String ackToken) + throws IOException { + IOException firstException = null; + + int retry = 0; + while (!retryStrategy.isRetriedOver(retry)) { + if (isClosed.get()) { + throw new RetryOverException("This sender is already closed", firstException); + } + + try { + if (ackToken == null) { + baseSender.send(buffers); + } else { + baseSender.sendWithAck(buffers, ackToken); } - - throw new RetryOverException("Sending data was retried over", firstException); - } - - public FluentdSender getBaseSender() - { - return baseSender; - } - - public RetryStrategy getRetryStrategy() - { - return retryStrategy; + return; + } catch (IOException e) { + firstException = e; + LOG.warn("Sender failed to send data. sender=" + this + ", retry=" + retry, e); + } + + try { + TimeUnit.MILLISECONDS.sleep(retryStrategy.getNextIntervalMillis(retry)); + } catch (InterruptedException e) { + LOG.debug("Interrupted while waiting", e); + Thread.currentThread().interrupt(); + } + retry++; } - public boolean isClosed() - { - return isClosed.get(); + throw new RetryOverException("Sending data was retried over", firstException); + } + + public FluentdSender getBaseSender() { + return baseSender; + } + + public RetryStrategy getRetryStrategy() { + return retryStrategy; + } + + public boolean isClosed() { + return isClosed.get(); + } + + @Override + public String toString() { + return "RetryableSender{" + + "baseSender=" + + baseSender + + ", retryStrategy=" + + retryStrategy + + ", isClosed=" + + isClosed + + "} " + + super.toString(); + } + + public static class RetryOverException extends IOException { + public RetryOverException(String s, Throwable throwable) { + super(s, throwable); } + } - @Override - public String toString() - { - return "RetryableSender{" + - "baseSender=" + baseSender + - ", retryStrategy=" + retryStrategy + - ", isClosed=" + isClosed + - "} " + super.toString(); - } - - public static class RetryOverException - extends IOException - { - public RetryOverException(String s, Throwable throwable) - { - super(s, throwable); - } - } - - public static class Config - extends FluentdSender.Config - { - } + public static class Config extends FluentdSender.Config {} } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSender.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSender.java index 7b319b08..39e17076 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSender.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSender.java @@ -16,14 +16,6 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; -import org.komamitsu.fluency.validation.Validatable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.net.ssl.SSLSocket; -import javax.net.ssl.SSLSocketFactory; - import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -31,121 +23,104 @@ import java.nio.ByteBuffer; import java.util.List; import java.util.concurrent.atomic.AtomicReference; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.SSLSocketFactory; +import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; +import org.komamitsu.fluency.validation.Validatable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class SSLSender - extends InetSocketSender -{ - private static final Logger LOG = LoggerFactory.getLogger(TCPSender.class); - private final AtomicReference socket = new AtomicReference<>(); - private final SSLSocketBuilder socketBuilder; - private final Config config; - - public SSLSender() - { - this(new Config()); - } - - public SSLSender(Config config) - { - this(config, null); +public class SSLSender extends InetSocketSender { + private static final Logger LOG = LoggerFactory.getLogger(TCPSender.class); + private final AtomicReference socket = new AtomicReference<>(); + private final SSLSocketBuilder socketBuilder; + private final Config config; + + public SSLSender() { + this(new Config()); + } + + public SSLSender(Config config) { + this(config, null); + } + + public SSLSender(FailureDetector failureDetector) { + this(new Config(), failureDetector); + } + + public SSLSender(Config config, FailureDetector failureDetector) { + super(config, failureDetector); + config.validateValues(); + + socketBuilder = + new SSLSocketBuilder( + config.getHost(), + config.getPort(), + config.getConnectionTimeoutMilli(), + config.getReadTimeoutMilli(), + config.getSslSocketFactory()); + this.config = config; + } + + @Override + protected SSLSocket getOrCreateSocketInternal() throws IOException { + if (socket.get() == null) { + socket.set(socketBuilder.build()); } - - public SSLSender(FailureDetector failureDetector) - { - this(new Config(), failureDetector); + return socket.get(); + } + + @Override + protected void sendBuffers(SSLSocket sslSocket, List buffers) throws IOException { + for (ByteBuffer buffer : buffers) { + OutputStream outputStream = sslSocket.getOutputStream(); + if (buffer.hasArray()) { + outputStream.write(buffer.array(), buffer.position(), buffer.remaining()); + } else { + byte[] bytes = new byte[buffer.remaining()]; + buffer.get(bytes); + outputStream.write(bytes); + } } - - public SSLSender(Config config, FailureDetector failureDetector) - { - super(config, failureDetector); - config.validateValues(); - - socketBuilder = new SSLSocketBuilder( - config.getHost(), - config.getPort(), - config.getConnectionTimeoutMilli(), - config.getReadTimeoutMilli(), - config.getSslSocketFactory()); - this.config = config; + } + + @Override + protected void recvResponse(SSLSocket sslSocket, ByteBuffer buffer) throws IOException { + InputStream inputStream = sslSocket.getInputStream(); + byte[] tempBuf = new byte[buffer.remaining()]; + int read = inputStream.read(tempBuf); + if (read < 0) { + throw new SocketException("The connection is disconnected by the peer"); } - - @Override - protected SSLSocket getOrCreateSocketInternal() - throws IOException - { - if (socket.get() == null) { - socket.set(socketBuilder.build()); - } - return socket.get(); + buffer.put(tempBuf, 0, read); + } + + @Override + protected void closeSocket() throws IOException { + SSLSocket existingSocket; + if ((existingSocket = socket.getAndSet(null)) != null) { + existingSocket.close(); } + } - @Override - protected void sendBuffers(SSLSocket sslSocket, List buffers) - throws IOException - { - for (ByteBuffer buffer : buffers) { - OutputStream outputStream = sslSocket.getOutputStream(); - if (buffer.hasArray()) { - outputStream.write(buffer.array(), buffer.position(), buffer.remaining()); - } - else { - byte[] bytes = new byte[buffer.remaining()]; - buffer.get(bytes); - outputStream.write(bytes); - } - } - } + @Override + public String toString() { + return "SSLSender{" + "config=" + config + "} " + super.toString(); + } - @Override - protected void recvResponse(SSLSocket sslSocket, ByteBuffer buffer) - throws IOException - { - InputStream inputStream = sslSocket.getInputStream(); - byte[] tempBuf = new byte[buffer.remaining()]; - int read = inputStream.read(tempBuf); - if (read < 0) { - throw new SocketException("The connection is disconnected by the peer"); - } - buffer.put(tempBuf, 0, read); + public static class Config extends InetSocketSender.Config implements Validatable { + void validateValues() { + validate(); } - @Override - protected void closeSocket() - throws IOException - { - SSLSocket existingSocket; - if ((existingSocket = socket.getAndSet(null)) != null) { - existingSocket.close(); - } - } + private SSLSocketFactory sslSocketFactory = (SSLSocketFactory) SSLSocketFactory.getDefault(); - @Override - public String toString() - { - return "SSLSender{" + - "config=" + config + - "} " + super.toString(); + public SSLSocketFactory getSslSocketFactory() { + return sslSocketFactory; } - public static class Config - extends InetSocketSender.Config - implements Validatable - { - void validateValues() - { - validate(); - } - - private SSLSocketFactory sslSocketFactory = (SSLSocketFactory) SSLSocketFactory.getDefault(); - - public SSLSocketFactory getSslSocketFactory() - { - return sslSocketFactory; - } - - public void setSslSocketFactory(SSLSocketFactory sslSocketFactory) - { - this.sslSocketFactory = sslSocketFactory; - } + public void setSslSocketFactory(SSLSocketFactory sslSocketFactory) { + this.sslSocketFactory = sslSocketFactory; } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSocketBuilder.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSocketBuilder.java index 97f57d10..c3257f65 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSocketBuilder.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSocketBuilder.java @@ -16,53 +16,48 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import javax.net.ssl.SSLSocket; -import javax.net.ssl.SSLSocketFactory; -import javax.net.SocketFactory; - import java.io.IOException; import java.net.InetSocketAddress; import java.net.Socket; +import javax.net.SocketFactory; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.SSLSocketFactory; -public class SSLSocketBuilder -{ - private final String host; - private final int port; - private final int connectionTimeoutMilli; - private final int readTimeoutMilli; - private final SSLSocketFactory sslSocketFactory; +public class SSLSocketBuilder { + private final String host; + private final int port; + private final int connectionTimeoutMilli; + private final int readTimeoutMilli; + private final SSLSocketFactory sslSocketFactory; - public SSLSocketBuilder(String host, Integer port, int connectionTimeoutMilli, int readTimeoutMilli, SocketFactory sslSocketFactory) - { - this.host = host; - this.port = port; - this.connectionTimeoutMilli = connectionTimeoutMilli; - this.readTimeoutMilli = readTimeoutMilli; - this.sslSocketFactory = (SSLSocketFactory)sslSocketFactory; - } + public SSLSocketBuilder( + String host, + Integer port, + int connectionTimeoutMilli, + int readTimeoutMilli, + SocketFactory sslSocketFactory) { + this.host = host; + this.port = port; + this.connectionTimeoutMilli = connectionTimeoutMilli; + this.readTimeoutMilli = readTimeoutMilli; + this.sslSocketFactory = (SSLSocketFactory) sslSocketFactory; + } - public SSLSocket build() - throws IOException - { - Socket socket = this.sslSocketFactory.createSocket(); - try { - socket.connect(new InetSocketAddress(host, port), connectionTimeoutMilli); - socket.setTcpNoDelay(true); - socket.setSoTimeout(readTimeoutMilli); - } - catch (Throwable e) { - socket.close(); - throw e; - } - return (SSLSocket) socket; + public SSLSocket build() throws IOException { + Socket socket = this.sslSocketFactory.createSocket(); + try { + socket.connect(new InetSocketAddress(host, port), connectionTimeoutMilli); + socket.setTcpNoDelay(true); + socket.setSoTimeout(readTimeoutMilli); + } catch (Throwable e) { + socket.close(); + throw e; } + return (SSLSocket) socket; + } - @Override - public String toString() - { - return "SSLSocketBuilder{" + - "host='" + host + '\'' + - ", port=" + port + - '}'; - } + @Override + public String toString() { + return "SSLSocketBuilder{" + "host='" + host + '\'' + ", port=" + port + '}'; + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/TCPSender.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/TCPSender.java index 1eb99fb4..90b6310b 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/TCPSender.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/TCPSender.java @@ -16,11 +16,6 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; -import org.komamitsu.fluency.validation.Validatable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketException; @@ -28,100 +23,87 @@ import java.nio.channels.SocketChannel; import java.util.List; import java.util.concurrent.atomic.AtomicReference; +import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; +import org.komamitsu.fluency.validation.Validatable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class TCPSender - extends InetSocketSender -{ - private static final Logger LOG = LoggerFactory.getLogger(TCPSender.class); - private final AtomicReference channel = new AtomicReference<>(); - private final Config config; +public class TCPSender extends InetSocketSender { + private static final Logger LOG = LoggerFactory.getLogger(TCPSender.class); + private final AtomicReference channel = new AtomicReference<>(); + private final Config config; - public TCPSender() - { - this(new Config()); - } + public TCPSender() { + this(new Config()); + } - public TCPSender(Config config) - { - this(config, null); - } + public TCPSender(Config config) { + this(config, null); + } - public TCPSender(FailureDetector failureDetector) - { - this(new Config(), failureDetector); - } + public TCPSender(FailureDetector failureDetector) { + this(new Config(), failureDetector); + } - public TCPSender(Config config, FailureDetector failureDetector) - { - super(config, failureDetector); - config.validateValues(); - this.config = config; - } + public TCPSender(Config config, FailureDetector failureDetector) { + super(config, failureDetector); + config.validateValues(); + this.config = config; + } - @Override - protected SocketChannel getOrCreateSocketInternal() - throws IOException - { - if (channel.get() == null) { - SocketChannel socketChannel = SocketChannel.open(); - try { - socketChannel.socket().connect(new InetSocketAddress(config.getHost(), config.getPort()), config.getConnectionTimeoutMilli()); - socketChannel.socket().setTcpNoDelay(true); - socketChannel.socket().setSoTimeout(config.getReadTimeoutMilli()); - } - catch (Throwable e) { - // In case of java.net.UnknownHostException and so on, the internal socket can be leaked. - // So the SocketChannel should be closed here to avoid a socket leak. - socketChannel.close(); - throw e; - } - channel.set(socketChannel); - } - return channel.get(); + @Override + protected SocketChannel getOrCreateSocketInternal() throws IOException { + if (channel.get() == null) { + SocketChannel socketChannel = SocketChannel.open(); + try { + socketChannel + .socket() + .connect( + new InetSocketAddress(config.getHost(), config.getPort()), + config.getConnectionTimeoutMilli()); + socketChannel.socket().setTcpNoDelay(true); + socketChannel.socket().setSoTimeout(config.getReadTimeoutMilli()); + } catch (Throwable e) { + // In case of java.net.UnknownHostException and so on, the internal socket can be leaked. + // So the SocketChannel should be closed here to avoid a socket leak. + socketChannel.close(); + throw e; + } + channel.set(socketChannel); } + return channel.get(); + } - @Override - protected void sendBuffers(SocketChannel socketChannel, List buffers) - throws IOException - { - socketChannel.write(buffers.toArray(new ByteBuffer[0])); - } + @Override + protected void sendBuffers(SocketChannel socketChannel, List buffers) + throws IOException { + socketChannel.write(buffers.toArray(new ByteBuffer[0])); + } - @Override - protected void recvResponse(SocketChannel socketChannel, ByteBuffer buffer) - throws IOException - { - int read = socketChannel.read(buffer); - if (read < 0) { - throw new SocketException("The connection is disconnected by the peer"); - } + @Override + protected void recvResponse(SocketChannel socketChannel, ByteBuffer buffer) throws IOException { + int read = socketChannel.read(buffer); + if (read < 0) { + throw new SocketException("The connection is disconnected by the peer"); } + } - @Override - protected void closeSocket() - throws IOException - { - SocketChannel existingSocketChannel; - if ((existingSocketChannel = channel.getAndSet(null)) != null) { - existingSocketChannel.close(); - } + @Override + protected void closeSocket() throws IOException { + SocketChannel existingSocketChannel; + if ((existingSocketChannel = channel.getAndSet(null)) != null) { + existingSocketChannel.close(); } + } - @Override - public String toString() - { - return "TCPSender{" + - "config=" + config + - "} " + super.toString(); - } + @Override + public String toString() { + return "TCPSender{" + "config=" + config + "} " + super.toString(); + } - public static class Config - extends InetSocketSender.Config - implements Validatable - { - void validateValues() - { - validate(); - } + public static class Config extends InetSocketSender.Config implements Validatable { + void validateValues() { + validate(); } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/FailureDetectStrategy.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/FailureDetectStrategy.java index 63b69c89..749d478f 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/FailureDetectStrategy.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/FailureDetectStrategy.java @@ -16,20 +16,16 @@ package org.komamitsu.fluency.fluentd.ingester.sender.failuredetect; -public abstract class FailureDetectStrategy -{ - protected final Config config; +public abstract class FailureDetectStrategy { + protected final Config config; - protected FailureDetectStrategy(Config config) - { - this.config = config; - } + protected FailureDetectStrategy(Config config) { + this.config = config; + } - public abstract void heartbeat(long now); + public abstract void heartbeat(long now); - public abstract boolean isAvailable(); + public abstract boolean isAvailable(); - public static class Config - { - } + public static class Config {} } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/FailureDetector.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/FailureDetector.java index ecaa4235..880f40e3 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/FailureDetector.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/FailureDetector.java @@ -16,121 +16,104 @@ package org.komamitsu.fluency.fluentd.ingester.sender.failuredetect; +import java.io.Closeable; +import java.util.concurrent.atomic.AtomicReference; import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.Heartbeater; import org.komamitsu.fluency.validation.Validatable; import org.komamitsu.fluency.validation.annotation.Min; -import java.io.Closeable; -import java.util.concurrent.atomic.AtomicReference; - -public class FailureDetector - implements Heartbeater.Callback, Closeable -{ - private final FailureDetectStrategy failureDetectStrategy; - private final Heartbeater heartbeater; - private final AtomicReference lastFailureTimestampMillis = new AtomicReference<>(); - private final Config config; - - public FailureDetector(FailureDetectStrategy failureDetectStrategy, Heartbeater heartbeater, Config config) - { - config.validateValues(); - this.failureDetectStrategy = failureDetectStrategy; - this.heartbeater = heartbeater; - this.heartbeater.setCallback(this); - this.heartbeater.start(); - this.config = config; +public class FailureDetector implements Heartbeater.Callback, Closeable { + private final FailureDetectStrategy failureDetectStrategy; + private final Heartbeater heartbeater; + private final AtomicReference lastFailureTimestampMillis = new AtomicReference<>(); + private final Config config; + + public FailureDetector( + FailureDetectStrategy failureDetectStrategy, Heartbeater heartbeater, Config config) { + config.validateValues(); + this.failureDetectStrategy = failureDetectStrategy; + this.heartbeater = heartbeater; + this.heartbeater.setCallback(this); + this.heartbeater.start(); + this.config = config; + } + + public FailureDetector(FailureDetectStrategy failureDetectStrategy, Heartbeater heartbeater) { + this(failureDetectStrategy, heartbeater, new Config()); + } + + @Override + public void onHeartbeat() { + failureDetectStrategy.heartbeat(System.currentTimeMillis()); + } + + @Override + public void onFailure(Throwable cause) { + lastFailureTimestampMillis.set(System.currentTimeMillis()); + } + + public boolean isAvailable() { + Long failureTimestamp = lastFailureTimestampMillis.get(); + if (failureTimestamp != null) { + if (failureTimestamp > System.currentTimeMillis() - config.getFailureIntervalMillis()) { + return false; + } else { + lastFailureTimestampMillis.set(null); + } } - - public FailureDetector(FailureDetectStrategy failureDetectStrategy, Heartbeater heartbeater) - { - this(failureDetectStrategy, heartbeater, new Config()); + return failureDetectStrategy.isAvailable(); + } + + @Override + public void close() { + heartbeater.close(); + } + + public FailureDetectStrategy getFailureDetectStrategy() { + return failureDetectStrategy; + } + + public Heartbeater getHeartbeater() { + return heartbeater; + } + + public int getFailureIntervalMillis() { + return config.getFailureIntervalMillis(); + } + + @Override + public String toString() { + return "FailureDetector{" + + "failureDetectStrategy=" + + failureDetectStrategy + + ", heartbeater=" + + heartbeater + + ", lastFailureTimestampMillis=" + + lastFailureTimestampMillis + + ", config=" + + config + + '}'; + } + + public static class Config implements Validatable { + @Min(0) + private int failureIntervalMillis = 3 * 1000; + + public int getFailureIntervalMillis() { + return failureIntervalMillis; } - @Override - public void onHeartbeat() - { - failureDetectStrategy.heartbeat(System.currentTimeMillis()); + public void setFailureIntervalMillis(int failureIntervalMillis) { + this.failureIntervalMillis = failureIntervalMillis; } - @Override - public void onFailure(Throwable cause) - { - lastFailureTimestampMillis.set(System.currentTimeMillis()); - } - - public boolean isAvailable() - { - Long failureTimestamp = lastFailureTimestampMillis.get(); - if (failureTimestamp != null) { - if (failureTimestamp > System.currentTimeMillis() - config.getFailureIntervalMillis()) { - return false; - } - else { - lastFailureTimestampMillis.set(null); - } - } - return failureDetectStrategy.isAvailable(); + void validateValues() { + validate(); } @Override - public void close() - { - heartbeater.close(); - } - - public FailureDetectStrategy getFailureDetectStrategy() - { - return failureDetectStrategy; - } - - public Heartbeater getHeartbeater() - { - return heartbeater; - } - - public int getFailureIntervalMillis() - { - return config.getFailureIntervalMillis(); - } - - @Override - public String toString() - { - return "FailureDetector{" + - "failureDetectStrategy=" + failureDetectStrategy + - ", heartbeater=" + heartbeater + - ", lastFailureTimestampMillis=" + lastFailureTimestampMillis + - ", config=" + config + - '}'; - } - - public static class Config - implements Validatable - { - @Min(0) - private int failureIntervalMillis = 3 * 1000; - - public int getFailureIntervalMillis() - { - return failureIntervalMillis; - } - - public void setFailureIntervalMillis(int failureIntervalMillis) - { - this.failureIntervalMillis = failureIntervalMillis; - } - - void validateValues() - { - validate(); - } - - @Override - public String toString() - { - return "Config{" + - "failureIntervalMillis=" + failureIntervalMillis + - '}'; - } + public String toString() { + return "Config{" + "failureIntervalMillis=" + failureIntervalMillis + '}'; } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/PhiAccrualFailureDetectStrategy.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/PhiAccrualFailureDetectStrategy.java index 064b0891..b93ef3e0 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/PhiAccrualFailureDetectStrategy.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/PhiAccrualFailureDetectStrategy.java @@ -21,90 +21,77 @@ import org.komamitsu.fluency.validation.annotation.DecimalMin; import org.komamitsu.fluency.validation.annotation.Min; -public class PhiAccrualFailureDetectStrategy - extends FailureDetectStrategy -{ - private final PhiAccuralFailureDetector failureDetector; - private final Config config; - - public PhiAccrualFailureDetectStrategy() - { - this(new Config()); +public class PhiAccrualFailureDetectStrategy extends FailureDetectStrategy { + private final PhiAccuralFailureDetector failureDetector; + private final Config config; + + public PhiAccrualFailureDetectStrategy() { + this(new Config()); + } + + public PhiAccrualFailureDetectStrategy(Config config) { + super(config); + config.validateValues(); + this.config = config; + failureDetector = + new PhiAccuralFailureDetector.Builder() + .setThreshold(config.getPhiThreshold()) + .setMaxSampleSize(config.getArrivalWindowSize()) + .build(); + } + + @Override + public void heartbeat(long now) { + failureDetector.heartbeat(now); + } + + @Override + public boolean isAvailable() { + return failureDetector.isAvailable(); + } + + public double getPhiThreshold() { + return config.getPhiThreshold(); + } + + public int getArrivalWindowSize() { + return config.getArrivalWindowSize(); + } + + @Override + public String toString() { + return "PhiAccrualFailureDetectStrategy{" + + "failureDetector=" + + failureDetector + + "} " + + super.toString(); + } + + public static class Config extends FailureDetectStrategy.Config implements Validatable { + @DecimalMin(value = "0", inclusive = false) + private double phiThreshold = 16; + + @Min(value = 0, inclusive = false) + private int arrivalWindowSize = 100; + + public double getPhiThreshold() { + return phiThreshold; } - public PhiAccrualFailureDetectStrategy(Config config) - { - super(config); - config.validateValues(); - this.config = config; - failureDetector = new PhiAccuralFailureDetector.Builder(). - setThreshold(config.getPhiThreshold()). - setMaxSampleSize(config.getArrivalWindowSize()). - build(); + public void setPhiThreshold(float phiThreshold) { + this.phiThreshold = phiThreshold; } - @Override - public void heartbeat(long now) - { - failureDetector.heartbeat(now); + public int getArrivalWindowSize() { + return arrivalWindowSize; } - @Override - public boolean isAvailable() - { - return failureDetector.isAvailable(); + void validateValues() { + validate(); } - public double getPhiThreshold() - { - return config.getPhiThreshold(); - } - - public int getArrivalWindowSize() - { - return config.getArrivalWindowSize(); - } - - @Override - public String toString() - { - return "PhiAccrualFailureDetectStrategy{" + - "failureDetector=" + failureDetector + - "} " + super.toString(); - } - - public static class Config - extends FailureDetectStrategy.Config - implements Validatable - { - @DecimalMin(value = "0", inclusive = false) - private double phiThreshold = 16; - @Min(value = 0, inclusive = false) - private int arrivalWindowSize = 100; - - public double getPhiThreshold() - { - return phiThreshold; - } - - public void setPhiThreshold(float phiThreshold) - { - this.phiThreshold = phiThreshold; - } - - public int getArrivalWindowSize() - { - return arrivalWindowSize; - } - - void validateValues() - { - validate(); - } - - public void setArrivalWindowSize(int arrivalWindowSize) - { - this.arrivalWindowSize = arrivalWindowSize; - } + public void setArrivalWindowSize(int arrivalWindowSize) { + this.arrivalWindowSize = arrivalWindowSize; } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/Heartbeater.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/Heartbeater.java index 1f025834..196e55e6 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/Heartbeater.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/Heartbeater.java @@ -16,125 +16,98 @@ package org.komamitsu.fluency.fluentd.ingester.sender.heartbeat; -import org.komamitsu.fluency.util.ExecutorServiceUtils; -import org.komamitsu.fluency.validation.Validatable; -import org.komamitsu.fluency.validation.annotation.Min; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.Closeable; import java.io.IOException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import org.komamitsu.fluency.util.ExecutorServiceUtils; +import org.komamitsu.fluency.validation.Validatable; +import org.komamitsu.fluency.validation.annotation.Min; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public abstract class Heartbeater - implements Closeable -{ - private static final Logger LOG = LoggerFactory.getLogger(Heartbeater.class); - private final Config config; - private final ScheduledExecutorService executorService; - private final AtomicReference callback = new AtomicReference<>(); - - protected Heartbeater(Config config) - { - config.validateValues(); - this.config = config; - executorService = ExecutorServiceUtils.newScheduledDaemonThreadPool(1); +public abstract class Heartbeater implements Closeable { + private static final Logger LOG = LoggerFactory.getLogger(Heartbeater.class); + private final Config config; + private final ScheduledExecutorService executorService; + private final AtomicReference callback = new AtomicReference<>(); + + protected Heartbeater(Config config) { + config.validateValues(); + this.config = config; + executorService = ExecutorServiceUtils.newScheduledDaemonThreadPool(1); + } + + public void start() { + executorService.scheduleAtFixedRate( + this::ping, config.getIntervalMillis(), config.getIntervalMillis(), TimeUnit.MILLISECONDS); + } + + protected abstract void invoke() throws IOException; + + protected void ping() { + try { + invoke(); + } catch (Throwable e) { + LOG.warn("ping(): failed, config=" + config); + Callback callback = this.callback.get(); + if (callback != null) { + callback.onFailure(e); + } } + } - public void start() - { - executorService.scheduleAtFixedRate( - this::ping, - config.getIntervalMillis(), - config.getIntervalMillis(), - TimeUnit.MILLISECONDS); + protected void pong() { + Callback callback = this.callback.get(); + if (callback != null) { + callback.onHeartbeat(); } + } - protected abstract void invoke() - throws IOException; - - protected void ping() - { - try { - invoke(); - } - catch (Throwable e) { - LOG.warn("ping(): failed, config=" + config); - Callback callback = this.callback.get(); - if (callback != null) { - callback.onFailure(e); - } - } - } + public void setCallback(Callback callback) { + this.callback.set(callback); + } - protected void pong() - { - Callback callback = this.callback.get(); - if (callback != null) { - callback.onHeartbeat(); - } - } + @Override + public void close() { + ExecutorServiceUtils.finishExecutorService(executorService); + } - public void setCallback(Callback callback) - { - this.callback.set(callback); - } + public int getIntervalMillis() { + return config.getIntervalMillis(); + } - @Override - public void close() - { - ExecutorServiceUtils.finishExecutorService(executorService); - } + @Override + public String toString() { + return "Heartbeater{" + "config=" + config + '}'; + } - public int getIntervalMillis() - { - return config.getIntervalMillis(); - } + public interface Callback { + void onHeartbeat(); - @Override - public String toString() - { - return "Heartbeater{" + - "config=" + config + - '}'; + void onFailure(Throwable cause); + } + + public static class Config implements Validatable { + @Min(100) + private int intervalMillis = 1000; + + public int getIntervalMillis() { + return intervalMillis; } - public interface Callback - { - void onHeartbeat(); + public void setIntervalMillis(int intervalMillis) { + this.intervalMillis = intervalMillis; + } - void onFailure(Throwable cause); + void validateValues() { + validate(); } - public static class Config - implements Validatable - { - @Min(100) - private int intervalMillis = 1000; - - public int getIntervalMillis() - { - return intervalMillis; - } - - public void setIntervalMillis(int intervalMillis) - { - this.intervalMillis = intervalMillis; - } - - void validateValues() - { - validate(); - } - - @Override - public String toString() - { - return "Config{" + - "intervalMillis=" + intervalMillis + - '}'; - } + @Override + public String toString() { + return "Config{" + "intervalMillis=" + intervalMillis + '}'; } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/InetSocketHeartbeater.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/InetSocketHeartbeater.java index bd288d55..6e60da28 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/InetSocketHeartbeater.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/InetSocketHeartbeater.java @@ -16,88 +16,67 @@ package org.komamitsu.fluency.fluentd.ingester.sender.heartbeat; +import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; +public abstract class InetSocketHeartbeater extends Heartbeater { + private static final Logger LOG = LoggerFactory.getLogger(InetSocketHeartbeater.class); + private final Config config; -public abstract class InetSocketHeartbeater - extends Heartbeater -{ - private static final Logger LOG = LoggerFactory.getLogger(InetSocketHeartbeater.class); - private final Config config; + protected InetSocketHeartbeater(Config config) { + super(config); + this.config = config; + } - protected InetSocketHeartbeater(Config config) - { - super(config); - this.config = config; - } + protected abstract void invoke() throws IOException; - protected abstract void invoke() - throws IOException; + public String getHost() { + return config.getHost(); + } - public String getHost() - { - return config.getHost(); - } + public int getPort() { + return config.getPort(); + } - public int getPort() - { - return config.getPort(); - } + public int getIntervalMillis() { + return config.getIntervalMillis(); + } + + @Override + public String toString() { + return "InetHeartbeater{" + "config=" + config + '}'; + } - public int getIntervalMillis() - { - return config.getIntervalMillis(); + public interface Callback { + void onHeartbeat(); + + void onFailure(Throwable cause); + } + + public static class Config extends Heartbeater.Config { + private String host = "127.0.0.1"; + private int port = 24224; + + public String getHost() { + return host; } - @Override - public String toString() - { - return "InetHeartbeater{" + - "config=" + config + - '}'; + public void setHost(String host) { + this.host = host; } - public interface Callback - { - void onHeartbeat(); + public int getPort() { + return port; + } - void onFailure(Throwable cause); + public void setPort(int port) { + this.port = port; } - public static class Config - extends Heartbeater.Config - { - private String host = "127.0.0.1"; - private int port = 24224; - - public String getHost() - { - return host; - } - - public void setHost(String host) - { - this.host = host; - } - - public int getPort() - { - return port; - } - - public void setPort(int port) - { - this.port = port; - } - - @Override - public String toString() { - return "Config{" + - "host='" + host + '\'' + - ", port=" + port + - "} " + super.toString(); - } + @Override + public String toString() { + return "Config{" + "host='" + host + '\'' + ", port=" + port + "} " + super.toString(); } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/SSLHeartbeater.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/SSLHeartbeater.java index e8cd6a0e..8931be09 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/SSLHeartbeater.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/SSLHeartbeater.java @@ -16,109 +16,96 @@ package org.komamitsu.fluency.fluentd.ingester.sender.heartbeat; +import java.io.IOException; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.SSLSocketFactory; import org.komamitsu.fluency.fluentd.ingester.sender.SSLSocketBuilder; import org.komamitsu.fluency.validation.Validatable; import org.komamitsu.fluency.validation.annotation.Min; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.net.ssl.SSLSocket; -import javax.net.ssl.SSLSocketFactory; - -import java.io.IOException; - -public class SSLHeartbeater - extends InetSocketHeartbeater -{ - private static final Logger LOG = LoggerFactory.getLogger(SSLHeartbeater.class); - private final Config config; - private final SSLSocketBuilder sslSocketBuilder; - - public SSLHeartbeater() - { - this(new Config()); - } - - public SSLHeartbeater(final Config config) - { - super(config); - config.validateValues(); - this.config = config; - sslSocketBuilder = new SSLSocketBuilder( +public class SSLHeartbeater extends InetSocketHeartbeater { + private static final Logger LOG = LoggerFactory.getLogger(SSLHeartbeater.class); + private final Config config; + private final SSLSocketBuilder sslSocketBuilder; + + public SSLHeartbeater() { + this(new Config()); + } + + public SSLHeartbeater(final Config config) { + super(config); + config.validateValues(); + this.config = config; + sslSocketBuilder = + new SSLSocketBuilder( config.getHost(), config.getPort(), config.connectionTimeoutMilli, config.readTimeoutMilli, - config.getSslSocketFactory() - ); + config.getSslSocketFactory()); + } + + @Override + protected void invoke() throws IOException { + try (SSLSocket sslSocket = sslSocketBuilder.build()) { + LOG.trace( + "SSLHeartbeat: remotePort={}, localPort={}", + sslSocket.getPort(), + sslSocket.getLocalPort()); + // Try SSL handshake + sslSocket.getSession(); + pong(); + } + } + + @Override + public String toString() { + return "SSLHeartbeater{" + + "config=" + + config + + ", sslSocketBuilder=" + + sslSocketBuilder + + "} " + + super.toString(); + } + + public static class Config extends InetSocketHeartbeater.Config implements Validatable { + @Min(10) + private int connectionTimeoutMilli = 5000; + + @Min(10) + private int readTimeoutMilli = 5000; + + public int getConnectionTimeoutMilli() { + return connectionTimeoutMilli; } - @Override - protected void invoke() - throws IOException - { - try (SSLSocket sslSocket = sslSocketBuilder.build()) { - LOG.trace("SSLHeartbeat: remotePort={}, localPort={}", sslSocket.getPort(), sslSocket.getLocalPort()); - // Try SSL handshake - sslSocket.getSession(); - pong(); - } + public void setConnectionTimeoutMilli(int connectionTimeoutMilli) { + this.connectionTimeoutMilli = connectionTimeoutMilli; } - @Override - public String toString() - { - return "SSLHeartbeater{" + - "config=" + config + - ", sslSocketBuilder=" + sslSocketBuilder + - "} " + super.toString(); + public int getReadTimeoutMilli() { + return readTimeoutMilli; } - public static class Config - extends InetSocketHeartbeater.Config - implements Validatable - { - @Min(10) - private int connectionTimeoutMilli = 5000; - @Min(10) - private int readTimeoutMilli = 5000; - - public int getConnectionTimeoutMilli() - { - return connectionTimeoutMilli; - } - - public void setConnectionTimeoutMilli(int connectionTimeoutMilli) - { - this.connectionTimeoutMilli = connectionTimeoutMilli; - } - - public int getReadTimeoutMilli() - { - return readTimeoutMilli; - } - - public void setReadTimeoutMilli(int readTimeoutMilli) - { - this.readTimeoutMilli = readTimeoutMilli; - } - - private SSLSocketFactory sslSocketFactory = (SSLSocketFactory) SSLSocketFactory.getDefault(); - - public SSLSocketFactory getSslSocketFactory() - { - return sslSocketFactory; - } - - public void setSslSocketFactory(SSLSocketFactory sslSocketFactory) - { - this.sslSocketFactory = sslSocketFactory; - } - - void validateValues() - { - validate(); - } + public void setReadTimeoutMilli(int readTimeoutMilli) { + this.readTimeoutMilli = readTimeoutMilli; + } + + private SSLSocketFactory sslSocketFactory = (SSLSocketFactory) SSLSocketFactory.getDefault(); + + public SSLSocketFactory getSslSocketFactory() { + return sslSocketFactory; + } + + public void setSslSocketFactory(SSLSocketFactory sslSocketFactory) { + this.sslSocketFactory = sslSocketFactory; } -} + void validateValues() { + validate(); + } + } +} diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/TCPHeartbeater.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/TCPHeartbeater.java index 3f1c73a1..87003f45 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/TCPHeartbeater.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/TCPHeartbeater.java @@ -16,52 +16,41 @@ package org.komamitsu.fluency.fluentd.ingester.sender.heartbeat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.net.InetSocketAddress; import java.nio.channels.SocketChannel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class TCPHeartbeater - extends InetSocketHeartbeater -{ - private static final Logger LOG = LoggerFactory.getLogger(TCPHeartbeater.class); - private final Config config; - - public TCPHeartbeater() - { - this(new Config()); - } - - public TCPHeartbeater(final Config config) - { - super(config); - this.config = config; - } - - @Override - protected void invoke() - throws IOException - { - try (SocketChannel socketChannel = SocketChannel.open(new InetSocketAddress(config.getHost(), config.getPort()))) { - LOG.trace("TCPHeartbeat: remotePort={}, localPort={}", - socketChannel.socket().getPort(), socketChannel.socket().getLocalPort()); - pong(); - } +public class TCPHeartbeater extends InetSocketHeartbeater { + private static final Logger LOG = LoggerFactory.getLogger(TCPHeartbeater.class); + private final Config config; + + public TCPHeartbeater() { + this(new Config()); + } + + public TCPHeartbeater(final Config config) { + super(config); + this.config = config; + } + + @Override + protected void invoke() throws IOException { + try (SocketChannel socketChannel = + SocketChannel.open(new InetSocketAddress(config.getHost(), config.getPort()))) { + LOG.trace( + "TCPHeartbeat: remotePort={}, localPort={}", + socketChannel.socket().getPort(), + socketChannel.socket().getLocalPort()); + pong(); } + } - @Override - public String toString() - { - return "TCPHeartbeater{" + - "config=" + config + - "} " + super.toString(); - } + @Override + public String toString() { + return "TCPHeartbeater{" + "config=" + config + "} " + super.toString(); + } - public static class Config - extends InetSocketHeartbeater.Config - { - } + public static class Config extends InetSocketHeartbeater.Config {} } - diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UDPHeartbeater.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UDPHeartbeater.java index 0af77b53..1e79200c 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UDPHeartbeater.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UDPHeartbeater.java @@ -16,54 +16,41 @@ package org.komamitsu.fluency.fluentd.ingester.sender.heartbeat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.nio.ByteBuffer; import java.nio.channels.DatagramChannel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class UDPHeartbeater - extends InetSocketHeartbeater -{ - private static final Logger LOG = LoggerFactory.getLogger(UDPHeartbeater.class); - private final SocketAddress socketAddress; - - public UDPHeartbeater() - { - this(new Config()); - } - - public UDPHeartbeater(final Config config) - { - super(config); - socketAddress = new InetSocketAddress(config.getHost(), config.getPort()); - } - - @Override - protected void invoke() - throws IOException - { - try (DatagramChannel datagramChannel = DatagramChannel.open()) { - ByteBuffer byteBuffer = ByteBuffer.allocate(0); - datagramChannel.send(byteBuffer, socketAddress); - datagramChannel.receive(byteBuffer); - pong(); - } +public class UDPHeartbeater extends InetSocketHeartbeater { + private static final Logger LOG = LoggerFactory.getLogger(UDPHeartbeater.class); + private final SocketAddress socketAddress; + + public UDPHeartbeater() { + this(new Config()); + } + + public UDPHeartbeater(final Config config) { + super(config); + socketAddress = new InetSocketAddress(config.getHost(), config.getPort()); + } + + @Override + protected void invoke() throws IOException { + try (DatagramChannel datagramChannel = DatagramChannel.open()) { + ByteBuffer byteBuffer = ByteBuffer.allocate(0); + datagramChannel.send(byteBuffer, socketAddress); + datagramChannel.receive(byteBuffer); + pong(); } + } - @Override - public String toString() - { - return "UDPHeartbeater{" + - "socketAddress=" + socketAddress + - "} " + super.toString(); - } + @Override + public String toString() { + return "UDPHeartbeater{" + "socketAddress=" + socketAddress + "} " + super.toString(); + } - public static class Config - extends InetSocketHeartbeater.Config - { - } + public static class Config extends InetSocketHeartbeater.Config {} } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ConstantRetryStrategy.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ConstantRetryStrategy.java index 08c2ce0c..c0be6ca6 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ConstantRetryStrategy.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ConstantRetryStrategy.java @@ -19,60 +19,44 @@ import org.komamitsu.fluency.validation.Validatable; import org.komamitsu.fluency.validation.annotation.Min; -public class ConstantRetryStrategy - extends RetryStrategy -{ - private final Config config; - - public ConstantRetryStrategy(Config config) - { - super(config); - config.validateValues(); - this.config = config; +public class ConstantRetryStrategy extends RetryStrategy { + private final Config config; + + public ConstantRetryStrategy(Config config) { + super(config); + config.validateValues(); + this.config = config; + } + + @Override + public int getNextIntervalMillis(int retryCount) { + return config.getRetryIntervalMillis(); + } + + @Override + public String toString() { + return "ConstantRetryStrategy{" + "config=" + config + "} " + super.toString(); + } + + public static class Config extends RetryStrategy.Config implements Validatable { + @Min(10) + private int retryIntervalMillis = 2000; + + public int getRetryIntervalMillis() { + return retryIntervalMillis; } - @Override - public int getNextIntervalMillis(int retryCount) - { - return config.getRetryIntervalMillis(); + public void setRetryIntervalMillis(int retryIntervalMillis) { + this.retryIntervalMillis = retryIntervalMillis; } - @Override - public String toString() - { - return "ConstantRetryStrategy{" + - "config=" + config + - "} " + super.toString(); + void validateValues() { + validate(); } - public static class Config - extends RetryStrategy.Config - implements Validatable - { - @Min(10) - private int retryIntervalMillis = 2000; - - public int getRetryIntervalMillis() - { - return retryIntervalMillis; - } - - public void setRetryIntervalMillis(int retryIntervalMillis) - { - this.retryIntervalMillis = retryIntervalMillis; - } - - void validateValues() - { - validate(); - } - - @Override - public String toString() - { - return "Config{" + - "retryIntervalMillis=" + retryIntervalMillis + - "} " + super.toString(); - } + @Override + public String toString() { + return "Config{" + "retryIntervalMillis=" + retryIntervalMillis + "} " + super.toString(); } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ExponentialBackOffRetryStrategy.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ExponentialBackOffRetryStrategy.java index a8f03c60..920f0995 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ExponentialBackOffRetryStrategy.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ExponentialBackOffRetryStrategy.java @@ -19,92 +19,77 @@ import org.komamitsu.fluency.validation.Validatable; import org.komamitsu.fluency.validation.annotation.Min; -public class ExponentialBackOffRetryStrategy - extends RetryStrategy -{ - private final Config config; - - public ExponentialBackOffRetryStrategy() - { - this(new Config()); +public class ExponentialBackOffRetryStrategy extends RetryStrategy { + private final Config config; + + public ExponentialBackOffRetryStrategy() { + this(new Config()); + } + + public ExponentialBackOffRetryStrategy(Config config) { + super(config); + config.validateValues(); + this.config = config; + } + + @Override + public int getNextIntervalMillis(int retryCount) { + int interval = config.getBaseIntervalMillis() * ((int) Math.pow(2.0, (double) retryCount)); + if (interval > config.getMaxIntervalMillis()) { + return config.getMaxIntervalMillis(); } + return interval; + } - public ExponentialBackOffRetryStrategy(Config config) - { - super(config); - config.validateValues(); - this.config = config; + public int getBaseIntervalMillis() { + return config.getBaseIntervalMillis(); + } + + public int getMaxIntervalMillis() { + return config.getMaxIntervalMillis(); + } + + @Override + public String toString() { + return "ExponentialBackOffRetryStrategy{" + "config=" + config + "} " + super.toString(); + } + + public static class Config extends RetryStrategy.Config implements Validatable { + @Min(10) + private int baseIntervalMillis = 400; + + @Min(10) + private int maxIntervalMillis = 30 * 1000; + + public int getBaseIntervalMillis() { + return baseIntervalMillis; } - @Override - public int getNextIntervalMillis(int retryCount) - { - int interval = config.getBaseIntervalMillis() * ((int) Math.pow(2.0, (double) retryCount)); - if (interval > config.getMaxIntervalMillis()) { - return config.getMaxIntervalMillis(); - } - return interval; + public void setBaseIntervalMillis(int baseIntervalMillis) { + this.baseIntervalMillis = baseIntervalMillis; } - public int getBaseIntervalMillis() - { - return config.getBaseIntervalMillis(); + public int getMaxIntervalMillis() { + return maxIntervalMillis; } - public int getMaxIntervalMillis() - { - return config.getMaxIntervalMillis(); + public void setMaxIntervalMillis(int maxIntervalMillis) { + this.maxIntervalMillis = maxIntervalMillis; } - @Override - public String toString() - { - return "ExponentialBackOffRetryStrategy{" + - "config=" + config + - "} " + super.toString(); + void validateValues() { + validate(); } - public static class Config - extends RetryStrategy.Config - implements Validatable - { - @Min(10) - private int baseIntervalMillis = 400; - @Min(10) - private int maxIntervalMillis = 30 * 1000; - - public int getBaseIntervalMillis() - { - return baseIntervalMillis; - } - - public void setBaseIntervalMillis(int baseIntervalMillis) - { - this.baseIntervalMillis = baseIntervalMillis; - } - - public int getMaxIntervalMillis() - { - return maxIntervalMillis; - } - - public void setMaxIntervalMillis(int maxIntervalMillis) - { - this.maxIntervalMillis = maxIntervalMillis; - } - - void validateValues() - { - validate(); - } - - @Override - public String toString() - { - return "Config{" + - "baseIntervalMillis=" + baseIntervalMillis + - ", maxIntervalMillis=" + maxIntervalMillis + - "} " + super.toString(); - } + @Override + public String toString() { + return "Config{" + + "baseIntervalMillis=" + + baseIntervalMillis + + ", maxIntervalMillis=" + + maxIntervalMillis + + "} " + + super.toString(); } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/RetryStrategy.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/RetryStrategy.java index dc4649a7..3a29e362 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/RetryStrategy.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/RetryStrategy.java @@ -16,59 +16,45 @@ package org.komamitsu.fluency.fluentd.ingester.sender.retry; -import org.komamitsu.fluency.validation.Validatable; import org.komamitsu.fluency.validation.annotation.Min; -public abstract class RetryStrategy -{ - private final Config config; +public abstract class RetryStrategy { + private final Config config; - protected RetryStrategy(Config config) - { - this.config = config; - } + protected RetryStrategy(Config config) { + this.config = config; + } - public abstract int getNextIntervalMillis(int retryCount); + public abstract int getNextIntervalMillis(int retryCount); - public boolean isRetriedOver(int retryCount) - { - return retryCount > config.getMaxRetryCount(); - } + public boolean isRetriedOver(int retryCount) { + return retryCount > config.getMaxRetryCount(); + } - public int getMaxRetryCount() - { - return config.getMaxRetryCount(); - } + public int getMaxRetryCount() { + return config.getMaxRetryCount(); + } - @Override - public String toString() - { - return "RetryStrategy{" + - "config=" + config + - '}'; - } + @Override + public String toString() { + return "RetryStrategy{" + "config=" + config + '}'; + } - public static class Config - { - @Min(0) - private int maxRetryCount = 7; + public static class Config { + @Min(0) + private int maxRetryCount = 7; - public int getMaxRetryCount() - { - return maxRetryCount; - } + public int getMaxRetryCount() { + return maxRetryCount; + } - public void setMaxRetryCount(int maxRetryCount) - { - this.maxRetryCount = maxRetryCount; - } + public void setMaxRetryCount(int maxRetryCount) { + this.maxRetryCount = maxRetryCount; + } - @Override - public String toString() - { - return "Config{" + - "maxRetryCount=" + maxRetryCount + - '}'; - } + @Override + public String toString() { + return "Config{" + "maxRetryCount=" + maxRetryCount + '}'; } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/recordformat/FluentdRecordFormatter.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/recordformat/FluentdRecordFormatter.java index 2a538a01..20f8d02e 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/recordformat/FluentdRecordFormatter.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/recordformat/FluentdRecordFormatter.java @@ -17,106 +17,85 @@ package org.komamitsu.fluency.fluentd.recordformat; import com.fasterxml.jackson.core.JsonProcessingException; -import org.komamitsu.fluency.recordformat.AbstractRecordFormatter; -import org.komamitsu.fluency.recordformat.RecordFormatter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Map; +import org.komamitsu.fluency.recordformat.AbstractRecordFormatter; +import org.komamitsu.fluency.recordformat.RecordFormatter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class FluentdRecordFormatter - extends AbstractRecordFormatter - implements RecordFormatter -{ - private static final Logger LOG = LoggerFactory.getLogger(FluentdRecordFormatter.class); +public class FluentdRecordFormatter extends AbstractRecordFormatter implements RecordFormatter { + private static final Logger LOG = LoggerFactory.getLogger(FluentdRecordFormatter.class); - public FluentdRecordFormatter() - { - this(new Config()); - } + public FluentdRecordFormatter() { + this(new Config()); + } - public FluentdRecordFormatter(Config config) - { - super(config); - } + public FluentdRecordFormatter(Config config) { + super(config); + } - @Override - public byte[] format(String tag, Object timestamp, Map data) - { - try { - return objectMapperForMessagePack.writeValueAsBytes(Arrays.asList(timestamp, data)); - } - catch (JsonProcessingException e) { - throw new IllegalArgumentException( - String.format( - "Failed to convert the record to MessagePack format: cause=%s, tag=%s, timestamp=%s, recordCount=%d", - e.getMessage(), - tag, timestamp, data.size()) - ); - } + @Override + public byte[] format(String tag, Object timestamp, Map data) { + try { + return objectMapperForMessagePack.writeValueAsBytes(Arrays.asList(timestamp, data)); + } catch (JsonProcessingException e) { + throw new IllegalArgumentException( + String.format( + "Failed to convert the record to MessagePack format: cause=%s, tag=%s, timestamp=%s, recordCount=%d", + e.getMessage(), tag, timestamp, data.size())); } + } - @Override - public byte[] formatFromMessagePack(String tag, Object timestamp, byte[] mapValue, int offset, int len) - { - try { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - // 2 items array - outputStream.write(0x92); - objectMapperForMessagePack.writeValue(outputStream, timestamp); - outputStream.write(mapValue, offset, len); - outputStream.close(); + @Override + public byte[] formatFromMessagePack( + String tag, Object timestamp, byte[] mapValue, int offset, int len) { + try { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + // 2 items array + outputStream.write(0x92); + objectMapperForMessagePack.writeValue(outputStream, timestamp); + outputStream.write(mapValue, offset, len); + outputStream.close(); - return outputStream.toByteArray(); - } - catch (IOException e) { - throw new IllegalArgumentException( - String.format( - "Failed to convert the record to MessagePack format: cause=%s, tag=%s, timestamp=%s, dataSize=%s", - e.getMessage(), - tag, timestamp, mapValue.length) - ); - } + return outputStream.toByteArray(); + } catch (IOException e) { + throw new IllegalArgumentException( + String.format( + "Failed to convert the record to MessagePack format: cause=%s, tag=%s, timestamp=%s, dataSize=%s", + e.getMessage(), tag, timestamp, mapValue.length)); } + } - @Override - public byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue) - { - int mapValueLen = mapValue.remaining(); - try { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - // 2 items array - outputStream.write(0x92); - objectMapperForMessagePack.writeValue(outputStream, timestamp); - while (mapValue.hasRemaining()) { - outputStream.write(mapValue.get()); - } - outputStream.close(); + @Override + public byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue) { + int mapValueLen = mapValue.remaining(); + try { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + // 2 items array + outputStream.write(0x92); + objectMapperForMessagePack.writeValue(outputStream, timestamp); + while (mapValue.hasRemaining()) { + outputStream.write(mapValue.get()); + } + outputStream.close(); - return outputStream.toByteArray(); - } - catch (IOException e) { - throw new IllegalArgumentException( - String.format( - "Failed to convert the record to MessagePack format: cause=%s, tag=%s, timestamp=%s, dataSize=%s", - e.getMessage(), - tag, timestamp, mapValueLen) - ); - } + return outputStream.toByteArray(); + } catch (IOException e) { + throw new IllegalArgumentException( + String.format( + "Failed to convert the record to MessagePack format: cause=%s, tag=%s, timestamp=%s, dataSize=%s", + e.getMessage(), tag, timestamp, mapValueLen)); } + } - @Override - public String formatName() - { - return "fluentd-packedforward"; - } + @Override + public String formatName() { + return "fluentd-packedforward"; + } - public static class Config - extends RecordFormatter.Config - { - } + public static class Config extends RecordFormatter.Config {} } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/buffer/BufferForFluentdTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/buffer/BufferForFluentdTest.java index 04b634e6..6c0281ed 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/buffer/BufferForFluentdTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/buffer/BufferForFluentdTest.java @@ -16,8 +16,21 @@ package org.komamitsu.fluency.buffer; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.Test; import org.komamitsu.fluency.EventTime; import org.komamitsu.fluency.fluentd.ingester.FluentdIngester; @@ -33,271 +46,296 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class BufferForFluentdTest -{ - private final FluentdRecordFormatter fluentdRecordFormatter = - new FluentdRecordFormatter(new FluentdRecordFormatter.Config()); +class BufferForFluentdTest { + private final FluentdRecordFormatter fluentdRecordFormatter = + new FluentdRecordFormatter(new FluentdRecordFormatter.Config()); - @Test - void withFluentdFormat() - throws IOException, InterruptedException - { - for (Integer loopCount : Arrays.asList(100, 1000, 10000, 100000)) { - new BufferTestHelper().baseTestMessageBuffer(loopCount, true, true, false, - new Buffer(new Buffer.Config(), fluentdRecordFormatter)); - new BufferTestHelper().baseTestMessageBuffer(loopCount, false, true, false, - new Buffer(new Buffer.Config(), fluentdRecordFormatter)); - new BufferTestHelper().baseTestMessageBuffer(loopCount, true, false, false, - new Buffer(new Buffer.Config(), fluentdRecordFormatter)); - new BufferTestHelper().baseTestMessageBuffer(loopCount, false, false, false, - new Buffer(new Buffer.Config(), fluentdRecordFormatter)); - new BufferTestHelper().baseTestMessageBuffer(loopCount, true, true, true, - new Buffer(new Buffer.Config(), fluentdRecordFormatter)); - new BufferTestHelper().baseTestMessageBuffer(loopCount, false, true, true, - new Buffer(new Buffer.Config(), fluentdRecordFormatter)); - new BufferTestHelper().baseTestMessageBuffer(loopCount, true, false, true, - new Buffer(new Buffer.Config(), fluentdRecordFormatter)); - new BufferTestHelper().baseTestMessageBuffer(loopCount, false, false, true, - new Buffer(new Buffer.Config(), fluentdRecordFormatter)); - } + @Test + void withFluentdFormat() throws IOException, InterruptedException { + for (Integer loopCount : Arrays.asList(100, 1000, 10000, 100000)) { + new BufferTestHelper() + .baseTestMessageBuffer( + loopCount, + true, + true, + false, + new Buffer(new Buffer.Config(), fluentdRecordFormatter)); + new BufferTestHelper() + .baseTestMessageBuffer( + loopCount, + false, + true, + false, + new Buffer(new Buffer.Config(), fluentdRecordFormatter)); + new BufferTestHelper() + .baseTestMessageBuffer( + loopCount, + true, + false, + false, + new Buffer(new Buffer.Config(), fluentdRecordFormatter)); + new BufferTestHelper() + .baseTestMessageBuffer( + loopCount, + false, + false, + false, + new Buffer(new Buffer.Config(), fluentdRecordFormatter)); + new BufferTestHelper() + .baseTestMessageBuffer( + loopCount, true, true, true, new Buffer(new Buffer.Config(), fluentdRecordFormatter)); + new BufferTestHelper() + .baseTestMessageBuffer( + loopCount, + false, + true, + true, + new Buffer(new Buffer.Config(), fluentdRecordFormatter)); + new BufferTestHelper() + .baseTestMessageBuffer( + loopCount, + true, + false, + true, + new Buffer(new Buffer.Config(), fluentdRecordFormatter)); + new BufferTestHelper() + .baseTestMessageBuffer( + loopCount, + false, + false, + true, + new Buffer(new Buffer.Config(), fluentdRecordFormatter)); } + } - static class BufferTestHelper - { - private static final Logger LOG = LoggerFactory.getLogger(BufferTestHelper.class); - private final String longStr; - private HashMap tagCounts; - private String minName; - private String maxName; - private int minAge; - private int maxAge; - private int longCommentCount; + static class BufferTestHelper { + private static final Logger LOG = LoggerFactory.getLogger(BufferTestHelper.class); + private final String longStr; + private HashMap tagCounts; + private String minName; + private String maxName; + private int minAge; + private int maxAge; + private int longCommentCount; - BufferTestHelper() - { - StringBuilder stringBuilder = new StringBuilder(); - for (int i = 0; i < 300; i++) { - stringBuilder.append("xxxxxxxxxx"); - } - longStr = stringBuilder.toString(); - tagCounts = new HashMap<>(); - minName = "zzzzzzzzzzzzzzzzzzzz"; - maxName = ""; - minAge = Integer.MAX_VALUE; - maxAge = Integer.MIN_VALUE; - longCommentCount = 0; - } + BufferTestHelper() { + StringBuilder stringBuilder = new StringBuilder(); + for (int i = 0; i < 300; i++) { + stringBuilder.append("xxxxxxxxxx"); + } + longStr = stringBuilder.toString(); + tagCounts = new HashMap<>(); + minName = "zzzzzzzzzzzzzzzzzzzz"; + maxName = ""; + minAge = Integer.MAX_VALUE; + maxAge = Integer.MIN_VALUE; + longCommentCount = 0; + } - void baseTestMessageBuffer(final int loopCount, final boolean multiTags, final boolean syncFlush, final boolean eventTime, final Buffer buffer) - throws IOException, InterruptedException - { - assertThat(buffer.getBufferUsage()).isEqualTo(0f); - assertThat(buffer.getAllocatedSize()).isEqualTo(0L); - assertThat(buffer.getBufferedDataSize()).isEqualTo(0L); + void baseTestMessageBuffer( + final int loopCount, + final boolean multiTags, + final boolean syncFlush, + final boolean eventTime, + final Buffer buffer) + throws IOException, InterruptedException { + assertThat(buffer.getBufferUsage()).isEqualTo(0f); + assertThat(buffer.getAllocatedSize()).isEqualTo(0L); + assertThat(buffer.getBufferedDataSize()).isEqualTo(0L); - final int concurrency = 4; - final CountDownLatch latch = new CountDownLatch(concurrency); + final int concurrency = 4; + final CountDownLatch latch = new CountDownLatch(concurrency); - TCPSender.Config config = new TCPSender.Config(); - config.setPort(24229); - final MockTCPSender sender = new MockTCPSender(config); - Ingester ingester = new FluentdIngester(new FluentdIngester.Config(), sender); + TCPSender.Config config = new TCPSender.Config(); + config.setPort(24229); + final MockTCPSender sender = new MockTCPSender(config); + Ingester ingester = new FluentdIngester(new FluentdIngester.Config(), sender); - Runnable emitTask = () -> { - try { - for (int i = 0; i < loopCount; i++) { - HashMap data = new HashMap<>(); - data.put("name", String.format("komamitsu%06d", i)); - data.put("age", i); - data.put("comment", i % 31 == 0 ? longStr : "hello"); - String tag = multiTags ? String.format("foodb%d.bartbl%d", i % 4, i % 4) : "foodb.bartbl"; - if (eventTime) { - buffer.append(tag, new EventTime((int) (System.currentTimeMillis() / 1000), 999999999), data); - } - else { - buffer.append(tag, System.currentTimeMillis(), data); - } - } - latch.countDown(); + Runnable emitTask = + () -> { + try { + for (int i = 0; i < loopCount; i++) { + HashMap data = new HashMap<>(); + data.put("name", String.format("komamitsu%06d", i)); + data.put("age", i); + data.put("comment", i % 31 == 0 ? longStr : "hello"); + String tag = + multiTags ? String.format("foodb%d.bartbl%d", i % 4, i % 4) : "foodb.bartbl"; + if (eventTime) { + buffer.append( + tag, + new EventTime((int) (System.currentTimeMillis() / 1000), 999999999), + data); + } else { + buffer.append(tag, System.currentTimeMillis(), data); } - catch (Exception e) { - e.printStackTrace(); - } - }; - - final ExecutorService flushService = Executors.newSingleThreadExecutor(); - if (!syncFlush) { - flushService.execute(() -> { - while (!flushService.isShutdown()) { - try { - TimeUnit.MILLISECONDS.sleep(100L); - buffer.flush(ingester, false); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - catch (IOException e) { - e.printStackTrace(); - return; - } - } - }); + } + latch.countDown(); + } catch (Exception e) { + e.printStackTrace(); } + }; - long start = System.currentTimeMillis(); - final ExecutorService executorService = Executors.newFixedThreadPool(concurrency); - for (int i = 0; i < concurrency; i++) { - executorService.execute(emitTask); - } - assertTrue(latch.await(10, TimeUnit.SECONDS)); - assertThat(buffer.getBufferUsage()).isGreaterThan(0f); - assertThat(buffer.getAllocatedSize()).isGreaterThan(0L); - assertThat(buffer.getBufferedDataSize()).isGreaterThan(0L); + final ExecutorService flushService = Executors.newSingleThreadExecutor(); + if (!syncFlush) { + flushService.execute( + () -> { + while (!flushService.isShutdown()) { + try { + TimeUnit.MILLISECONDS.sleep(100L); + buffer.flush(ingester, false); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (IOException e) { + e.printStackTrace(); + return; + } + } + }); + } - buffer.flush(ingester, true); - buffer.close(); - long end = System.currentTimeMillis(); + long start = System.currentTimeMillis(); + final ExecutorService executorService = Executors.newFixedThreadPool(concurrency); + for (int i = 0; i < concurrency; i++) { + executorService.execute(emitTask); + } + assertTrue(latch.await(10, TimeUnit.SECONDS)); + assertThat(buffer.getBufferUsage()).isGreaterThan(0f); + assertThat(buffer.getAllocatedSize()).isGreaterThan(0L); + assertThat(buffer.getBufferedDataSize()).isGreaterThan(0L); - executorService.shutdown(); - executorService.awaitTermination(10, TimeUnit.SECONDS); - if (!executorService.isTerminated()) { - executorService.shutdownNow(); - } - flushService.shutdown(); - flushService.awaitTermination(10, TimeUnit.SECONDS); - if (!flushService.isTerminated()) { - flushService.shutdownNow(); - } - buffer.close(); // Just in case - assertThat(buffer.getBufferUsage()).isEqualTo(0f); - assertThat(buffer.getAllocatedSize()).isEqualTo(0L); - assertThat(buffer.getBufferedDataSize()).isEqualTo(0L); + buffer.flush(ingester, true); + buffer.close(); + long end = System.currentTimeMillis(); - int totalLoopCount = concurrency * loopCount; + executorService.shutdown(); + executorService.awaitTermination(10, TimeUnit.SECONDS); + if (!executorService.isTerminated()) { + executorService.shutdownNow(); + } + flushService.shutdown(); + flushService.awaitTermination(10, TimeUnit.SECONDS); + if (!flushService.isTerminated()) { + flushService.shutdownNow(); + } + buffer.close(); // Just in case + assertThat(buffer.getBufferUsage()).isEqualTo(0f); + assertThat(buffer.getAllocatedSize()).isEqualTo(0L); + assertThat(buffer.getBufferedDataSize()).isEqualTo(0L); - int recordCount = 0; - ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); - objectMapper.disable(JsonParser.Feature.AUTO_CLOSE_SOURCE); - for (MockTCPSender.Event event : sender.getEvents()) { - byte[] bytes = event.getAllBytes(); + int totalLoopCount = concurrency * loopCount; - MessageUnpacker messageUnpacker = MessagePack.newDefaultUnpacker(bytes); - assertEquals(3, messageUnpacker.unpackArrayHeader()); + int recordCount = 0; + ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); + objectMapper.disable(JsonParser.Feature.AUTO_CLOSE_SOURCE); + for (MockTCPSender.Event event : sender.getEvents()) { + byte[] bytes = event.getAllBytes(); - String tag = messageUnpacker.unpackString(); - byte[] payload = messageUnpacker.readPayload(messageUnpacker.unpackBinaryHeader()); - messageUnpacker = MessagePack.newDefaultUnpacker(payload); - while (messageUnpacker.hasNext()) { - assertEquals(2, messageUnpacker.unpackArrayHeader()); + MessageUnpacker messageUnpacker = MessagePack.newDefaultUnpacker(bytes); + assertEquals(3, messageUnpacker.unpackArrayHeader()); - ImmutableValue timestamp = messageUnpacker.unpackValue(); + String tag = messageUnpacker.unpackString(); + byte[] payload = messageUnpacker.readPayload(messageUnpacker.unpackBinaryHeader()); + messageUnpacker = MessagePack.newDefaultUnpacker(payload); + while (messageUnpacker.hasNext()) { + assertEquals(2, messageUnpacker.unpackArrayHeader()); - int size = messageUnpacker.unpackMapHeader(); - assertEquals(3, size); - Map data = new HashMap<>(); - for (int i = 0; i < size; i++) { - String key = messageUnpacker.unpackString(); - ImmutableValue value = messageUnpacker.unpackValue(); - if (value.isStringValue()) { - data.put(key, value.asStringValue().asString()); - } - else if (value.isIntegerValue()) { - data.put(key, value.asIntegerValue().asInt()); - } - } + ImmutableValue timestamp = messageUnpacker.unpackValue(); - analyzeResult(tag, timestamp, data, start, end, eventTime); - recordCount++; - } + int size = messageUnpacker.unpackMapHeader(); + assertEquals(3, size); + Map data = new HashMap<>(); + for (int i = 0; i < size; i++) { + String key = messageUnpacker.unpackString(); + ImmutableValue value = messageUnpacker.unpackValue(); + if (value.isStringValue()) { + data.put(key, value.asStringValue().asString()); + } else if (value.isIntegerValue()) { + data.put(key, value.asIntegerValue().asInt()); } + } - assertEquals(totalLoopCount, recordCount); - - if (multiTags) { - assertEquals(4, tagCounts.size()); - for (int i = 0; i < 4; i++) { - int count = tagCounts.get(String.format("foodb%d.bartbl%d", i, i)); - assertTrue(totalLoopCount / 4 - 4 <= count && count <= totalLoopCount / 4 + 4); - } - } - else { - assertEquals(1, tagCounts.size()); - int count = tagCounts.get("foodb.bartbl"); - assertEquals(totalLoopCount, count); - } + analyzeResult(tag, timestamp, data, start, end, eventTime); + recordCount++; + } + } - assertEquals("komamitsu000000", minName); - assertEquals(String.format("komamitsu%06d", loopCount - 1), maxName); - assertEquals(0, minAge); - assertEquals(loopCount - 1, maxAge); + assertEquals(totalLoopCount, recordCount); - assertTrue(totalLoopCount / 31 - 5 <= longCommentCount && longCommentCount <= totalLoopCount / 31 + 5); + if (multiTags) { + assertEquals(4, tagCounts.size()); + for (int i = 0; i < 4; i++) { + int count = tagCounts.get(String.format("foodb%d.bartbl%d", i, i)); + assertTrue(totalLoopCount / 4 - 4 <= count && count <= totalLoopCount / 4 + 4); } + } else { + assertEquals(1, tagCounts.size()); + int count = tagCounts.get("foodb.bartbl"); + assertEquals(totalLoopCount, count); + } - private void analyzeResult(String tag, ImmutableValue timestamp, Map data, long start, long end, boolean eventTime) - { - Integer count = tagCounts.get(tag); - if (count == null) { - count = 0; - } - tagCounts.put(tag, count + 1); + assertEquals("komamitsu000000", minName); + assertEquals(String.format("komamitsu%06d", loopCount - 1), maxName); + assertEquals(0, minAge); + assertEquals(loopCount - 1, maxAge); - if (eventTime) { - assertThat(timestamp.isExtensionValue()).isTrue(); - ExtensionValue tsInEventTime = timestamp.asExtensionValue(); - assertThat(tsInEventTime.getType()).isEqualTo((byte) 0x00); - ByteBuffer secondsAndNanoSeconds = ByteBuffer.wrap(tsInEventTime.getData()); - int seconds = secondsAndNanoSeconds.getInt(); - int nanoSeconds = secondsAndNanoSeconds.getInt(); - assertTrue(start / 1000 <= seconds && seconds <= end / 1000); - assertThat(nanoSeconds).isEqualTo(999999999); - } - else { - assertThat(timestamp.isIntegerValue()).isTrue(); - long tsInEpochMilli = timestamp.asIntegerValue().asLong(); - assertTrue(start <= tsInEpochMilli && tsInEpochMilli <= end); - } + assertTrue( + totalLoopCount / 31 - 5 <= longCommentCount + && longCommentCount <= totalLoopCount / 31 + 5); + } - assertEquals(3, data.size()); - String name = (String) data.get("name"); - int age = (Integer) data.get("age"); - String comment = (String) data.get("comment"); - if (name.compareTo(minName) < 0) { - minName = name; - } - if (name.compareTo(maxName) > 0) { - maxName = name; - } - if (age < minAge) { - minAge = age; - } - if (age > maxAge) { - maxAge = age; - } + private void analyzeResult( + String tag, + ImmutableValue timestamp, + Map data, + long start, + long end, + boolean eventTime) { + Integer count = tagCounts.get(tag); + if (count == null) { + count = 0; + } + tagCounts.put(tag, count + 1); - if (comment.equals("hello")) { - // expected - } - else if (comment.equals(longStr)) { - longCommentCount++; - } - else { - assertTrue(false); - } - } + if (eventTime) { + assertThat(timestamp.isExtensionValue()).isTrue(); + ExtensionValue tsInEventTime = timestamp.asExtensionValue(); + assertThat(tsInEventTime.getType()).isEqualTo((byte) 0x00); + ByteBuffer secondsAndNanoSeconds = ByteBuffer.wrap(tsInEventTime.getData()); + int seconds = secondsAndNanoSeconds.getInt(); + int nanoSeconds = secondsAndNanoSeconds.getInt(); + assertTrue(start / 1000 <= seconds && seconds <= end / 1000); + assertThat(nanoSeconds).isEqualTo(999999999); + } else { + assertThat(timestamp.isIntegerValue()).isTrue(); + long tsInEpochMilli = timestamp.asIntegerValue().asLong(); + assertTrue(start <= tsInEpochMilli && tsInEpochMilli <= end); + } + + assertEquals(3, data.size()); + String name = (String) data.get("name"); + int age = (Integer) data.get("age"); + String comment = (String) data.get("comment"); + if (name.compareTo(minName) < 0) { + minName = name; + } + if (name.compareTo(maxName) > 0) { + maxName = name; + } + if (age < minAge) { + minAge = age; + } + if (age > maxAge) { + maxAge = age; + } + + if (comment.equals("hello")) { + // expected + } else if (comment.equals(longStr)) { + longCommentCount++; + } else { + assertTrue(false); + } } -} \ No newline at end of file + } +} diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/AbstractFluentdServer.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/AbstractFluentdServer.java index bb319b4b..44327b9a 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/AbstractFluentdServer.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/AbstractFluentdServer.java @@ -16,19 +16,7 @@ package org.komamitsu.fluency.fluentd; -import org.msgpack.core.MessagePack; -import org.msgpack.core.MessageUnpacker; -import org.msgpack.value.ExtensionValue; -import org.msgpack.value.ImmutableArrayValue; -import org.msgpack.value.ImmutableValue; -import org.msgpack.value.MapValue; -import org.msgpack.value.RawValue; -import org.msgpack.value.StringValue; -import org.msgpack.value.Value; -import org.msgpack.value.ValueFactory; -import org.msgpack.value.ValueType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; import java.io.PipedInputStream; @@ -42,243 +30,244 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import org.msgpack.core.MessagePack; +import org.msgpack.core.MessageUnpacker; +import org.msgpack.value.ExtensionValue; +import org.msgpack.value.ImmutableArrayValue; +import org.msgpack.value.ImmutableValue; +import org.msgpack.value.MapValue; +import org.msgpack.value.StringValue; +import org.msgpack.value.Value; +import org.msgpack.value.ValueFactory; +import org.msgpack.value.ValueType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import static org.junit.jupiter.api.Assertions.assertEquals; +public abstract class AbstractFluentdServer extends MockTCPServer { + private static final Logger LOG = LoggerFactory.getLogger(AbstractFluentdServer.class); + private static final Charset CHARSET = StandardCharsets.UTF_8; + private FluentdEventHandler fluentdEventHandler; -public abstract class AbstractFluentdServer - extends MockTCPServer -{ - private static final Logger LOG = LoggerFactory.getLogger(AbstractFluentdServer.class); - private static final Charset CHARSET = StandardCharsets.UTF_8; - private FluentdEventHandler fluentdEventHandler; + AbstractFluentdServer(boolean sslEnabled) { + super(sslEnabled); + } - AbstractFluentdServer(boolean sslEnabled) - { - super(sslEnabled); + @Override + protected synchronized MockTCPServer.EventHandler getEventHandler() { + if (this.fluentdEventHandler == null) { + this.fluentdEventHandler = new FluentdEventHandler(getFluentdEventHandler()); } + return fluentdEventHandler; + } - @Override - protected synchronized MockTCPServer.EventHandler getEventHandler() - { - if (this.fluentdEventHandler == null) { - this.fluentdEventHandler = new FluentdEventHandler(getFluentdEventHandler()); - } - return fluentdEventHandler; - } + protected abstract EventHandler getFluentdEventHandler(); - protected abstract EventHandler getFluentdEventHandler(); + public interface EventHandler { + void onConnect(Socket acceptSocket); - public interface EventHandler - { - void onConnect(Socket acceptSocket); + void onReceive(String tag, long timestampMillis, MapValue data); - void onReceive(String tag, long timestampMillis, MapValue data); + void onClose(Socket accpetSocket); + } - void onClose(Socket accpetSocket); - } + private static class FluentdEventHandler implements MockTCPServer.EventHandler { + private static final StringValue KEY_OPTION_SIZE = ValueFactory.newString("size"); + private static final StringValue KEY_OPTION_CHUNK = ValueFactory.newString("chunk"); + private final EventHandler eventHandler; + private final ExecutorService executorService = Executors.newCachedThreadPool(); + private final Map fluentdTasks = + new ConcurrentHashMap(); - private static class FluentdEventHandler - implements MockTCPServer.EventHandler - { - private static final StringValue KEY_OPTION_SIZE = ValueFactory.newString("size"); - private static final StringValue KEY_OPTION_CHUNK = ValueFactory.newString("chunk"); - private final EventHandler eventHandler; - private final ExecutorService executorService = Executors.newCachedThreadPool(); - private final Map fluentdTasks = new ConcurrentHashMap(); + private FluentdEventHandler(EventHandler eventHandler) { + this.eventHandler = eventHandler; + } - private FluentdEventHandler(EventHandler eventHandler) - { - this.eventHandler = eventHandler; - } + private void ack(Socket acceptSocket, String ackResponseToken) throws IOException { + byte[] ackResponseTokenBytes = ackResponseToken.getBytes(CHARSET); + ByteBuffer byteBuffer = + ByteBuffer.allocate( + 1 /* map header */ + + 1 /* key header */ + + 3 /* key body */ + + 2 /* value header(including len) */ + + ackResponseTokenBytes.length); - private void ack(Socket acceptSocket, String ackResponseToken) - throws IOException - { - byte[] ackResponseTokenBytes = ackResponseToken.getBytes(CHARSET); - ByteBuffer byteBuffer = ByteBuffer.allocate( - 1 /* map header */ + - 1 /* key header */ + - 3 /* key body */ + - 2 /* value header(including len) */ + - ackResponseTokenBytes.length); + byteBuffer.put((byte) 0x81); /* map header */ + byteBuffer.put((byte) 0xA3); /* key header */ + byteBuffer.put("ack".getBytes(CHARSET)); /* key body */ + byteBuffer.put((byte) 0xD9); + byteBuffer.put((byte) ackResponseTokenBytes.length); + byteBuffer.put(ackResponseTokenBytes); + byteBuffer.flip(); + acceptSocket.getOutputStream().write(byteBuffer.array()); + } - byteBuffer.put((byte) 0x81); /* map header */ - byteBuffer.put((byte) 0xA3); /* key header */ - byteBuffer.put("ack".getBytes(CHARSET)); /* key body */ - byteBuffer.put((byte) 0xD9); - byteBuffer.put((byte) ackResponseTokenBytes.length); - byteBuffer.put(ackResponseTokenBytes); - byteBuffer.flip(); - acceptSocket.getOutputStream().write(byteBuffer.array()); - } + @Override + public void onConnect(final Socket acceptSocket) { + eventHandler.onConnect(acceptSocket); + try { + FluentdAcceptTask fluentdAcceptTask = new FluentdAcceptTask(acceptSocket); + fluentdTasks.put(acceptSocket, fluentdAcceptTask); + executorService.execute(fluentdAcceptTask); + } catch (IOException e) { + fluentdTasks.remove(acceptSocket); + throw new IllegalStateException("Failed to create FluentdAcceptTask", e); + } + } - @Override - public void onConnect(final Socket acceptSocket) - { - eventHandler.onConnect(acceptSocket); - try { - FluentdAcceptTask fluentdAcceptTask = new FluentdAcceptTask(acceptSocket); - fluentdTasks.put(acceptSocket, fluentdAcceptTask); - executorService.execute(fluentdAcceptTask); - } - catch (IOException e) { - fluentdTasks.remove(acceptSocket); - throw new IllegalStateException("Failed to create FluentdAcceptTask", e); - } - } + @Override + public void onReceive(Socket acceptSocket, int len, byte[] data) { + FluentdAcceptTask fluentdAcceptTask = fluentdTasks.get(acceptSocket); + if (fluentdAcceptTask == null) { + throw new IllegalStateException("fluentAccept is null: this=" + this); + } - @Override - public void onReceive(Socket acceptSocket, int len, byte[] data) - { - FluentdAcceptTask fluentdAcceptTask = fluentdTasks.get(acceptSocket); - if (fluentdAcceptTask == null) { - throw new IllegalStateException("fluentAccept is null: this=" + this); - } + LOG.trace( + "onReceived: local.port={}, remote.port={}, dataLen={}", + acceptSocket.getLocalPort(), + acceptSocket.getPort(), + len); + try { + fluentdAcceptTask.getPipedOutputStream().write(data, 0, len); + fluentdAcceptTask.getPipedOutputStream().flush(); + } catch (IOException e) { + throw new RuntimeException("Failed to call PipedOutputStream.write(): this=" + this); + } + } - LOG.trace("onReceived: local.port={}, remote.port={}, dataLen={}", acceptSocket.getLocalPort(), acceptSocket.getPort(), len); - try { - fluentdAcceptTask.getPipedOutputStream().write(data, 0, len); - fluentdAcceptTask.getPipedOutputStream().flush(); - } - catch (IOException e) { - throw new RuntimeException("Failed to call PipedOutputStream.write(): this=" + this); - } + @Override + public void onClose(Socket acceptSocket) { + eventHandler.onClose(acceptSocket); + FluentdAcceptTask fluentdAcceptTask = fluentdTasks.remove(acceptSocket); + if (fluentdAcceptTask == null) { + return; + } + try { + if (fluentdAcceptTask.getPipedInputStream() != null) { + fluentdAcceptTask.getPipedInputStream().close(); } - - @Override - public void onClose(Socket acceptSocket) - { - eventHandler.onClose(acceptSocket); - FluentdAcceptTask fluentdAcceptTask = fluentdTasks.remove(acceptSocket); - if (fluentdAcceptTask == null) { - return; - } - try { - if (fluentdAcceptTask.getPipedInputStream() != null) { - fluentdAcceptTask.getPipedInputStream().close(); - } - } - catch (IOException e) { - LOG.warn("Failed to close PipedInputStream"); - } - try { - if (fluentdAcceptTask.getPipedOutputStream() != null) { - fluentdAcceptTask.getPipedOutputStream().close(); - } - } - catch (IOException e) { - LOG.warn("Failed to close PipedOutputStream"); - } + } catch (IOException e) { + LOG.warn("Failed to close PipedInputStream"); + } + try { + if (fluentdAcceptTask.getPipedOutputStream() != null) { + fluentdAcceptTask.getPipedOutputStream().close(); } + } catch (IOException e) { + LOG.warn("Failed to close PipedOutputStream"); + } + } - private class FluentdAcceptTask - implements Runnable - { - private final Socket acceptSocket; - private final PipedInputStream pipedInputStream; - private final PipedOutputStream pipedOutputStream; + private class FluentdAcceptTask implements Runnable { + private final Socket acceptSocket; + private final PipedInputStream pipedInputStream; + private final PipedOutputStream pipedOutputStream; - private FluentdAcceptTask(Socket acceptSocket) - throws IOException - { - this.acceptSocket = acceptSocket; - this.pipedOutputStream = new PipedOutputStream(); - this.pipedInputStream = new PipedInputStream(pipedOutputStream); - } + private FluentdAcceptTask(Socket acceptSocket) throws IOException { + this.acceptSocket = acceptSocket; + this.pipedOutputStream = new PipedOutputStream(); + this.pipedInputStream = new PipedInputStream(pipedOutputStream); + } - PipedInputStream getPipedInputStream() - { - return pipedInputStream; - } + PipedInputStream getPipedInputStream() { + return pipedInputStream; + } - PipedOutputStream getPipedOutputStream() - { - return pipedOutputStream; - } + PipedOutputStream getPipedOutputStream() { + return pipedOutputStream; + } - @Override - public void run() - { - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(pipedInputStream); + @Override + public void run() { + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(pipedInputStream); - try { - while (!executorService.isTerminated()) { - ImmutableValue value; - try { - if (!unpacker.hasNext()) { - break; - } - value = unpacker.unpackValue(); - LOG.trace("Received a value: local.port={}, remote.port={}", acceptSocket.getLocalPort(), acceptSocket.getPort()); - } - catch (Exception e) { - LOG.debug("Fluentd accept task received IOException: {}", e.getMessage()); - break; - } - assertEquals(ValueType.ARRAY, value.getValueType()); - ImmutableArrayValue rootValue = value.asArrayValue(); - assertEquals(rootValue.size(), 3); - - String tag = rootValue.get(0).toString(); - Value secondValue = rootValue.get(1); - - // PackedForward - byte[] packedBytes = secondValue.asRawValue().asByteArray(); - MessageUnpacker eventsUnpacker = MessagePack.newDefaultUnpacker(packedBytes); - while (eventsUnpacker.hasNext()) { - ImmutableArrayValue arrayValue = eventsUnpacker.unpackValue().asArrayValue(); - assertEquals(2, arrayValue.size()); - Value timestampValue = arrayValue.get(0); - MapValue mapValue = arrayValue.get(1).asMapValue(); - long timestampMillis; - if (timestampValue.isIntegerValue()) { - timestampMillis = timestampValue.asIntegerValue().asLong() * 1000; - } - else if (timestampValue.isExtensionValue()) { - ExtensionValue extensionValue = timestampValue.asExtensionValue(); - if (extensionValue.getType() != 0) { - throw new IllegalArgumentException("Unexpected extension type: " + extensionValue.getType()); - } - byte[] data = extensionValue.getData(); - long seconds = ByteBuffer.wrap(data, 0, 4).order(ByteOrder.BIG_ENDIAN).getInt(); - long nanos = ByteBuffer.wrap(data, 4, 4).order(ByteOrder.BIG_ENDIAN).getInt(); - timestampMillis = seconds * 1000 + nanos / 1000000; - } - else { - throw new IllegalArgumentException("Unexpected value type: " + timestampValue); - } - eventHandler.onReceive(tag, timestampMillis, mapValue); - } + try { + while (!executorService.isTerminated()) { + ImmutableValue value; + try { + if (!unpacker.hasNext()) { + break; + } + value = unpacker.unpackValue(); + LOG.trace( + "Received a value: local.port={}, remote.port={}", + acceptSocket.getLocalPort(), + acceptSocket.getPort()); + } catch (Exception e) { + LOG.debug("Fluentd accept task received IOException: {}", e.getMessage()); + break; + } + assertEquals(ValueType.ARRAY, value.getValueType()); + ImmutableArrayValue rootValue = value.asArrayValue(); + assertEquals(rootValue.size(), 3); - // Option - Map map = rootValue.get(2).asMapValue().map(); - // "size" - assertEquals(map.get(KEY_OPTION_SIZE).asIntegerValue().asLong(), packedBytes.length); - // "chunk" - Value chunk = map.get(KEY_OPTION_CHUNK); - if (chunk != null) { - ack(acceptSocket, chunk.asStringValue().asString()); - } - } + String tag = rootValue.get(0).toString(); + Value secondValue = rootValue.get(1); - try { - LOG.debug("Closing unpacker: this={}, local.port={}, remote.port={}", this, acceptSocket.getLocalPort(), acceptSocket.getPort()); - unpacker.close(); - } - catch (IOException e) { - LOG.warn("Failed to close unpacker quietly: this={}, unpacker={}", this, unpacker); - } - } - catch (Throwable e) { - LOG.error("Fluentd server failed: this=" + this + ", local.port=" + acceptSocket.getLocalPort() + ", remote.port=" + acceptSocket.getPort(), e); - try { - acceptSocket.close(); - } - catch (IOException e1) { - LOG.warn("Failed to close accept socket quietly", e1); - } + // PackedForward + byte[] packedBytes = secondValue.asRawValue().asByteArray(); + MessageUnpacker eventsUnpacker = MessagePack.newDefaultUnpacker(packedBytes); + while (eventsUnpacker.hasNext()) { + ImmutableArrayValue arrayValue = eventsUnpacker.unpackValue().asArrayValue(); + assertEquals(2, arrayValue.size()); + Value timestampValue = arrayValue.get(0); + MapValue mapValue = arrayValue.get(1).asMapValue(); + long timestampMillis; + if (timestampValue.isIntegerValue()) { + timestampMillis = timestampValue.asIntegerValue().asLong() * 1000; + } else if (timestampValue.isExtensionValue()) { + ExtensionValue extensionValue = timestampValue.asExtensionValue(); + if (extensionValue.getType() != 0) { + throw new IllegalArgumentException( + "Unexpected extension type: " + extensionValue.getType()); } + byte[] data = extensionValue.getData(); + long seconds = ByteBuffer.wrap(data, 0, 4).order(ByteOrder.BIG_ENDIAN).getInt(); + long nanos = ByteBuffer.wrap(data, 4, 4).order(ByteOrder.BIG_ENDIAN).getInt(); + timestampMillis = seconds * 1000 + nanos / 1000000; + } else { + throw new IllegalArgumentException("Unexpected value type: " + timestampValue); + } + eventHandler.onReceive(tag, timestampMillis, mapValue); + } + + // Option + Map map = rootValue.get(2).asMapValue().map(); + // "size" + assertEquals(map.get(KEY_OPTION_SIZE).asIntegerValue().asLong(), packedBytes.length); + // "chunk" + Value chunk = map.get(KEY_OPTION_CHUNK); + if (chunk != null) { + ack(acceptSocket, chunk.asStringValue().asString()); } + } + + try { + LOG.debug( + "Closing unpacker: this={}, local.port={}, remote.port={}", + this, + acceptSocket.getLocalPort(), + acceptSocket.getPort()); + unpacker.close(); + } catch (IOException e) { + LOG.warn("Failed to close unpacker quietly: this={}, unpacker={}", this, unpacker); + } + } catch (Throwable e) { + LOG.error( + "Fluentd server failed: this=" + + this + + ", local.port=" + + acceptSocket.getLocalPort() + + ", remote.port=" + + acceptSocket.getPort(), + e); + try { + acceptSocket.close(); + } catch (IOException e1) { + LOG.warn("Failed to close accept socket quietly", e1); + } } + } } + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ConfigurableTestServer.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ConfigurableTestServer.java index 3e16625a..b74826c7 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ConfigurableTestServer.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ConfigurableTestServer.java @@ -25,87 +25,76 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; -public class ConfigurableTestServer -{ - private final boolean sslEnabled; +public class ConfigurableTestServer { + private final boolean sslEnabled; - public ConfigurableTestServer(boolean sslEnabled) - { - this.sslEnabled = sslEnabled; - } + public ConfigurableTestServer(boolean sslEnabled) { + this.sslEnabled = sslEnabled; + } - Exception run(final WithClientSocket withClientSocket, final WithServerPort withServerPort, long timeoutMilli) - throws Throwable - { - ExecutorService executorService = Executors.newCachedThreadPool(); - final AtomicReference serverSocket = new AtomicReference(); + Exception run( + final WithClientSocket withClientSocket, + final WithServerPort withServerPort, + long timeoutMilli) + throws Throwable { + ExecutorService executorService = Executors.newCachedThreadPool(); + final AtomicReference serverSocket = new AtomicReference(); - try { - if (sslEnabled) { - serverSocket.set(SSLTestSocketFactories.createServerSocket()); - } - else { - serverSocket.set(new ServerSocket(0)); - } + try { + if (sslEnabled) { + serverSocket.set(SSLTestSocketFactories.createServerSocket()); + } else { + serverSocket.set(new ServerSocket(0)); + } - final int serverPort = serverSocket.get().getLocalPort(); + final int serverPort = serverSocket.get().getLocalPort(); - Future serverSideFuture = executorService.submit(new Callable() - { + Future serverSideFuture = + executorService.submit( + new Callable() { @Override - public Void call() - throws Exception - { - Socket acceptSocket = serverSocket.get().accept(); - try { - withClientSocket.run(acceptSocket); - } - finally { - acceptSocket.close(); - } - return null; + public Void call() throws Exception { + Socket acceptSocket = serverSocket.get().accept(); + try { + withClientSocket.run(acceptSocket); + } finally { + acceptSocket.close(); + } + return null; } - }); + }); - Future testTaskFuture = executorService.submit(new Callable() - { + Future testTaskFuture = + executorService.submit( + new Callable() { @Override - public Void call() - throws Exception - { - withServerPort.run(serverPort); - return null; + public Void call() throws Exception { + withServerPort.run(serverPort); + return null; } - }); + }); - try { - testTaskFuture.get(timeoutMilli, TimeUnit.MILLISECONDS); - } - catch (Exception e) { - return e; - } - finally { - serverSideFuture.get(); - } - } - finally { - executorService.shutdownNow(); - if (serverSocket.get() != null) { - serverSocket.get().close(); - } - } - return null; + try { + testTaskFuture.get(timeoutMilli, TimeUnit.MILLISECONDS); + } catch (Exception e) { + return e; + } finally { + serverSideFuture.get(); + } + } finally { + executorService.shutdownNow(); + if (serverSocket.get() != null) { + serverSocket.get().close(); + } } + return null; + } - interface WithClientSocket - { - void run(Socket clientSocket) - throws Exception; - } + interface WithClientSocket { + void run(Socket clientSocket) throws Exception; + } - interface WithServerPort - { - void run(int serverPort) - throws Exception; - } + interface WithServerPort { + void run(int serverPort) throws Exception; + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/EventTimeTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/EventTimeTest.java index 63bd4d9c..7b4da294 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/EventTimeTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/EventTimeTest.java @@ -16,99 +16,96 @@ package org.komamitsu.fluency.fluentd; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.ByteBuffer; import org.junit.jupiter.api.Test; import org.komamitsu.fluency.EventTime; import org.msgpack.jackson.dataformat.MessagePackFactory; -import java.nio.ByteBuffer; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertThrows; - -class EventTimeTest -{ - @Test - void instantiation() +class EventTimeTest { + @Test + void instantiation() { { - { - long now = System.currentTimeMillis(); - EventTime eventTime = EventTime.fromEpoch(now / 1000); - assertThat(eventTime.getSeconds()).isEqualTo(now / 1000); - assertThat(eventTime.getNanoseconds()).isEqualTo(0L); - } - - { - long now = System.currentTimeMillis(); - EventTime eventTime = EventTime.fromEpoch(now / 1000, 999999999L); - assertThat(eventTime.getSeconds()).isEqualTo(now / 1000); - assertThat(eventTime.getNanoseconds()).isEqualTo(999999999L); - } - - { - long now = System.currentTimeMillis(); - EventTime eventTime = EventTime.fromEpochMilli(now); - assertThat(eventTime.getSeconds()).isEqualTo(now / 1000); - assertThat(eventTime.getNanoseconds()).isEqualTo(now % 1000 * 1000000); - } + long now = System.currentTimeMillis(); + EventTime eventTime = EventTime.fromEpoch(now / 1000); + assertThat(eventTime.getSeconds()).isEqualTo(now / 1000); + assertThat(eventTime.getNanoseconds()).isEqualTo(0L); + } - { - EventTime eventTime = EventTime.fromEpoch(0xFFFFFFFFL, 0xFFFFFFFFL); - assertThat(eventTime.getSeconds()).isEqualTo(0xFFFFFFFFL); - assertThat(eventTime.getNanoseconds()).isEqualTo(0xFFFFFFFFL); - } + { + long now = System.currentTimeMillis(); + EventTime eventTime = EventTime.fromEpoch(now / 1000, 999999999L); + assertThat(eventTime.getSeconds()).isEqualTo(now / 1000); + assertThat(eventTime.getNanoseconds()).isEqualTo(999999999L); } - @Test - void instantiationWithTooLargeSeconds() { - assertThrows(IllegalArgumentException.class, () -> { - EventTime.fromEpoch(0x100000000L, 0L); - }); + long now = System.currentTimeMillis(); + EventTime eventTime = EventTime.fromEpochMilli(now); + assertThat(eventTime.getSeconds()).isEqualTo(now / 1000); + assertThat(eventTime.getNanoseconds()).isEqualTo(now % 1000 * 1000000); } - @Test - void instantiationWithTooLargeNanoSeconds() { - assertThrows(IllegalArgumentException.class, () -> { - EventTime.fromEpoch(0L, 0x100000000L); - }); + EventTime eventTime = EventTime.fromEpoch(0xFFFFFFFFL, 0xFFFFFFFFL); + assertThat(eventTime.getSeconds()).isEqualTo(0xFFFFFFFFL); + assertThat(eventTime.getNanoseconds()).isEqualTo(0xFFFFFFFFL); } + } + + @Test + void instantiationWithTooLargeSeconds() { + assertThrows( + IllegalArgumentException.class, + () -> { + EventTime.fromEpoch(0x100000000L, 0L); + }); + } + + @Test + void instantiationWithTooLargeNanoSeconds() { + assertThrows( + IllegalArgumentException.class, + () -> { + EventTime.fromEpoch(0L, 0x100000000L); + }); + } - @Test - void serialize() - throws JsonProcessingException + @Test + void serialize() throws JsonProcessingException { { - { - long now = System.currentTimeMillis(); - EventTime eventTime = EventTime.fromEpoch(now / 1000, 999999999); - ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); - byte[] bytes = objectMapper.writeValueAsBytes(eventTime); - ByteBuffer byteBuffer = ByteBuffer.wrap(bytes); - assertThat(byteBuffer.get()).isEqualTo((byte) 0xD7); - assertThat(byteBuffer.get()).isEqualTo((byte) 0x00); - assertThat(byteBuffer.getInt()).isEqualTo((int) (now / 1000)); - assertThat(byteBuffer.getInt()).isEqualTo(999999999); - } + long now = System.currentTimeMillis(); + EventTime eventTime = EventTime.fromEpoch(now / 1000, 999999999); + ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); + byte[] bytes = objectMapper.writeValueAsBytes(eventTime); + ByteBuffer byteBuffer = ByteBuffer.wrap(bytes); + assertThat(byteBuffer.get()).isEqualTo((byte) 0xD7); + assertThat(byteBuffer.get()).isEqualTo((byte) 0x00); + assertThat(byteBuffer.getInt()).isEqualTo((int) (now / 1000)); + assertThat(byteBuffer.getInt()).isEqualTo(999999999); + } - { - EventTime eventTime = EventTime.fromEpoch(0xFFEEDDCCL, 0xFEDCBA98L); - ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); - byte[] bytes = objectMapper.writeValueAsBytes(eventTime); - ByteBuffer byteBuffer = ByteBuffer.wrap(bytes); - assertThat(byteBuffer.get()).isEqualTo((byte) 0xD7); - assertThat(byteBuffer.get()).isEqualTo((byte) 0x00); + { + EventTime eventTime = EventTime.fromEpoch(0xFFEEDDCCL, 0xFEDCBA98L); + ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); + byte[] bytes = objectMapper.writeValueAsBytes(eventTime); + ByteBuffer byteBuffer = ByteBuffer.wrap(bytes); + assertThat(byteBuffer.get()).isEqualTo((byte) 0xD7); + assertThat(byteBuffer.get()).isEqualTo((byte) 0x00); - assertThat(byteBuffer.get()).isEqualTo((byte) 0xFF); - assertThat(byteBuffer.get()).isEqualTo((byte) 0xEE); - assertThat(byteBuffer.get()).isEqualTo((byte) 0xDD); - assertThat(byteBuffer.get()).isEqualTo((byte) 0xCC); + assertThat(byteBuffer.get()).isEqualTo((byte) 0xFF); + assertThat(byteBuffer.get()).isEqualTo((byte) 0xEE); + assertThat(byteBuffer.get()).isEqualTo((byte) 0xDD); + assertThat(byteBuffer.get()).isEqualTo((byte) 0xCC); - assertThat(byteBuffer.get()).isEqualTo((byte) 0xFE); - assertThat(byteBuffer.get()).isEqualTo((byte) 0xDC); - assertThat(byteBuffer.get()).isEqualTo((byte) 0xBA); - assertThat(byteBuffer.get()).isEqualTo((byte) 0x98); - } + assertThat(byteBuffer.get()).isEqualTo((byte) 0xFE); + assertThat(byteBuffer.get()).isEqualTo((byte) 0xDC); + assertThat(byteBuffer.get()).isEqualTo((byte) 0xBA); + assertThat(byteBuffer.get()).isEqualTo((byte) 0x98); } + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/FluencyBuilderForFluentdTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/FluencyBuilderForFluentdTest.java index dd461baa..08d009f5 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/FluencyBuilderForFluentdTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/FluencyBuilderForFluentdTest.java @@ -16,7 +16,18 @@ package org.komamitsu.fluency.fluentd; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.*; + import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.channels.SocketChannel; +import java.util.Arrays; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.SSLSocketFactory; import org.junit.jupiter.api.Test; import org.komamitsu.fluency.Fluency; import org.komamitsu.fluency.buffer.Buffer; @@ -29,382 +40,361 @@ import org.komamitsu.fluency.fluentd.recordformat.FluentdRecordFormatter; import org.komamitsu.fluency.flusher.Flusher; -import javax.net.ssl.SSLSocket; -import javax.net.ssl.SSLSocketFactory; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.nio.channels.SocketChannel; -import java.util.Arrays; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.Mockito.*; - -class FluencyBuilderForFluentdTest -{ - private void assertBuffer(Buffer buffer) - { - assertThat(buffer.getMaxBufferSize()).isEqualTo(512 * 1024 * 1024L); - assertThat(buffer.getFileBackupDir()).isNull(); - assertThat(buffer.bufferFormatType()).isEqualTo("packed_forward"); - assertThat(buffer.getChunkExpandRatio()).isEqualTo(2f); - assertThat(buffer.getChunkRetentionSize()).isEqualTo(4 * 1024 * 1024); - assertThat(buffer.getChunkInitialSize()).isEqualTo(1 * 1024 * 1024); - assertThat(buffer.getChunkRetentionTimeMillis()).isEqualTo(1000); - assertThat(buffer.getJvmHeapBufferMode()).isFalse(); - } - - private void assertFlusher(Flusher flusher) - { - assertThat(flusher.isTerminated()).isFalse(); - assertThat(flusher.getFlushAttemptIntervalMillis()).isEqualTo(600); - assertThat(flusher.getWaitUntilBufferFlushed()).isEqualTo(60); - assertThat(flusher.getWaitUntilTerminated()).isEqualTo(60); +class FluencyBuilderForFluentdTest { + private void assertBuffer(Buffer buffer) { + assertThat(buffer.getMaxBufferSize()).isEqualTo(512 * 1024 * 1024L); + assertThat(buffer.getFileBackupDir()).isNull(); + assertThat(buffer.bufferFormatType()).isEqualTo("packed_forward"); + assertThat(buffer.getChunkExpandRatio()).isEqualTo(2f); + assertThat(buffer.getChunkRetentionSize()).isEqualTo(4 * 1024 * 1024); + assertThat(buffer.getChunkInitialSize()).isEqualTo(1 * 1024 * 1024); + assertThat(buffer.getChunkRetentionTimeMillis()).isEqualTo(1000); + assertThat(buffer.getJvmHeapBufferMode()).isFalse(); + } + + private void assertFlusher(Flusher flusher) { + assertThat(flusher.isTerminated()).isFalse(); + assertThat(flusher.getFlushAttemptIntervalMillis()).isEqualTo(600); + assertThat(flusher.getWaitUntilBufferFlushed()).isEqualTo(60); + assertThat(flusher.getWaitUntilTerminated()).isEqualTo(60); + } + + private void assertDefaultRetryableSender( + RetryableSender sender, Class expectedBaseClass) { + assertThat(sender.getRetryStrategy()).isInstanceOf(ExponentialBackOffRetryStrategy.class); + ExponentialBackOffRetryStrategy retryStrategy = + (ExponentialBackOffRetryStrategy) sender.getRetryStrategy(); + assertThat(retryStrategy.getMaxRetryCount()).isEqualTo(7); + assertThat(retryStrategy.getBaseIntervalMillis()).isEqualTo(400); + assertThat(sender.getBaseSender()).isInstanceOf(expectedBaseClass); + } + + private void assertDefaultFluentdSender( + FluentdSender sender, + String expectedHost, + int expectedPort, + Class expectedBaseClass) { + assertThat(sender).isInstanceOf(RetryableSender.class); + RetryableSender retryableSender = (RetryableSender) sender; + assertDefaultRetryableSender(retryableSender, expectedBaseClass); + + assertThat(retryableSender.getBaseSender()).isInstanceOf(InetSocketSender.class); + InetSocketSender networkSender = + (InetSocketSender) retryableSender.getBaseSender(); + assertThat(networkSender.getHost()).isEqualTo(expectedHost); + assertThat(networkSender.getPort()).isEqualTo(expectedPort); + assertThat(networkSender.getConnectionTimeoutMilli()).isEqualTo(5000); + assertThat(networkSender.getReadTimeoutMilli()).isEqualTo(5000); + + FailureDetector failureDetector = networkSender.getFailureDetector(); + assertThat(failureDetector).isNull(); + } + + @Test + void build() throws IOException { + try (Fluency fluency = new FluencyBuilderForFluentd().build()) { + assertBuffer(fluency.getBuffer()); + assertFlusher(fluency.getFlusher()); + assertDefaultFluentdSender( + (FluentdSender) fluency.getFlusher().getIngester().getSender(), + "127.0.0.1", + 24224, + TCPSender.class); } - - private void assertDefaultRetryableSender(RetryableSender sender, Class expectedBaseClass) - { - assertThat(sender.getRetryStrategy()).isInstanceOf(ExponentialBackOffRetryStrategy.class); - ExponentialBackOffRetryStrategy retryStrategy = (ExponentialBackOffRetryStrategy) sender.getRetryStrategy(); - assertThat(retryStrategy.getMaxRetryCount()).isEqualTo(7); - assertThat(retryStrategy.getBaseIntervalMillis()).isEqualTo(400); - assertThat(sender.getBaseSender()).isInstanceOf(expectedBaseClass); + } + + @Test + void buildWithCustomPort() throws IOException { + try (Fluency fluency = new FluencyBuilderForFluentd().build(54321)) { + assertBuffer(fluency.getBuffer()); + assertFlusher(fluency.getFlusher()); + assertDefaultFluentdSender( + (FluentdSender) fluency.getFlusher().getIngester().getSender(), + "127.0.0.1", + 54321, + TCPSender.class); } - - private void assertDefaultFluentdSender(FluentdSender sender, String expectedHost, int expectedPort, Class expectedBaseClass) - { - assertThat(sender).isInstanceOf(RetryableSender.class); - RetryableSender retryableSender = (RetryableSender) sender; - assertDefaultRetryableSender(retryableSender, expectedBaseClass); - - assertThat(retryableSender.getBaseSender()).isInstanceOf(InetSocketSender.class); - InetSocketSender networkSender = (InetSocketSender) retryableSender.getBaseSender(); - assertThat(networkSender.getHost()).isEqualTo(expectedHost); - assertThat(networkSender.getPort()).isEqualTo(expectedPort); - assertThat(networkSender.getConnectionTimeoutMilli()).isEqualTo(5000); - assertThat(networkSender.getReadTimeoutMilli()).isEqualTo(5000); - - FailureDetector failureDetector = networkSender.getFailureDetector(); - assertThat(failureDetector).isNull(); + } + + @Test + void buildWithCustomHostAndPort() throws IOException { + try (Fluency fluency = new FluencyBuilderForFluentd().build("192.168.0.99", 54321)) { + assertBuffer(fluency.getBuffer()); + assertFlusher(fluency.getFlusher()); + assertDefaultFluentdSender( + (FluentdSender) fluency.getFlusher().getIngester().getSender(), + "192.168.0.99", + 54321, + TCPSender.class); } - - @Test - void build() - throws IOException - { - try (Fluency fluency = new FluencyBuilderForFluentd().build()) { - assertBuffer(fluency.getBuffer()); - assertFlusher(fluency.getFlusher()); - assertDefaultFluentdSender( - (FluentdSender) fluency.getFlusher().getIngester().getSender(), - "127.0.0.1", - 24224, - TCPSender.class); - } + } + + @Test + void buildWithSsl() throws IOException { + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSslEnabled(true); + try (Fluency fluency = builder.build()) { + assertBuffer(fluency.getBuffer()); + assertFlusher(fluency.getFlusher()); + assertDefaultFluentdSender( + (FluentdSender) fluency.getFlusher().getIngester().getSender(), + "127.0.0.1", + 24224, + SSLSender.class); } - - @Test - void buildWithCustomPort() - throws IOException - { - try (Fluency fluency = new FluencyBuilderForFluentd().build(54321)) { - assertBuffer(fluency.getBuffer()); - assertFlusher(fluency.getFlusher()); - assertDefaultFluentdSender( - (FluentdSender) fluency.getFlusher().getIngester().getSender(), - "127.0.0.1", - 54321, - TCPSender.class); - } + } + + @Test + void buildWithSslAndCustomPort() throws IOException { + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSslEnabled(true); + try (Fluency fluency = builder.build(54321)) { + assertBuffer(fluency.getBuffer()); + assertFlusher(fluency.getFlusher()); + assertDefaultFluentdSender( + (FluentdSender) fluency.getFlusher().getIngester().getSender(), + "127.0.0.1", + 54321, + SSLSender.class); } - - @Test - void buildWithCustomHostAndPort() - throws IOException - { - try (Fluency fluency = new FluencyBuilderForFluentd().build("192.168.0.99", 54321)) { - assertBuffer(fluency.getBuffer()); - assertFlusher(fluency.getFlusher()); - assertDefaultFluentdSender( - (FluentdSender) fluency.getFlusher().getIngester().getSender(), - "192.168.0.99", - 54321, - TCPSender.class); - } + } + + @Test + void buildWithSslAndCustomHostAndPort() throws IOException { + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSslEnabled(true); + try (Fluency fluency = builder.build("192.168.0.99", 54321)) { + assertBuffer(fluency.getBuffer()); + assertFlusher(fluency.getFlusher()); + assertDefaultFluentdSender( + (FluentdSender) fluency.getFlusher().getIngester().getSender(), + "192.168.0.99", + 54321, + SSLSender.class); } - - @Test - void buildWithSsl() - throws IOException - { - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSslEnabled(true); - try (Fluency fluency = builder.build()) { - assertBuffer(fluency.getBuffer()); - assertFlusher(fluency.getFlusher()); - assertDefaultFluentdSender( - (FluentdSender) fluency.getFlusher().getIngester().getSender(), - "127.0.0.1", - 24224, - SSLSender.class); - } + } + + @Test + void buildWithSslSocketFactory() throws IOException { + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSslEnabled(true); + SSLSocketFactory sslSocketFactory = mock(SSLSocketFactory.class); + SSLSocket socket = mock(SSLSocket.class); + RuntimeException expectedException = new RuntimeException("Expected"); + doThrow(expectedException).when(socket).connect(any(InetSocketAddress.class), anyInt()); + doNothing().when(socket).close(); + doReturn(socket).when(sslSocketFactory).createSocket(); + builder.setSslSocketFactory(sslSocketFactory); + builder.setSenderMaxRetryCount(0); + try (Fluency fluency = builder.build("192.168.0.99", 54321)) { + fluency.emit("mytag", ImmutableMap.of("hello", "world")); } - - @Test - void buildWithSslAndCustomPort() - throws IOException - { - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSslEnabled(true); - try (Fluency fluency = builder.build(54321)) { - assertBuffer(fluency.getBuffer()); - assertFlusher(fluency.getFlusher()); - assertDefaultFluentdSender( - (FluentdSender) fluency.getFlusher().getIngester().getSender(), - "127.0.0.1", - 54321, - SSLSender.class); + verify(socket).connect(any(InetSocketAddress.class), anyInt()); + } + + @Test + void buildWithComplexConfig() throws IOException { + String tmpdir = System.getProperty("java.io.tmpdir"); + assertThat(tmpdir).isNotNull(); + + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setFlushAttemptIntervalMillis(200); + builder.setMaxBufferSize(Long.MAX_VALUE); + builder.setBufferChunkInitialSize(7 * 1024 * 1024); + builder.setBufferChunkRetentionSize(13 * 1024 * 1024); + builder.setBufferChunkRetentionTimeMillis(19 * 1000); + builder.setJvmHeapBufferMode(true); + builder.setSenderMaxRetryCount(99); + builder.setSenderBaseRetryIntervalMillis(20); + builder.setSenderMaxRetryIntervalMillis(100000); + builder.setConnectionTimeoutMilli(12345); + builder.setReadTimeoutMilli(9876); + builder.setAckResponseMode(true); + builder.setWaitUntilBufferFlushed(42); + builder.setWaitUntilFlusherTerminated(24); + builder.setFileBackupDir(tmpdir); + + try (Fluency fluency = + builder.build( + Arrays.asList( + new InetSocketAddress("333.333.333.333", 11111), + new InetSocketAddress("444.444.444.444", 22222)))) { + + assertThat(fluency.getBuffer()).isInstanceOf(Buffer.class); + Buffer buffer = fluency.getBuffer(); + assertThat(buffer.getMaxBufferSize()).isEqualTo(Long.MAX_VALUE); + assertThat(buffer.getFileBackupDir()).isEqualTo(tmpdir); + assertThat(buffer.bufferFormatType()).isEqualTo("packed_forward"); + assertThat(buffer.getChunkRetentionTimeMillis()).isEqualTo(19 * 1000); + assertThat(buffer.getChunkExpandRatio()).isEqualTo(2f); + assertThat(buffer.getChunkInitialSize()).isEqualTo(7 * 1024 * 1024); + assertThat(buffer.getChunkRetentionSize()).isEqualTo(13 * 1024 * 1024); + assertThat(buffer.getJvmHeapBufferMode()).isEqualTo(true); + + Flusher flusher = fluency.getFlusher(); + assertThat(flusher.isTerminated()).isFalse(); + assertThat(flusher.getFlushAttemptIntervalMillis()).isEqualTo(200); + assertThat(flusher.getWaitUntilBufferFlushed()).isEqualTo(42); + assertThat(flusher.getWaitUntilTerminated()).isEqualTo(24); + + assertThat(flusher.getIngester().getSender()).isInstanceOf(RetryableSender.class); + RetryableSender retryableSender = (RetryableSender) flusher.getIngester().getSender(); + assertThat(retryableSender.getRetryStrategy()) + .isInstanceOf(ExponentialBackOffRetryStrategy.class); + ExponentialBackOffRetryStrategy retryStrategy = + (ExponentialBackOffRetryStrategy) retryableSender.getRetryStrategy(); + assertThat(retryStrategy.getMaxRetryCount()).isEqualTo(99); + assertThat(retryStrategy.getBaseIntervalMillis()).isEqualTo(20); + assertThat(retryStrategy.getMaxIntervalMillis()).isEqualTo(100000); + + assertThat(retryableSender.getBaseSender()).isInstanceOf(MultiSender.class); + MultiSender multiSender = (MultiSender) retryableSender.getBaseSender(); + assertThat(multiSender.getSenders().size()).isEqualTo(2); + + assertThat(multiSender.getSenders().get(0)).isInstanceOf(TCPSender.class); + { + TCPSender sender = (TCPSender) multiSender.getSenders().get(0); + assertThat(sender.getHost()).isEqualTo("333.333.333.333"); + assertThat(sender.getPort()).isEqualTo(11111); + assertThat(sender.getConnectionTimeoutMilli()).isEqualTo(12345); + assertThat(sender.getReadTimeoutMilli()).isEqualTo(9876); + + FailureDetector failureDetector = sender.getFailureDetector(); + assertThat(failureDetector.getFailureIntervalMillis()).isEqualTo(3 * 1000); + assertThat(failureDetector.getFailureDetectStrategy()) + .isInstanceOf(PhiAccrualFailureDetectStrategy.class); + assertThat(failureDetector.getHeartbeater()).isInstanceOf(TCPHeartbeater.class); + { + TCPHeartbeater hb = (TCPHeartbeater) failureDetector.getHeartbeater(); + assertThat(hb.getHost()).isEqualTo("333.333.333.333"); + assertThat(hb.getPort()).isEqualTo(11111); } - } - - @Test - void buildWithSslAndCustomHostAndPort() - throws IOException - { - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSslEnabled(true); - try (Fluency fluency = builder.build("192.168.0.99", 54321)) { - assertBuffer(fluency.getBuffer()); - assertFlusher(fluency.getFlusher()); - assertDefaultFluentdSender( - (FluentdSender) fluency.getFlusher().getIngester().getSender(), - "192.168.0.99", - 54321, - SSLSender.class); + assertThat(failureDetector.getHeartbeater().getIntervalMillis()).isEqualTo(1000); + } + + assertThat(multiSender.getSenders().get(1)).isInstanceOf(TCPSender.class); + { + TCPSender sender = (TCPSender) multiSender.getSenders().get(1); + assertThat(sender.getHost()).isEqualTo("444.444.444.444"); + assertThat(sender.getPort()).isEqualTo(22222); + assertThat(sender.getConnectionTimeoutMilli()).isEqualTo(12345); + assertThat(sender.getReadTimeoutMilli()).isEqualTo(9876); + + FailureDetector failureDetector = sender.getFailureDetector(); + assertThat(failureDetector.getFailureIntervalMillis()).isEqualTo(3 * 1000); + assertThat(failureDetector.getFailureDetectStrategy()) + .isInstanceOf(PhiAccrualFailureDetectStrategy.class); + assertThat(failureDetector.getHeartbeater()).isInstanceOf(TCPHeartbeater.class); + { + TCPHeartbeater hb = (TCPHeartbeater) failureDetector.getHeartbeater(); + assertThat(hb.getHost()).isEqualTo("444.444.444.444"); + assertThat(hb.getPort()).isEqualTo(22222); } + assertThat(failureDetector.getHeartbeater().getIntervalMillis()).isEqualTo(1000); + } } - - @Test - void buildWithSslSocketFactory() - throws IOException - { - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSslEnabled(true); - SSLSocketFactory sslSocketFactory = mock(SSLSocketFactory.class); - SSLSocket socket = mock(SSLSocket.class); - RuntimeException expectedException = new RuntimeException("Expected"); - doThrow(expectedException).when(socket).connect(any(InetSocketAddress.class), anyInt()); - doNothing().when(socket).close(); - doReturn(socket).when(sslSocketFactory).createSocket(); - builder.setSslSocketFactory(sslSocketFactory); - builder.setSenderMaxRetryCount(0); - try (Fluency fluency = builder.build("192.168.0.99", 54321)) { - fluency.emit("mytag", ImmutableMap.of("hello", "world")); - } - verify(socket).connect(any(InetSocketAddress.class), anyInt()); - } - - @Test - void buildWithComplexConfig() - throws IOException - { - String tmpdir = System.getProperty("java.io.tmpdir"); - assertThat(tmpdir).isNotNull(); - - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setFlushAttemptIntervalMillis(200); - builder.setMaxBufferSize(Long.MAX_VALUE); - builder.setBufferChunkInitialSize(7 * 1024 * 1024); - builder.setBufferChunkRetentionSize(13 * 1024 * 1024); - builder.setBufferChunkRetentionTimeMillis(19 * 1000); - builder.setJvmHeapBufferMode(true); - builder.setSenderMaxRetryCount(99); - builder.setSenderBaseRetryIntervalMillis(20); - builder.setSenderMaxRetryIntervalMillis(100000); - builder.setConnectionTimeoutMilli(12345); - builder.setReadTimeoutMilli(9876); - builder.setAckResponseMode(true); - builder.setWaitUntilBufferFlushed(42); - builder.setWaitUntilFlusherTerminated(24); - builder.setFileBackupDir(tmpdir); - - try (Fluency fluency = builder.build( - Arrays.asList( - new InetSocketAddress("333.333.333.333", 11111), - new InetSocketAddress("444.444.444.444", 22222)))) { - - assertThat(fluency.getBuffer()).isInstanceOf(Buffer.class); - Buffer buffer = fluency.getBuffer(); - assertThat(buffer.getMaxBufferSize()).isEqualTo(Long.MAX_VALUE); - assertThat(buffer.getFileBackupDir()).isEqualTo(tmpdir); - assertThat(buffer.bufferFormatType()).isEqualTo("packed_forward"); - assertThat(buffer.getChunkRetentionTimeMillis()).isEqualTo(19 * 1000); - assertThat(buffer.getChunkExpandRatio()).isEqualTo(2f); - assertThat(buffer.getChunkInitialSize()).isEqualTo(7 * 1024 * 1024); - assertThat(buffer.getChunkRetentionSize()).isEqualTo(13 * 1024 * 1024); - assertThat(buffer.getJvmHeapBufferMode()).isEqualTo(true); - - Flusher flusher = fluency.getFlusher(); - assertThat(flusher.isTerminated()).isFalse(); - assertThat(flusher.getFlushAttemptIntervalMillis()).isEqualTo(200); - assertThat(flusher.getWaitUntilBufferFlushed()).isEqualTo(42); - assertThat(flusher.getWaitUntilTerminated()).isEqualTo(24); - - assertThat(flusher.getIngester().getSender()).isInstanceOf(RetryableSender.class); - RetryableSender retryableSender = (RetryableSender) flusher.getIngester().getSender(); - assertThat(retryableSender.getRetryStrategy()).isInstanceOf(ExponentialBackOffRetryStrategy.class); - ExponentialBackOffRetryStrategy retryStrategy = (ExponentialBackOffRetryStrategy) retryableSender.getRetryStrategy(); - assertThat(retryStrategy.getMaxRetryCount()).isEqualTo(99); - assertThat(retryStrategy.getBaseIntervalMillis()).isEqualTo(20); - assertThat(retryStrategy.getMaxIntervalMillis()).isEqualTo(100000); - - assertThat(retryableSender.getBaseSender()).isInstanceOf(MultiSender.class); - MultiSender multiSender = (MultiSender) retryableSender.getBaseSender(); - assertThat(multiSender.getSenders().size()).isEqualTo(2); - - assertThat(multiSender.getSenders().get(0)).isInstanceOf(TCPSender.class); - { - TCPSender sender = (TCPSender) multiSender.getSenders().get(0); - assertThat(sender.getHost()).isEqualTo("333.333.333.333"); - assertThat(sender.getPort()).isEqualTo(11111); - assertThat(sender.getConnectionTimeoutMilli()).isEqualTo(12345); - assertThat(sender.getReadTimeoutMilli()).isEqualTo(9876); - - FailureDetector failureDetector = sender.getFailureDetector(); - assertThat(failureDetector.getFailureIntervalMillis()).isEqualTo(3 * 1000); - assertThat(failureDetector.getFailureDetectStrategy()).isInstanceOf(PhiAccrualFailureDetectStrategy.class); - assertThat(failureDetector.getHeartbeater()).isInstanceOf(TCPHeartbeater.class); - { - TCPHeartbeater hb = (TCPHeartbeater) failureDetector.getHeartbeater(); - assertThat(hb.getHost()).isEqualTo("333.333.333.333"); - assertThat(hb.getPort()).isEqualTo(11111); - } - assertThat(failureDetector.getHeartbeater().getIntervalMillis()).isEqualTo(1000); - } - - assertThat(multiSender.getSenders().get(1)).isInstanceOf(TCPSender.class); - { - TCPSender sender = (TCPSender) multiSender.getSenders().get(1); - assertThat(sender.getHost()).isEqualTo("444.444.444.444"); - assertThat(sender.getPort()).isEqualTo(22222); - assertThat(sender.getConnectionTimeoutMilli()).isEqualTo(12345); - assertThat(sender.getReadTimeoutMilli()).isEqualTo(9876); - - - FailureDetector failureDetector = sender.getFailureDetector(); - assertThat(failureDetector.getFailureIntervalMillis()).isEqualTo(3 * 1000); - assertThat(failureDetector.getFailureDetectStrategy()).isInstanceOf(PhiAccrualFailureDetectStrategy.class); - assertThat(failureDetector.getHeartbeater()).isInstanceOf(TCPHeartbeater.class); - { - TCPHeartbeater hb = (TCPHeartbeater) failureDetector.getHeartbeater(); - assertThat(hb.getHost()).isEqualTo("444.444.444.444"); - assertThat(hb.getPort()).isEqualTo(22222); - } - assertThat(failureDetector.getHeartbeater().getIntervalMillis()).isEqualTo(1000); - } + } + + @Test + void buildWithSslAndComplexConfig() throws IOException { + String tmpdir = System.getProperty("java.io.tmpdir"); + assertThat(tmpdir).isNotNull(); + + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSslEnabled(true); + builder.setFlushAttemptIntervalMillis(200); + builder.setMaxBufferSize(Long.MAX_VALUE); + builder.setBufferChunkInitialSize(7 * 1024 * 1024); + builder.setBufferChunkRetentionSize(13 * 1024 * 1024); + builder.setJvmHeapBufferMode(true); + builder.setSenderMaxRetryCount(99); + builder.setSenderBaseRetryIntervalMillis(20); + builder.setSenderMaxRetryIntervalMillis(100000); + builder.setConnectionTimeoutMilli(12345); + builder.setReadTimeoutMilli(9876); + builder.setAckResponseMode(true); + builder.setWaitUntilBufferFlushed(42); + builder.setWaitUntilFlusherTerminated(24); + builder.setFileBackupDir(tmpdir); + + try (Fluency fluency = + builder.build( + Arrays.asList( + new InetSocketAddress("333.333.333.333", 11111), + new InetSocketAddress("444.444.444.444", 22222)))) { + + assertThat(fluency.getFlusher().getIngester().getSender()) + .isInstanceOf(RetryableSender.class); + RetryableSender retryableSender = + (RetryableSender) fluency.getFlusher().getIngester().getSender(); + assertThat(retryableSender.getRetryStrategy()) + .isInstanceOf(ExponentialBackOffRetryStrategy.class); + ExponentialBackOffRetryStrategy retryStrategy = + (ExponentialBackOffRetryStrategy) retryableSender.getRetryStrategy(); + assertThat(retryStrategy.getMaxRetryCount()).isEqualTo(99); + assertThat(retryStrategy.getBaseIntervalMillis()).isEqualTo(20); + assertThat(retryStrategy.getMaxIntervalMillis()).isEqualTo(100000); + + assertThat(retryableSender.getBaseSender()).isInstanceOf(MultiSender.class); + MultiSender multiSender = (MultiSender) retryableSender.getBaseSender(); + assertThat(multiSender.getSenders().size()).isEqualTo(2); + + assertThat(multiSender.getSenders().get(0)).isInstanceOf(SSLSender.class); + { + SSLSender sender = (SSLSender) multiSender.getSenders().get(0); + assertThat(sender.getHost()).isEqualTo("333.333.333.333"); + assertThat(sender.getPort()).isEqualTo(11111); + assertThat(sender.getConnectionTimeoutMilli()).isEqualTo(12345); + assertThat(sender.getReadTimeoutMilli()).isEqualTo(9876); + + FailureDetector failureDetector = sender.getFailureDetector(); + assertThat(failureDetector.getFailureIntervalMillis()).isEqualTo(3 * 1000); + assertThat(failureDetector.getFailureDetectStrategy()) + .isInstanceOf(PhiAccrualFailureDetectStrategy.class); + assertThat(failureDetector.getHeartbeater()).isInstanceOf(SSLHeartbeater.class); + { + SSLHeartbeater hb = (SSLHeartbeater) failureDetector.getHeartbeater(); + assertThat(hb.getHost()).isEqualTo("333.333.333.333"); + assertThat(hb.getPort()).isEqualTo(11111); } - } - - @Test - void buildWithSslAndComplexConfig() - throws IOException - { - String tmpdir = System.getProperty("java.io.tmpdir"); - assertThat(tmpdir).isNotNull(); - - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSslEnabled(true); - builder.setFlushAttemptIntervalMillis(200); - builder.setMaxBufferSize(Long.MAX_VALUE); - builder.setBufferChunkInitialSize(7 * 1024 * 1024); - builder.setBufferChunkRetentionSize(13 * 1024 * 1024); - builder.setJvmHeapBufferMode(true); - builder.setSenderMaxRetryCount(99); - builder.setSenderBaseRetryIntervalMillis(20); - builder.setSenderMaxRetryIntervalMillis(100000); - builder.setConnectionTimeoutMilli(12345); - builder.setReadTimeoutMilli(9876); - builder.setAckResponseMode(true); - builder.setWaitUntilBufferFlushed(42); - builder.setWaitUntilFlusherTerminated(24); - builder.setFileBackupDir(tmpdir); - - try (Fluency fluency = builder.build( - Arrays.asList( - new InetSocketAddress("333.333.333.333", 11111), - new InetSocketAddress("444.444.444.444", 22222)))) { - - assertThat(fluency.getFlusher().getIngester().getSender()).isInstanceOf(RetryableSender.class); - RetryableSender retryableSender = (RetryableSender) fluency.getFlusher().getIngester().getSender(); - assertThat(retryableSender.getRetryStrategy()).isInstanceOf(ExponentialBackOffRetryStrategy.class); - ExponentialBackOffRetryStrategy retryStrategy = (ExponentialBackOffRetryStrategy) retryableSender.getRetryStrategy(); - assertThat(retryStrategy.getMaxRetryCount()).isEqualTo(99); - assertThat(retryStrategy.getBaseIntervalMillis()).isEqualTo(20); - assertThat(retryStrategy.getMaxIntervalMillis()).isEqualTo(100000); - - assertThat(retryableSender.getBaseSender()).isInstanceOf(MultiSender.class); - MultiSender multiSender = (MultiSender) retryableSender.getBaseSender(); - assertThat(multiSender.getSenders().size()).isEqualTo(2); - - assertThat(multiSender.getSenders().get(0)).isInstanceOf(SSLSender.class); - { - SSLSender sender = (SSLSender) multiSender.getSenders().get(0); - assertThat(sender.getHost()).isEqualTo("333.333.333.333"); - assertThat(sender.getPort()).isEqualTo(11111); - assertThat(sender.getConnectionTimeoutMilli()).isEqualTo(12345); - assertThat(sender.getReadTimeoutMilli()).isEqualTo(9876); - - FailureDetector failureDetector = sender.getFailureDetector(); - assertThat(failureDetector.getFailureIntervalMillis()).isEqualTo(3 * 1000); - assertThat(failureDetector.getFailureDetectStrategy()).isInstanceOf(PhiAccrualFailureDetectStrategy.class); - assertThat(failureDetector.getHeartbeater()).isInstanceOf(SSLHeartbeater.class); - { - SSLHeartbeater hb = (SSLHeartbeater) failureDetector.getHeartbeater(); - assertThat(hb.getHost()).isEqualTo("333.333.333.333"); - assertThat(hb.getPort()).isEqualTo(11111); - } - assertThat(failureDetector.getHeartbeater().getIntervalMillis()).isEqualTo(1000); - } - - assertThat(multiSender.getSenders().get(1)).isInstanceOf(SSLSender.class); - { - SSLSender sender = (SSLSender) multiSender.getSenders().get(1); - assertThat(sender.getHost()).isEqualTo("444.444.444.444"); - assertThat(sender.getPort()).isEqualTo(22222); - assertThat(sender.getConnectionTimeoutMilli()).isEqualTo(12345); - assertThat(sender.getReadTimeoutMilli()).isEqualTo(9876); - - FailureDetector failureDetector = sender.getFailureDetector(); - assertThat(failureDetector.getFailureIntervalMillis()).isEqualTo(3 * 1000); - assertThat(failureDetector.getFailureDetectStrategy()).isInstanceOf(PhiAccrualFailureDetectStrategy.class); - assertThat(failureDetector.getHeartbeater()).isInstanceOf(SSLHeartbeater.class); - { - SSLHeartbeater hb = (SSLHeartbeater) failureDetector.getHeartbeater(); - assertThat(hb.getHost()).isEqualTo("444.444.444.444"); - assertThat(hb.getPort()).isEqualTo(22222); - } - assertThat(failureDetector.getHeartbeater().getIntervalMillis()).isEqualTo(1000); - } + assertThat(failureDetector.getHeartbeater().getIntervalMillis()).isEqualTo(1000); + } + + assertThat(multiSender.getSenders().get(1)).isInstanceOf(SSLSender.class); + { + SSLSender sender = (SSLSender) multiSender.getSenders().get(1); + assertThat(sender.getHost()).isEqualTo("444.444.444.444"); + assertThat(sender.getPort()).isEqualTo(22222); + assertThat(sender.getConnectionTimeoutMilli()).isEqualTo(12345); + assertThat(sender.getReadTimeoutMilli()).isEqualTo(9876); + + FailureDetector failureDetector = sender.getFailureDetector(); + assertThat(failureDetector.getFailureIntervalMillis()).isEqualTo(3 * 1000); + assertThat(failureDetector.getFailureDetectStrategy()) + .isInstanceOf(PhiAccrualFailureDetectStrategy.class); + assertThat(failureDetector.getHeartbeater()).isInstanceOf(SSLHeartbeater.class); + { + SSLHeartbeater hb = (SSLHeartbeater) failureDetector.getHeartbeater(); + assertThat(hb.getHost()).isEqualTo("444.444.444.444"); + assertThat(hb.getPort()).isEqualTo(22222); } + assertThat(failureDetector.getHeartbeater().getIntervalMillis()).isEqualTo(1000); + } } - - @Test - void defaultRecordFormatter() - { - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - assertThat(builder.getRecordFormatter()).isInstanceOf(FluentdRecordFormatter.class); - } - - @Test - void customRecordFormatter() - { - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setRecordFormatter(new CustomFluentdRecordFormatter()); - assertThat(builder.getRecordFormatter()).isInstanceOf(CustomFluentdRecordFormatter.class); - } - - private static class CustomFluentdRecordFormatter extends FluentdRecordFormatter - { - } + } + + @Test + void defaultRecordFormatter() { + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + assertThat(builder.getRecordFormatter()).isInstanceOf(FluentdRecordFormatter.class); + } + + @Test + void customRecordFormatter() { + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setRecordFormatter(new CustomFluentdRecordFormatter()); + assertThat(builder.getRecordFormatter()).isInstanceOf(CustomFluentdRecordFormatter.class); + } + + private static class CustomFluentdRecordFormatter extends FluentdRecordFormatter {} } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/FluencyTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/FluencyTest.java index ec799abb..f5852dc7 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/FluencyTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/FluencyTest.java @@ -16,10 +16,28 @@ package org.komamitsu.fluency.fluentd; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.komamitsu.fluency.fluentd.SSLTestSocketFactories.SSL_CLIENT_SOCKET_FACTORY; +import static org.mockito.Mockito.mock; + import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.module.SimpleModule; import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -39,259 +57,250 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Stream; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.komamitsu.fluency.fluentd.SSLTestSocketFactories.SSL_CLIENT_SOCKET_FACTORY; -import static org.mockito.Mockito.mock; - -class FluencyTest -{ - private static final Logger LOG = LoggerFactory.getLogger(FluencyTest.class); - private static final StringValue KEY_OPTION_SIZE = ValueFactory.newString("size"); - private static final StringValue KEY_OPTION_CHUNK = ValueFactory.newString("chunk"); - private Ingester ingester; - - @BeforeEach - void setUp() - { - ingester = mock(Ingester.class); - } - - @Test - void testSenderErrorHandler() - throws IOException, InterruptedException - { - final CountDownLatch countDownLatch = new CountDownLatch(1); - final AtomicReference errorContainer = new AtomicReference<>(); - - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSenderMaxRetryCount(1); - builder.setErrorHandler(e -> { - errorContainer.set(e); - countDownLatch.countDown(); +class FluencyTest { + private static final Logger LOG = LoggerFactory.getLogger(FluencyTest.class); + private static final StringValue KEY_OPTION_SIZE = ValueFactory.newString("size"); + private static final StringValue KEY_OPTION_CHUNK = ValueFactory.newString("chunk"); + private Ingester ingester; + + @BeforeEach + void setUp() { + ingester = mock(Ingester.class); + } + + @Test + void testSenderErrorHandler() throws IOException, InterruptedException { + final CountDownLatch countDownLatch = new CountDownLatch(1); + final AtomicReference errorContainer = new AtomicReference<>(); + + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSenderMaxRetryCount(1); + builder.setErrorHandler( + e -> { + errorContainer.set(e); + countDownLatch.countDown(); }); - try (Fluency fluency = builder.build(Integer.MAX_VALUE)) { - HashMap event = new HashMap<>(); - event.put("name", "foo"); - fluency.emit("tag", event); - - if (!countDownLatch.await(10, TimeUnit.SECONDS)) { - throw new AssertionError("Timeout"); - } + try (Fluency fluency = builder.build(Integer.MAX_VALUE)) { + HashMap event = new HashMap<>(); + event.put("name", "foo"); + fluency.emit("tag", event); - assertThat(errorContainer.get()).isInstanceOf(RetryableSender.RetryOverException.class); - } - } + if (!countDownLatch.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("Timeout"); + } - static Stream sslFlagsProvider() - { - return Stream.of(false, true); + assertThat(errorContainer.get()).isInstanceOf(RetryableSender.RetryOverException.class); } - - @ParameterizedTest - @MethodSource("sslFlagsProvider") - void testWithoutAckResponse(final boolean sslEnabled) - throws Throwable - { - Exception exception = new ConfigurableTestServer(sslEnabled).run( + } + + static Stream sslFlagsProvider() { + return Stream.of(false, true); + } + + @ParameterizedTest + @MethodSource("sslFlagsProvider") + void testWithoutAckResponse(final boolean sslEnabled) throws Throwable { + Exception exception = + new ConfigurableTestServer(sslEnabled) + .run( clientSocket -> { - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(clientSocket.getInputStream()); - assertEquals(3, unpacker.unpackArrayHeader()); - assertEquals("foo.bar", unpacker.unpackString()); - ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue(); - Map map = unpacker.unpackValue().asMapValue().map(); - assertEquals(1, map.size()); - assertEquals(rawValue.asByteArray().length, map.get(KEY_OPTION_SIZE).asIntegerValue().asInt()); - unpacker.close(); + MessageUnpacker unpacker = + MessagePack.newDefaultUnpacker(clientSocket.getInputStream()); + assertEquals(3, unpacker.unpackArrayHeader()); + assertEquals("foo.bar", unpacker.unpackString()); + ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue(); + Map map = unpacker.unpackValue().asMapValue().map(); + assertEquals(1, map.size()); + assertEquals( + rawValue.asByteArray().length, + map.get(KEY_OPTION_SIZE).asIntegerValue().asInt()); + unpacker.close(); }, serverPort -> { - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSslEnabled(sslEnabled); - if (sslEnabled) { - builder.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - } - - try (Fluency fluency = builder.build(serverPort)) { - fluency.emit("foo.bar", new HashMap<>()); - } - }, 5000); - assertNull(exception); - } - - @ParameterizedTest - @MethodSource("sslFlagsProvider") - void testWithAckResponseButNotReceiveToken(final boolean sslEnabled) - throws Throwable - { - Exception exception = new ConfigurableTestServer(sslEnabled).run( + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSslEnabled(sslEnabled); + if (sslEnabled) { + builder.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + } + + try (Fluency fluency = builder.build(serverPort)) { + fluency.emit("foo.bar", new HashMap<>()); + } + }, + 5000); + assertNull(exception); + } + + @ParameterizedTest + @MethodSource("sslFlagsProvider") + void testWithAckResponseButNotReceiveToken(final boolean sslEnabled) throws Throwable { + Exception exception = + new ConfigurableTestServer(sslEnabled) + .run( clientSocket -> { - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(clientSocket.getInputStream()); - assertEquals(3, unpacker.unpackArrayHeader()); - assertEquals("foo.bar", unpacker.unpackString()); - ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue(); - Map map = unpacker.unpackValue().asMapValue().map(); - assertEquals(2, map.size()); - assertEquals(rawValue.asByteArray().length, map.get(KEY_OPTION_SIZE).asIntegerValue().asInt()); - assertNotNull(map.get(KEY_OPTION_CHUNK).asRawValue().asString()); - unpacker.close(); + MessageUnpacker unpacker = + MessagePack.newDefaultUnpacker(clientSocket.getInputStream()); + assertEquals(3, unpacker.unpackArrayHeader()); + assertEquals("foo.bar", unpacker.unpackString()); + ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue(); + Map map = unpacker.unpackValue().asMapValue().map(); + assertEquals(2, map.size()); + assertEquals( + rawValue.asByteArray().length, + map.get(KEY_OPTION_SIZE).asIntegerValue().asInt()); + assertNotNull(map.get(KEY_OPTION_CHUNK).asRawValue().asString()); + unpacker.close(); }, serverPort -> { - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSslEnabled(sslEnabled); - builder.setAckResponseMode(true); - if (sslEnabled) { - builder.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - } - - try (Fluency fluency = builder.build(serverPort)) { - fluency.emit("foo.bar", new HashMap<>()); - } - }, 5000); - assertEquals(exception.getClass(), TimeoutException.class); - } - - @ParameterizedTest - @MethodSource("sslFlagsProvider") - void testWithAckResponseButWrongReceiveToken(final boolean sslEnabled) - throws Throwable - { - Exception exception = new ConfigurableTestServer(sslEnabled).run( + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSslEnabled(sslEnabled); + builder.setAckResponseMode(true); + if (sslEnabled) { + builder.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + } + + try (Fluency fluency = builder.build(serverPort)) { + fluency.emit("foo.bar", new HashMap<>()); + } + }, + 5000); + assertEquals(exception.getClass(), TimeoutException.class); + } + + @ParameterizedTest + @MethodSource("sslFlagsProvider") + void testWithAckResponseButWrongReceiveToken(final boolean sslEnabled) throws Throwable { + Exception exception = + new ConfigurableTestServer(sslEnabled) + .run( clientSocket -> { - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(clientSocket.getInputStream()); - assertEquals(3, unpacker.unpackArrayHeader()); - assertEquals("foo.bar", unpacker.unpackString()); - ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue(); - Map map = unpacker.unpackValue().asMapValue().map(); - assertEquals(2, map.size()); - assertEquals(rawValue.asByteArray().length, map.get(KEY_OPTION_SIZE).asIntegerValue().asInt()); - assertNotNull(map.get(KEY_OPTION_CHUNK).asRawValue().asString()); - - MessagePacker packer = MessagePack.newDefaultPacker(clientSocket.getOutputStream()); - packer.packMapHeader(1) - .packString("ack").packString(UUID.randomUUID().toString()) - .close(); - - // Close the input stream after closing the output stream to avoid closing a socket too early - unpacker.close(); + MessageUnpacker unpacker = + MessagePack.newDefaultUnpacker(clientSocket.getInputStream()); + assertEquals(3, unpacker.unpackArrayHeader()); + assertEquals("foo.bar", unpacker.unpackString()); + ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue(); + Map map = unpacker.unpackValue().asMapValue().map(); + assertEquals(2, map.size()); + assertEquals( + rawValue.asByteArray().length, + map.get(KEY_OPTION_SIZE).asIntegerValue().asInt()); + assertNotNull(map.get(KEY_OPTION_CHUNK).asRawValue().asString()); + + MessagePacker packer = + MessagePack.newDefaultPacker(clientSocket.getOutputStream()); + packer + .packMapHeader(1) + .packString("ack") + .packString(UUID.randomUUID().toString()) + .close(); + + // Close the input stream after closing the output stream to avoid closing a + // socket too early + unpacker.close(); }, serverPort -> { - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSslEnabled(sslEnabled); - builder.setAckResponseMode(true); - if (sslEnabled) { - builder.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - } - - try (Fluency fluency = builder.build(serverPort)) { - fluency.emit("foo.bar", new HashMap<>()); - } - }, 5000); - assertEquals(exception.getClass(), TimeoutException.class); - } - - @ParameterizedTest - @MethodSource("sslFlagsProvider") - public void testWithAckResponseWithProperToken(final boolean sslEnabled) - throws Throwable - { - Exception exception = new ConfigurableTestServer(sslEnabled).run( + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSslEnabled(sslEnabled); + builder.setAckResponseMode(true); + if (sslEnabled) { + builder.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + } + + try (Fluency fluency = builder.build(serverPort)) { + fluency.emit("foo.bar", new HashMap<>()); + } + }, + 5000); + assertEquals(exception.getClass(), TimeoutException.class); + } + + @ParameterizedTest + @MethodSource("sslFlagsProvider") + public void testWithAckResponseWithProperToken(final boolean sslEnabled) throws Throwable { + Exception exception = + new ConfigurableTestServer(sslEnabled) + .run( clientSocket -> { - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(clientSocket.getInputStream()); - assertEquals(3, unpacker.unpackArrayHeader()); - assertEquals("foo.bar", unpacker.unpackString()); - ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue(); - Map map = unpacker.unpackValue().asMapValue().map(); - assertEquals(2, map.size()); - assertEquals(rawValue.asByteArray().length, map.get(KEY_OPTION_SIZE).asIntegerValue().asInt()); - String ackResponseToken = map.get(KEY_OPTION_CHUNK).asRawValue().asString(); - assertNotNull(ackResponseToken); - - MessagePacker packer = MessagePack.newDefaultPacker(clientSocket.getOutputStream()); - packer.packMapHeader(1) - .packString("ack").packString(ackResponseToken) - .close(); - - // Close the input stream after closing the output stream to avoid closing a socket too early - unpacker.close(); + MessageUnpacker unpacker = + MessagePack.newDefaultUnpacker(clientSocket.getInputStream()); + assertEquals(3, unpacker.unpackArrayHeader()); + assertEquals("foo.bar", unpacker.unpackString()); + ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue(); + Map map = unpacker.unpackValue().asMapValue().map(); + assertEquals(2, map.size()); + assertEquals( + rawValue.asByteArray().length, + map.get(KEY_OPTION_SIZE).asIntegerValue().asInt()); + String ackResponseToken = map.get(KEY_OPTION_CHUNK).asRawValue().asString(); + assertNotNull(ackResponseToken); + + MessagePacker packer = + MessagePack.newDefaultPacker(clientSocket.getOutputStream()); + packer.packMapHeader(1).packString("ack").packString(ackResponseToken).close(); + + // Close the input stream after closing the output stream to avoid closing a + // socket too early + unpacker.close(); }, serverPort -> { - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSslEnabled(sslEnabled); - builder.setAckResponseMode(true); - if (sslEnabled) { - builder.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - } - - try (Fluency fluency = builder.build(serverPort)) { - fluency.emit("foo.bar", new HashMap<>()); - } - }, 5000); - assertNull(exception); - } - - @ParameterizedTest - @MethodSource("sslFlagsProvider") - void testBufferWithJacksonModule() - throws IOException - { - AtomicBoolean serialized = new AtomicBoolean(); - SimpleModule simpleModule = new SimpleModule(); - simpleModule.addSerializer(Foo.class, new FooSerializer(serialized)); - - FluentdRecordFormatter.Config recordFormatterConfig = new FluentdRecordFormatter.Config(); - recordFormatterConfig.setJacksonModules(Collections.singletonList(simpleModule)); - - Fluency fluency = new FluencyBuilder() - .buildFromIngester(new FluentdRecordFormatter(recordFormatterConfig), ingester); - - Map event = new HashMap<>(); - Foo foo = new Foo(); - foo.s = "Hello"; - event.put("foo", foo); - fluency.emit("tag", event); - - assertThat(serialized.get()).isEqualTo(true); - } - - static class Foo - { - String s; + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSslEnabled(sslEnabled); + builder.setAckResponseMode(true); + if (sslEnabled) { + builder.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + } + + try (Fluency fluency = builder.build(serverPort)) { + fluency.emit("foo.bar", new HashMap<>()); + } + }, + 5000); + assertNull(exception); + } + + @ParameterizedTest + @MethodSource("sslFlagsProvider") + void testBufferWithJacksonModule() throws IOException { + AtomicBoolean serialized = new AtomicBoolean(); + SimpleModule simpleModule = new SimpleModule(); + simpleModule.addSerializer(Foo.class, new FooSerializer(serialized)); + + FluentdRecordFormatter.Config recordFormatterConfig = new FluentdRecordFormatter.Config(); + recordFormatterConfig.setJacksonModules(Collections.singletonList(simpleModule)); + + Fluency fluency = + new FluencyBuilder() + .buildFromIngester(new FluentdRecordFormatter(recordFormatterConfig), ingester); + + Map event = new HashMap<>(); + Foo foo = new Foo(); + foo.s = "Hello"; + event.put("foo", foo); + fluency.emit("tag", event); + + assertThat(serialized.get()).isEqualTo(true); + } + + static class Foo { + String s; + } + + public static class FooSerializer extends StdSerializer { + final AtomicBoolean serialized; + + FooSerializer(AtomicBoolean serialized) { + super(Foo.class); + this.serialized = serialized; } - public static class FooSerializer - extends StdSerializer - { - final AtomicBoolean serialized; - - FooSerializer(AtomicBoolean serialized) - { - super(Foo.class); - this.serialized = serialized; - } - - @Override - public void serialize(Foo value, JsonGenerator gen, SerializerProvider provider) - throws IOException - { - gen.writeStartObject(); - gen.writeStringField("s", "Foo:" + value.s); - gen.writeEndObject(); - serialized.set(true); - } + @Override + public void serialize(Foo value, JsonGenerator gen, SerializerProvider provider) + throws IOException { + gen.writeStartObject(); + gen.writeStringField("s", "Foo:" + value.s); + gen.writeEndObject(); + serialized.set(true); } + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/FluencyTestWithMockServer.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/FluencyTestWithMockServer.java index 59a1bae2..aa91970f 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/FluencyTestWithMockServer.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/FluencyTestWithMockServer.java @@ -16,8 +16,31 @@ package org.komamitsu.fluency.fluentd; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.komamitsu.fluency.fluentd.SSLTestSocketFactories.SSL_CLIENT_SOCKET_FACTORY; + import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.komamitsu.fluency.EventTime; @@ -37,699 +60,752 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.net.Socket; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Stream; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.komamitsu.fluency.fluentd.SSLTestSocketFactories.SSL_CLIENT_SOCKET_FACTORY; - -class FluencyTestWithMockServer -{ - private static final Logger LOG = LoggerFactory.getLogger(FluencyTestWithMockServer.class); - private static final int SMALL_BUF_SIZE = 4 * 1024 * 1024; - private static final String TMPDIR = System.getProperty("java.io.tmpdir"); - - private FailureDetector createFailureDetector(Heartbeater hb) - { - return new FailureDetector(new PhiAccrualFailureDetectStrategy(), hb); - } - - private FluentdSender getSingleTCPSender(int port) - { - TCPSender.Config config = new TCPSender.Config(); - config.setPort(port); - return new TCPSender(config); - } - - private FluentdSender getDoubleTCPSender(int firstPort, int secondPort) - { - TCPSender.Config config0 = new TCPSender.Config(); - config0.setPort(firstPort); - - TCPHeartbeater.Config hbConfig0 = new TCPHeartbeater.Config(); - hbConfig0.setPort(firstPort); - - TCPSender.Config config1 = new TCPSender.Config(); - config1.setPort(secondPort); - - TCPHeartbeater.Config hbConfig1 = new TCPHeartbeater.Config(); - hbConfig1.setPort(secondPort); - - return new MultiSender(new MultiSender.Config(), - ImmutableList.of( - new TCPSender(config0, - createFailureDetector(new TCPHeartbeater(hbConfig0))), - new TCPSender(config1, - createFailureDetector(new TCPHeartbeater(hbConfig1))))); - } - - private FluentdSender getSingleSSLSender(int port) - { - SSLSender.Config config = new SSLSender.Config(); - config.setPort(port); - config.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - return new SSLSender(config); - } - - private FluentdSender getDoubleSSLSender(int firstPort, int secondPort) - { - SSLSender.Config config0 = new SSLSender.Config(); - config0.setPort(firstPort); - config0.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - - TCPHeartbeater.Config hbConfig0 = new TCPHeartbeater.Config(); - hbConfig0.setPort(firstPort); - - SSLSender.Config config1 = new SSLSender.Config(); - config1.setPort(secondPort); - config1.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - - TCPHeartbeater.Config hbConfig1 = new TCPHeartbeater.Config(); - hbConfig1.setPort(secondPort); - - return new MultiSender(new MultiSender.Config(), - ImmutableList.of( - new SSLSender(config0, - createFailureDetector(new TCPHeartbeater(hbConfig0))), - new SSLSender(config1, - createFailureDetector(new TCPHeartbeater(hbConfig1))))); - } - - static Stream optionsProvider() - { - return Stream.of( - // TCP - new Options(false, false, false, false, false, false, false), // Normal - new Options(true, false, false, false, false, false, false), // Failover - new Options(false, true, false, true, false, false, false), // File backup + Ack response - new Options(false, false, true, false, false, false, false), // Close instead of flush - new Options(false, false, false, true, false, false, false), // Ack response - new Options(false, false, false, false, true, false, false), // Small buffer - new Options(false, false, false, false, false, false, true), // JVM heap - new Options(true, false, false, false, true, false, false), // Failover + Small buffer - new Options(false, false, true, false, true, false, false), // Close instead of flush + Small buffer - new Options(false, false, false, true, true, false, false), // Ack response + Small buffer - new Options(true, false, true, false, false, false, false), // Failover + Close instead of flush - new Options(false, true, true, true, false, false, false), // File backup + Ack response + Close instead of flush - new Options(false, false, true, true, false, false, false), // Ack response + Close instead of flush - new Options(false, false, false, false, false, false, false, EmitType.MAP_WITH_EVENT_TIME), // EmitType = MAP_WITH_EVENT_TIME - new Options(false, false, false, false, false, false, false, EmitType.MSGPACK_MAP_VALUE_BYTES), // EmitType = MSGPACK_MAP_VALUE_BYTES - new Options(false, false, false, false, false, false, false, EmitType.MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME), // EmitType = MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME - new Options(false, false, false, false, false, false, false, EmitType.MSGPACK_MAP_VALUE_BYTEBUFFER), // EmitType = MSGPACK_MAP_VALUE_BYTEBUFFER - new Options(false, false, false, false, false, false, false, EmitType.MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME), // EmitType = MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME - // SSL - new Options(false, false, false, false, false, true, false), // Normal - new Options(true, false, false, false, false, true, false), // Failover - new Options(false, true, false, true, false, true, false), // File backup + Ack response - new Options(false, false, true, false, false, true, false), // Close instead of flush - new Options(false, false, false, true, false, true, false), // Ack response - new Options(false, false, false, false, true, true, false), // Small buffer - new Options(false, false, false, false, false, true, true), // JVM heap - new Options(true, false, false, false, true, true, false), // Failover + Small buffer - new Options(false, false, true, false, true, true, false), // Close instead of flush + Small buffer - new Options(false, false, false, true, true, true, false), // Ack response + Small buffer - new Options(true, false, true, false, false, true, false), // Failover + Close instead of flush - new Options(false, true, true, true, false, true, false), // File backup + Ack response + Close instead of flush - new Options(false, false, true, true, false, true, false), // Ack response + Close instead of flush - new Options(false, false, false, false, false, true, false, EmitType.MAP_WITH_EVENT_TIME), // EmitType = MAP_WITH_EVENT_TIME - new Options(false, false, false, false, false, true, false, EmitType.MSGPACK_MAP_VALUE_BYTES), // EmitType = MSGPACK_MAP_VALUE_BYTES - new Options(false, false, false, false, false, true, false, EmitType.MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME), // EmitType = MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME - new Options(false, false, false, false, false, true, false, EmitType.MSGPACK_MAP_VALUE_BYTEBUFFER), // EmitType = MSGPACK_MAP_VALUE_BYTEBUFFER - new Options(false, false, false, false, false, true, false, EmitType.MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME) // EmitType = MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME +class FluencyTestWithMockServer { + private static final Logger LOG = LoggerFactory.getLogger(FluencyTestWithMockServer.class); + private static final int SMALL_BUF_SIZE = 4 * 1024 * 1024; + private static final String TMPDIR = System.getProperty("java.io.tmpdir"); + + private FailureDetector createFailureDetector(Heartbeater hb) { + return new FailureDetector(new PhiAccrualFailureDetectStrategy(), hb); + } + + private FluentdSender getSingleTCPSender(int port) { + TCPSender.Config config = new TCPSender.Config(); + config.setPort(port); + return new TCPSender(config); + } + + private FluentdSender getDoubleTCPSender(int firstPort, int secondPort) { + TCPSender.Config config0 = new TCPSender.Config(); + config0.setPort(firstPort); + + TCPHeartbeater.Config hbConfig0 = new TCPHeartbeater.Config(); + hbConfig0.setPort(firstPort); + + TCPSender.Config config1 = new TCPSender.Config(); + config1.setPort(secondPort); + + TCPHeartbeater.Config hbConfig1 = new TCPHeartbeater.Config(); + hbConfig1.setPort(secondPort); + + return new MultiSender( + new MultiSender.Config(), + ImmutableList.of( + new TCPSender(config0, createFailureDetector(new TCPHeartbeater(hbConfig0))), + new TCPSender(config1, createFailureDetector(new TCPHeartbeater(hbConfig1))))); + } + + private FluentdSender getSingleSSLSender(int port) { + SSLSender.Config config = new SSLSender.Config(); + config.setPort(port); + config.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + return new SSLSender(config); + } + + private FluentdSender getDoubleSSLSender(int firstPort, int secondPort) { + SSLSender.Config config0 = new SSLSender.Config(); + config0.setPort(firstPort); + config0.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + + TCPHeartbeater.Config hbConfig0 = new TCPHeartbeater.Config(); + hbConfig0.setPort(firstPort); + + SSLSender.Config config1 = new SSLSender.Config(); + config1.setPort(secondPort); + config1.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + + TCPHeartbeater.Config hbConfig1 = new TCPHeartbeater.Config(); + hbConfig1.setPort(secondPort); + + return new MultiSender( + new MultiSender.Config(), + ImmutableList.of( + new SSLSender(config0, createFailureDetector(new TCPHeartbeater(hbConfig0))), + new SSLSender(config1, createFailureDetector(new TCPHeartbeater(hbConfig1))))); + } + + static Stream optionsProvider() { + return Stream.of( + // TCP + new Options(false, false, false, false, false, false, false), // Normal + new Options(true, false, false, false, false, false, false), // Failover + new Options(false, true, false, true, false, false, false), // File backup + Ack response + new Options(false, false, true, false, false, false, false), // Close instead of flush + new Options(false, false, false, true, false, false, false), // Ack response + new Options(false, false, false, false, true, false, false), // Small buffer + new Options(false, false, false, false, false, false, true), // JVM heap + new Options(true, false, false, false, true, false, false), // Failover + Small buffer + new Options( + false, false, true, false, true, false, false), // Close instead of flush + Small buffer + new Options(false, false, false, true, true, false, false), // Ack response + Small buffer + new Options( + true, false, true, false, false, false, false), // Failover + Close instead of flush + new Options( + false, true, true, true, false, false, + false), // File backup + Ack response + Close instead of flush + new Options( + false, false, true, true, false, false, false), // Ack response + Close instead of flush + new Options( + false, + false, + false, + false, + false, + false, + false, + EmitType.MAP_WITH_EVENT_TIME), // EmitType = MAP_WITH_EVENT_TIME + new Options( + false, + false, + false, + false, + false, + false, + false, + EmitType.MSGPACK_MAP_VALUE_BYTES), // EmitType = MSGPACK_MAP_VALUE_BYTES + new Options( + false, + false, + false, + false, + false, + false, + false, + EmitType.MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME), // EmitType = + // MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME + new Options( + false, + false, + false, + false, + false, + false, + false, + EmitType.MSGPACK_MAP_VALUE_BYTEBUFFER), // EmitType = MSGPACK_MAP_VALUE_BYTEBUFFER + new Options( + false, + false, + false, + false, + false, + false, + false, + EmitType.MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME), // EmitType = + // MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME + // SSL + new Options(false, false, false, false, false, true, false), // Normal + new Options(true, false, false, false, false, true, false), // Failover + new Options(false, true, false, true, false, true, false), // File backup + Ack response + new Options(false, false, true, false, false, true, false), // Close instead of flush + new Options(false, false, false, true, false, true, false), // Ack response + new Options(false, false, false, false, true, true, false), // Small buffer + new Options(false, false, false, false, false, true, true), // JVM heap + new Options(true, false, false, false, true, true, false), // Failover + Small buffer + new Options( + false, false, true, false, true, true, false), // Close instead of flush + Small buffer + new Options(false, false, false, true, true, true, false), // Ack response + Small buffer + new Options( + true, false, true, false, false, true, false), // Failover + Close instead of flush + new Options( + false, true, true, true, false, true, + false), // File backup + Ack response + Close instead of flush + new Options( + false, false, true, true, false, true, false), // Ack response + Close instead of flush + new Options( + false, + false, + false, + false, + false, + true, + false, + EmitType.MAP_WITH_EVENT_TIME), // EmitType = MAP_WITH_EVENT_TIME + new Options( + false, + false, + false, + false, + false, + true, + false, + EmitType.MSGPACK_MAP_VALUE_BYTES), // EmitType = MSGPACK_MAP_VALUE_BYTES + new Options( + false, + false, + false, + false, + false, + true, + false, + EmitType.MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME), // EmitType = + // MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME + new Options( + false, + false, + false, + false, + false, + true, + false, + EmitType.MSGPACK_MAP_VALUE_BYTEBUFFER), // EmitType = MSGPACK_MAP_VALUE_BYTEBUFFER + new Options( + false, + false, + false, + false, + false, + true, + false, + EmitType.MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME) // EmitType = + // MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME ); - } - - @ParameterizedTest - @MethodSource("optionsProvider") - void testFluencyUsingAsyncFlusher(final Options options) - throws Exception - { - testFluencyBase(localPorts -> { - FluentdSender sender; - int fluentdPort = localPorts.get(0); - if (options.failover) { - int secondaryFluentdPort = localPorts.get(1); - if (options.sslEnabled) { - sender = getDoubleSSLSender(fluentdPort, secondaryFluentdPort); - } - else { - sender = getDoubleTCPSender(fluentdPort, secondaryFluentdPort); - } - } - else { - if (options.sslEnabled) { - sender = getSingleSSLSender(fluentdPort); - } - else { - sender = getSingleTCPSender(fluentdPort); - } - } - - FluentdIngester.Config ingesterConfig = new FluentdIngester.Config(); - if (options.ackResponse) { - ingesterConfig.setAckResponseMode(true); + } + + @ParameterizedTest + @MethodSource("optionsProvider") + void testFluencyUsingAsyncFlusher(final Options options) throws Exception { + testFluencyBase( + localPorts -> { + FluentdSender sender; + int fluentdPort = localPorts.get(0); + if (options.failover) { + int secondaryFluentdPort = localPorts.get(1); + if (options.sslEnabled) { + sender = getDoubleSSLSender(fluentdPort, secondaryFluentdPort); + } else { + sender = getDoubleTCPSender(fluentdPort, secondaryFluentdPort); } - - Buffer.Config bufferConfig = new Buffer.Config(); - if (options.smallBuffer) { - bufferConfig.setChunkRetentionSize(SMALL_BUF_SIZE); - bufferConfig.setMaxBufferSize(SMALL_BUF_SIZE + 1); - } - if (options.fileBackup) { - bufferConfig.setFileBackupDir(TMPDIR); - bufferConfig.setFileBackupPrefix("testFluencyUsingAsyncFlusher" + options.hashCode()); - } - if (options.jvmHeap) { - bufferConfig.setJvmHeapBufferMode(true); + } else { + if (options.sslEnabled) { + sender = getSingleSSLSender(fluentdPort); + } else { + sender = getSingleTCPSender(fluentdPort); } - - Flusher.Config flusherConfig = new Flusher.Config(); - flusherConfig.setWaitUntilBufferFlushed(10); - flusherConfig.setWaitUntilTerminated(10); - - Buffer buffer = new Buffer(bufferConfig, new FluentdRecordFormatter(new FluentdRecordFormatter.Config())); - Flusher flusher = new Flusher(flusherConfig, buffer, new FluentdIngester(ingesterConfig, sender)); - - return new Fluency(buffer, flusher); - }, options); + } + + FluentdIngester.Config ingesterConfig = new FluentdIngester.Config(); + if (options.ackResponse) { + ingesterConfig.setAckResponseMode(true); + } + + Buffer.Config bufferConfig = new Buffer.Config(); + if (options.smallBuffer) { + bufferConfig.setChunkRetentionSize(SMALL_BUF_SIZE); + bufferConfig.setMaxBufferSize(SMALL_BUF_SIZE + 1); + } + if (options.fileBackup) { + bufferConfig.setFileBackupDir(TMPDIR); + bufferConfig.setFileBackupPrefix("testFluencyUsingAsyncFlusher" + options.hashCode()); + } + if (options.jvmHeap) { + bufferConfig.setJvmHeapBufferMode(true); + } + + Flusher.Config flusherConfig = new Flusher.Config(); + flusherConfig.setWaitUntilBufferFlushed(10); + flusherConfig.setWaitUntilTerminated(10); + + Buffer buffer = + new Buffer( + bufferConfig, new FluentdRecordFormatter(new FluentdRecordFormatter.Config())); + Flusher flusher = + new Flusher(flusherConfig, buffer, new FluentdIngester(ingesterConfig, sender)); + + return new Fluency(buffer, flusher); + }, + options); + } + + private void testFluencyBase(final FluencyFactory fluencyFactory, final Options options) + throws Exception { + LOG.info("testFluencyBase starts: options={}", options); + + final ArrayList localPorts = new ArrayList<>(); + + final MockFluentdServer fluentd = new MockFluentdServer(options.sslEnabled); + fluentd.start(); + + final MockFluentdServer secondaryFluentd = new MockFluentdServer(options.sslEnabled, fluentd); + secondaryFluentd.start(); + + localPorts.add(fluentd.getLocalPort()); + localPorts.add(secondaryFluentd.getLocalPort()); + + final AtomicReference fluency = + new AtomicReference<>(fluencyFactory.generate(localPorts)); + if (options.fileBackup) { + fluency.get().clearBackupFiles(); } - private void testFluencyBase(final FluencyFactory fluencyFactory, final Options options) - throws Exception - { - LOG.info("testFluencyBase starts: options={}", options); - - final ArrayList localPorts = new ArrayList<>(); - - final MockFluentdServer fluentd = new MockFluentdServer(options.sslEnabled); - fluentd.start(); - - final MockFluentdServer secondaryFluentd = new MockFluentdServer(options.sslEnabled, fluentd); - secondaryFluentd.start(); - - localPorts.add(fluentd.getLocalPort()); - localPorts.add(secondaryFluentd.getLocalPort()); + final ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); - final AtomicReference fluency = new AtomicReference<>(fluencyFactory.generate(localPorts)); - if (options.fileBackup) { - fluency.get().clearBackupFiles(); - } - - final ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); + final int maxNameLen = 200; + final HashMap nameLenTable = new HashMap<>(maxNameLen); + for (int i = 1; i <= maxNameLen; i++) { + StringBuilder stringBuilder = new StringBuilder(); + for (int j = 0; j < i; j++) { + stringBuilder.append('x'); + } + nameLenTable.put(i, stringBuilder.toString()); + } - final int maxNameLen = 200; - final HashMap nameLenTable = new HashMap<>(maxNameLen); - for (int i = 1; i <= maxNameLen; i++) { - StringBuilder stringBuilder = new StringBuilder(); - for (int j = 0; j < i; j++) { - stringBuilder.append('x'); - } - nameLenTable.put(i, stringBuilder.toString()); - } + final AtomicLong ageEventsSum = new AtomicLong(); + final AtomicLong nameEventsLength = new AtomicLong(); + final AtomicLong tag0EventsCounter = new AtomicLong(); + final AtomicLong tag1EventsCounter = new AtomicLong(); + final AtomicLong tag2EventsCounter = new AtomicLong(); + final AtomicLong tag3EventsCounter = new AtomicLong(); + + try { + final Random random = new Random(); + int concurrency = 10; + final int reqNum = 6000; + long start = System.currentTimeMillis(); + final CountDownLatch latch = new CountDownLatch(concurrency); + final AtomicBoolean shouldFailOver = new AtomicBoolean(true); + + final AtomicBoolean shouldStopFluentd = new AtomicBoolean(true); + final AtomicBoolean shouldStopFluency = new AtomicBoolean(true); + final CountDownLatch fluentdCloseWaitLatch = new CountDownLatch(concurrency); + final CountDownLatch fluencyCloseWaitLatch = new CountDownLatch(concurrency); + + ExecutorService es = Executors.newCachedThreadPool(); + for (int i = 0; i < concurrency; i++) { + es.execute( + () -> { + for (int i1 = 0; i1 < reqNum; i1++) { + if (Thread.currentThread().isInterrupted()) { + LOG.info("Interrupted..."); + break; + } - final AtomicLong ageEventsSum = new AtomicLong(); - final AtomicLong nameEventsLength = new AtomicLong(); - final AtomicLong tag0EventsCounter = new AtomicLong(); - final AtomicLong tag1EventsCounter = new AtomicLong(); - final AtomicLong tag2EventsCounter = new AtomicLong(); - final AtomicLong tag3EventsCounter = new AtomicLong(); - - try { - final Random random = new Random(); - int concurrency = 10; - final int reqNum = 6000; - long start = System.currentTimeMillis(); - final CountDownLatch latch = new CountDownLatch(concurrency); - final AtomicBoolean shouldFailOver = new AtomicBoolean(true); - - final AtomicBoolean shouldStopFluentd = new AtomicBoolean(true); - final AtomicBoolean shouldStopFluency = new AtomicBoolean(true); - final CountDownLatch fluentdCloseWaitLatch = new CountDownLatch(concurrency); - final CountDownLatch fluencyCloseWaitLatch = new CountDownLatch(concurrency); - - ExecutorService es = Executors.newCachedThreadPool(); - for (int i = 0; i < concurrency; i++) { - es.execute(() -> { - for (int i1 = 0; i1 < reqNum; i1++) { - if (Thread.currentThread().isInterrupted()) { - LOG.info("Interrupted..."); - break; - } - - if (options.failover) { - if (i1 == reqNum / 2) { - if (shouldFailOver.getAndSet(false)) { - LOG.info("Failing over..."); - try { - secondaryFluentd.stop(); - } - catch (IOException e) { - LOG.warn("Failed to stop secondary fluentd", e); - } - } - } - } - else if (options.fileBackup) { - if (i1 == reqNum / 2) { - if (shouldStopFluentd.getAndSet(false)) { - LOG.info("Stopping Fluentd..."); - try { - fluentd.stop(); - secondaryFluentd.stop(); - } - catch (IOException e) { - LOG.warn("Failed to stop Fluentd", e); - } - } - - fluentdCloseWaitLatch.countDown(); - try { - assertTrue(fluentdCloseWaitLatch.await(20, TimeUnit.SECONDS)); - } - catch (InterruptedException e) { - LOG.warn("Interrupted", e); - } - - if (shouldStopFluency.getAndSet(false)) { - LOG.info("Stopping Fluency..."); - try { - fluency.get().close(); - TimeUnit.SECONDS.sleep(2); - } - catch (Exception e) { - LOG.warn("Failed to stop Fluency", e); - } - - LOG.info("Restarting Fluentd..."); - try { - fluentd.start(); - secondaryFluentd.start(); - LOG.info("Restarting Fluency..."); - fluency.set(fluencyFactory.generate(Arrays.asList(fluentd.getLocalPort(), secondaryFluentd.getLocalPort()))); - TimeUnit.SECONDS.sleep(2); - } - catch (Exception e) { - LOG.warn("Failed to restart Fluentd", e); - } - } - - fluencyCloseWaitLatch.countDown(); - try { - assertTrue(fluencyCloseWaitLatch.await(20, TimeUnit.SECONDS)); - } - catch (InterruptedException e) { - LOG.warn("Interrupted", e); - } - } - } - int tagNum = i1 % 4; - final String tag = String.format("foodb%d.bartbl%d", tagNum, tagNum); - switch (tagNum) { - case 0: - tag0EventsCounter.incrementAndGet(); - break; - case 1: - tag1EventsCounter.incrementAndGet(); - break; - case 2: - tag2EventsCounter.incrementAndGet(); - break; - case 3: - tag3EventsCounter.incrementAndGet(); - break; - default: - throw new RuntimeException("Never reach here"); - } - - int rand = random.nextInt(maxNameLen); - final Map hashMap = new HashMap(); - String name = nameLenTable.get(rand + 1); - nameEventsLength.addAndGet(name.length()); - hashMap.put("name", name); - rand = random.nextInt(100); - int age = rand; - ageEventsSum.addAndGet(age); - hashMap.put("age", age); - hashMap.put("comment", "hello, world"); - hashMap.put("rate", 1.23); - - try { - Exception exception = null; - for (int retry = 0; retry < 10; retry++) { - try { - switch (options.emitType) { - case MAP: { - fluency.get().emit(tag, hashMap); - break; - } - case MAP_WITH_EVENT_TIME: { - EventTime eventTime = EventTime.fromEpochMilli(System.currentTimeMillis()); - fluency.get().emit(tag, eventTime, hashMap); - break; - } - case MSGPACK_MAP_VALUE_BYTES: { - byte[] bytes = objectMapper.writeValueAsBytes(hashMap); - fluency.get().emit(tag, bytes, 0, bytes.length); - break; - } - case MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME: { - EventTime eventTime = EventTime.fromEpochMilli(System.currentTimeMillis()); - byte[] bytes = objectMapper.writeValueAsBytes(hashMap); - fluency.get().emit(tag, eventTime, bytes, 0, bytes.length); - break; - } - case MSGPACK_MAP_VALUE_BYTEBUFFER: { - byte[] bytes = objectMapper.writeValueAsBytes(hashMap); - fluency.get().emit(tag, ByteBuffer.wrap(bytes)); - break; - } - case MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME: { - EventTime eventTime = EventTime.fromEpochMilli(System.currentTimeMillis()); - byte[] bytes = objectMapper.writeValueAsBytes(hashMap); - fluency.get().emit(tag, eventTime, ByteBuffer.wrap(bytes)); - break; - } - } - exception = null; - break; - } - catch (Exception e) { - exception = e; - try { - TimeUnit.SECONDS.sleep(1); - } - catch (InterruptedException e1) { - } - } - } - if (exception != null) { - throw exception; - } - } - catch (Exception e) { - LOG.warn("Exception occurred", e); - } + if (options.failover) { + if (i1 == reqNum / 2) { + if (shouldFailOver.getAndSet(false)) { + LOG.info("Failing over..."); + try { + secondaryFluentd.stop(); + } catch (IOException e) { + LOG.warn("Failed to stop secondary fluentd", e); + } + } + } + } else if (options.fileBackup) { + if (i1 == reqNum / 2) { + if (shouldStopFluentd.getAndSet(false)) { + LOG.info("Stopping Fluentd..."); + try { + fluentd.stop(); + secondaryFluentd.stop(); + } catch (IOException e) { + LOG.warn("Failed to stop Fluentd", e); + } } - latch.countDown(); - }); - } - - if (!latch.await(60, TimeUnit.SECONDS)) { - fail("Sending all requests is timed out"); - } - - if (options.closeInsteadOfFlush) { - fluency.get().close(); - } - else { - fluency.get().flush(); - fluency.get().waitUntilAllBufferFlushed(20); - } - fluentd.waitUntilEventsStop(); - fluentd.stop(); + fluentdCloseWaitLatch.countDown(); + try { + assertTrue(fluentdCloseWaitLatch.await(20, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + LOG.warn("Interrupted", e); + } - secondaryFluentd.waitUntilEventsStop(); - secondaryFluentd.stop(); + if (shouldStopFluency.getAndSet(false)) { + LOG.info("Stopping Fluency..."); + try { + fluency.get().close(); + TimeUnit.SECONDS.sleep(2); + } catch (Exception e) { + LOG.warn("Failed to stop Fluency", e); + } + + LOG.info("Restarting Fluentd..."); + try { + fluentd.start(); + secondaryFluentd.start(); + LOG.info("Restarting Fluency..."); + fluency.set( + fluencyFactory.generate( + Arrays.asList( + fluentd.getLocalPort(), secondaryFluentd.getLocalPort()))); + TimeUnit.SECONDS.sleep(2); + } catch (Exception e) { + LOG.warn("Failed to restart Fluentd", e); + } + } - if (options.failover) { - assertThat(fluentd.connectCounter.get()).isGreaterThan(0L); - assertThat(fluentd.connectCounter.get()).isLessThanOrEqualTo(10L); - assertThat(fluentd.closeCounter.get()).isGreaterThan(0L); - assertThat(fluentd.closeCounter.get()).isLessThanOrEqualTo(10L); - } - else { - assertThat(fluentd.connectCounter.get()).isGreaterThan(0L); - assertThat(fluentd.connectCounter.get()).isLessThanOrEqualTo(2L); - if (options.closeInsteadOfFlush) { - assertThat(fluentd.closeCounter.get()).isGreaterThan(0L); + fluencyCloseWaitLatch.countDown(); + try { + assertTrue(fluencyCloseWaitLatch.await(20, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + LOG.warn("Interrupted", e); + } + } } - else { - assertThat(fluentd.closeCounter.get()).isEqualTo(0L); + int tagNum = i1 % 4; + final String tag = String.format("foodb%d.bartbl%d", tagNum, tagNum); + switch (tagNum) { + case 0: + tag0EventsCounter.incrementAndGet(); + break; + case 1: + tag1EventsCounter.incrementAndGet(); + break; + case 2: + tag2EventsCounter.incrementAndGet(); + break; + case 3: + tag3EventsCounter.incrementAndGet(); + break; + default: + throw new RuntimeException("Never reach here"); } - assertThat(fluentd.closeCounter.get()).isLessThanOrEqualTo(2L); - } - assertEquals((long) concurrency * reqNum, fluentd.ageEventsCounter.get()); - assertEquals(ageEventsSum.get(), fluentd.ageEventsSum.get()); - assertEquals((long) concurrency * reqNum, fluentd.nameEventsCounter.get()); - assertEquals(nameEventsLength.get(), fluentd.nameEventsLength.get()); - assertEquals(tag0EventsCounter.get(), fluentd.tag0EventsCounter.get()); - assertEquals(tag1EventsCounter.get(), fluentd.tag1EventsCounter.get()); - assertEquals(tag2EventsCounter.get(), fluentd.tag2EventsCounter.get()); - assertEquals(tag3EventsCounter.get(), fluentd.tag3EventsCounter.get()); - - System.out.println(System.currentTimeMillis() - start); - } - finally { - fluency.get().close(); - fluentd.stop(); - secondaryFluentd.stop(); + int rand = random.nextInt(maxNameLen); + final Map hashMap = new HashMap(); + String name = nameLenTable.get(rand + 1); + nameEventsLength.addAndGet(name.length()); + hashMap.put("name", name); + rand = random.nextInt(100); + int age = rand; + ageEventsSum.addAndGet(age); + hashMap.put("age", age); + hashMap.put("comment", "hello, world"); + hashMap.put("rate", 1.23); + + try { + Exception exception = null; + for (int retry = 0; retry < 10; retry++) { + try { + switch (options.emitType) { + case MAP: + { + fluency.get().emit(tag, hashMap); + break; + } + case MAP_WITH_EVENT_TIME: + { + EventTime eventTime = + EventTime.fromEpochMilli(System.currentTimeMillis()); + fluency.get().emit(tag, eventTime, hashMap); + break; + } + case MSGPACK_MAP_VALUE_BYTES: + { + byte[] bytes = objectMapper.writeValueAsBytes(hashMap); + fluency.get().emit(tag, bytes, 0, bytes.length); + break; + } + case MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME: + { + EventTime eventTime = + EventTime.fromEpochMilli(System.currentTimeMillis()); + byte[] bytes = objectMapper.writeValueAsBytes(hashMap); + fluency.get().emit(tag, eventTime, bytes, 0, bytes.length); + break; + } + case MSGPACK_MAP_VALUE_BYTEBUFFER: + { + byte[] bytes = objectMapper.writeValueAsBytes(hashMap); + fluency.get().emit(tag, ByteBuffer.wrap(bytes)); + break; + } + case MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME: + { + EventTime eventTime = + EventTime.fromEpochMilli(System.currentTimeMillis()); + byte[] bytes = objectMapper.writeValueAsBytes(hashMap); + fluency.get().emit(tag, eventTime, ByteBuffer.wrap(bytes)); + break; + } + } + exception = null; + break; + } catch (Exception e) { + exception = e; + try { + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException e1) { + } + } + } + if (exception != null) { + throw exception; + } + } catch (Exception e) { + LOG.warn("Exception occurred", e); + } + } + latch.countDown(); + }); + } + + if (!latch.await(60, TimeUnit.SECONDS)) { + fail("Sending all requests is timed out"); + } + + if (options.closeInsteadOfFlush) { + fluency.get().close(); + } else { + fluency.get().flush(); + fluency.get().waitUntilAllBufferFlushed(20); + } + + fluentd.waitUntilEventsStop(); + fluentd.stop(); + + secondaryFluentd.waitUntilEventsStop(); + secondaryFluentd.stop(); + + if (options.failover) { + assertThat(fluentd.connectCounter.get()).isGreaterThan(0L); + assertThat(fluentd.connectCounter.get()).isLessThanOrEqualTo(10L); + assertThat(fluentd.closeCounter.get()).isGreaterThan(0L); + assertThat(fluentd.closeCounter.get()).isLessThanOrEqualTo(10L); + } else { + assertThat(fluentd.connectCounter.get()).isGreaterThan(0L); + assertThat(fluentd.connectCounter.get()).isLessThanOrEqualTo(2L); + if (options.closeInsteadOfFlush) { + assertThat(fluentd.closeCounter.get()).isGreaterThan(0L); + } else { + assertThat(fluentd.closeCounter.get()).isEqualTo(0L); } + assertThat(fluentd.closeCounter.get()).isLessThanOrEqualTo(2L); + } + + assertEquals((long) concurrency * reqNum, fluentd.ageEventsCounter.get()); + assertEquals(ageEventsSum.get(), fluentd.ageEventsSum.get()); + assertEquals((long) concurrency * reqNum, fluentd.nameEventsCounter.get()); + assertEquals(nameEventsLength.get(), fluentd.nameEventsLength.get()); + assertEquals(tag0EventsCounter.get(), fluentd.tag0EventsCounter.get()); + assertEquals(tag1EventsCounter.get(), fluentd.tag1EventsCounter.get()); + assertEquals(tag2EventsCounter.get(), fluentd.tag2EventsCounter.get()); + assertEquals(tag3EventsCounter.get(), fluentd.tag3EventsCounter.get()); + + System.out.println(System.currentTimeMillis() - start); + } finally { + fluency.get().close(); + fluentd.stop(); + secondaryFluentd.stop(); } - - private enum EmitType - { - MAP, MAP_WITH_EVENT_TIME, - MSGPACK_MAP_VALUE_BYTES, MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME, - MSGPACK_MAP_VALUE_BYTEBUFFER, MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME, + } + + private enum EmitType { + MAP, + MAP_WITH_EVENT_TIME, + MSGPACK_MAP_VALUE_BYTES, + MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME, + MSGPACK_MAP_VALUE_BYTEBUFFER, + MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME, + } + + interface FluencyFactory { + Fluency generate(List localPort) throws IOException; + } + + public static class Options { + private final boolean failover; + private final boolean fileBackup; + private final boolean closeInsteadOfFlush; + private final boolean ackResponse; + private final boolean smallBuffer; + private final boolean sslEnabled; + private final boolean jvmHeap; + private final EmitType emitType; + + Options( + boolean failover, + boolean fileBackup, + boolean closeInsteadOfFlush, + boolean ackResponse, + boolean smallBuffer, + boolean sslEnabled, + boolean jvmHeap) { + this( + failover, + fileBackup, + closeInsteadOfFlush, + ackResponse, + smallBuffer, + sslEnabled, + jvmHeap, + EmitType.MAP); } - interface FluencyFactory - { - Fluency generate(List localPort) - throws IOException; + Options( + boolean failover, + boolean fileBackup, + boolean closeInsteadOfFlush, + boolean ackResponse, + boolean smallBuffer, + boolean sslEnabled, + boolean jvmHeap, + EmitType emitType) { + this.failover = failover; + this.fileBackup = fileBackup; + this.closeInsteadOfFlush = closeInsteadOfFlush; + this.ackResponse = ackResponse; + this.smallBuffer = smallBuffer; + this.sslEnabled = sslEnabled; + this.jvmHeap = jvmHeap; + this.emitType = emitType; } - public static class Options - { - private final boolean failover; - private final boolean fileBackup; - private final boolean closeInsteadOfFlush; - private final boolean ackResponse; - private final boolean smallBuffer; - private final boolean sslEnabled; - private final boolean jvmHeap; - private final EmitType emitType; - - Options( - boolean failover, - boolean fileBackup, - boolean closeInsteadOfFlush, - boolean ackResponse, - boolean smallBuffer, - boolean sslEnabled, - boolean jvmHeap) - { - this(failover, fileBackup, closeInsteadOfFlush, ackResponse, smallBuffer, sslEnabled, jvmHeap, EmitType.MAP); - } - - Options( - boolean failover, - boolean fileBackup, - boolean closeInsteadOfFlush, - boolean ackResponse, - boolean smallBuffer, - boolean sslEnabled, - boolean jvmHeap, - EmitType emitType) - { - this.failover = failover; - this.fileBackup = fileBackup; - this.closeInsteadOfFlush = closeInsteadOfFlush; - this.ackResponse = ackResponse; - this.smallBuffer = smallBuffer; - this.sslEnabled = sslEnabled; - this.jvmHeap = jvmHeap; - this.emitType = emitType; - } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Options)) { + return false; + } + + Options options = (Options) o; + + if (failover != options.failover) { + return false; + } + if (fileBackup != options.fileBackup) { + return false; + } + if (closeInsteadOfFlush != options.closeInsteadOfFlush) { + return false; + } + if (ackResponse != options.ackResponse) { + return false; + } + if (smallBuffer != options.smallBuffer) { + return false; + } + if (sslEnabled != options.sslEnabled) { + return false; + } + return emitType == options.emitType; + } - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (!(o instanceof Options)) { - return false; - } + @Override + public int hashCode() { + int result = (failover ? 1 : 0); + result = 31 * result + (fileBackup ? 1 : 0); + result = 31 * result + (closeInsteadOfFlush ? 1 : 0); + result = 31 * result + (ackResponse ? 1 : 0); + result = 31 * result + (smallBuffer ? 1 : 0); + result = 31 * result + (sslEnabled ? 1 : 0); + result = 31 * result + (emitType != null ? emitType.hashCode() : 0); + return result; + } - Options options = (Options) o; + @Override + public String toString() { + return "Options{" + + "failover=" + + failover + + ", fileBackup=" + + fileBackup + + ", closeInsteadOfFlush=" + + closeInsteadOfFlush + + ", ackResponse=" + + ackResponse + + ", smallBuffer=" + + smallBuffer + + ", sslEnabled=" + + sslEnabled + + ", emitType=" + + emitType + + '}'; + } + } + + private static class MockFluentdServer extends AbstractFluentdServer { + private final AtomicLong connectCounter; + private final AtomicLong ageEventsCounter; + private final AtomicLong ageEventsSum; + private final AtomicLong nameEventsCounter; + private final AtomicLong nameEventsLength; + private final AtomicLong tag0EventsCounter; + private final AtomicLong tag1EventsCounter; + private final AtomicLong tag2EventsCounter; + private final AtomicLong tag3EventsCounter; + private final AtomicLong closeCounter; + private final long startTimestampMillis; + + MockFluentdServer(boolean useSsl) throws Exception { + super(useSsl); + connectCounter = new AtomicLong(); + ageEventsCounter = new AtomicLong(); + ageEventsSum = new AtomicLong(); + nameEventsCounter = new AtomicLong(); + nameEventsLength = new AtomicLong(); + tag0EventsCounter = new AtomicLong(); + tag1EventsCounter = new AtomicLong(); + tag2EventsCounter = new AtomicLong(); + tag3EventsCounter = new AtomicLong(); + closeCounter = new AtomicLong(); + startTimestampMillis = System.currentTimeMillis(); + } - if (failover != options.failover) { - return false; - } - if (fileBackup != options.fileBackup) { - return false; - } - if (closeInsteadOfFlush != options.closeInsteadOfFlush) { - return false; - } - if (ackResponse != options.ackResponse) { - return false; - } - if (smallBuffer != options.smallBuffer) { - return false; - } - if (sslEnabled != options.sslEnabled) { - return false; - } - return emitType == options.emitType; - } + MockFluentdServer(boolean useSsl, MockFluentdServer base) throws Exception { + super(useSsl); + connectCounter = base.connectCounter; + ageEventsCounter = base.ageEventsCounter; + ageEventsSum = base.ageEventsSum; + nameEventsCounter = base.nameEventsCounter; + nameEventsLength = base.nameEventsLength; + tag0EventsCounter = base.tag0EventsCounter; + tag1EventsCounter = base.tag1EventsCounter; + tag2EventsCounter = base.tag2EventsCounter; + tag3EventsCounter = base.tag3EventsCounter; + closeCounter = base.closeCounter; + startTimestampMillis = System.currentTimeMillis(); + } + @Override + protected EventHandler getFluentdEventHandler() { + return new EventHandler() { @Override - public int hashCode() - { - int result = (failover ? 1 : 0); - result = 31 * result + (fileBackup ? 1 : 0); - result = 31 * result + (closeInsteadOfFlush ? 1 : 0); - result = 31 * result + (ackResponse ? 1 : 0); - result = 31 * result + (smallBuffer ? 1 : 0); - result = 31 * result + (sslEnabled ? 1 : 0); - result = 31 * result + (emitType != null ? emitType.hashCode() : 0); - return result; + public void onConnect(Socket acceptSocket) { + connectCounter.incrementAndGet(); } @Override - public String toString() - { - return "Options{" + - "failover=" + failover + - ", fileBackup=" + fileBackup + - ", closeInsteadOfFlush=" + closeInsteadOfFlush + - ", ackResponse=" + ackResponse + - ", smallBuffer=" + smallBuffer + - ", sslEnabled=" + sslEnabled + - ", emitType=" + emitType + - '}'; - } - } - - private static class MockFluentdServer - extends AbstractFluentdServer - { - private final AtomicLong connectCounter; - private final AtomicLong ageEventsCounter; - private final AtomicLong ageEventsSum; - private final AtomicLong nameEventsCounter; - private final AtomicLong nameEventsLength; - private final AtomicLong tag0EventsCounter; - private final AtomicLong tag1EventsCounter; - private final AtomicLong tag2EventsCounter; - private final AtomicLong tag3EventsCounter; - private final AtomicLong closeCounter; - private final long startTimestampMillis; - - MockFluentdServer(boolean useSsl) - throws Exception - { - super(useSsl); - connectCounter = new AtomicLong(); - ageEventsCounter = new AtomicLong(); - ageEventsSum = new AtomicLong(); - nameEventsCounter = new AtomicLong(); - nameEventsLength = new AtomicLong(); - tag0EventsCounter = new AtomicLong(); - tag1EventsCounter = new AtomicLong(); - tag2EventsCounter = new AtomicLong(); - tag3EventsCounter = new AtomicLong(); - closeCounter = new AtomicLong(); - startTimestampMillis = System.currentTimeMillis(); - } - - MockFluentdServer(boolean useSsl, MockFluentdServer base) - throws Exception - { - super(useSsl); - connectCounter = base.connectCounter; - ageEventsCounter = base.ageEventsCounter; - ageEventsSum = base.ageEventsSum; - nameEventsCounter = base.nameEventsCounter; - nameEventsLength = base.nameEventsLength; - tag0EventsCounter = base.tag0EventsCounter; - tag1EventsCounter = base.tag1EventsCounter; - tag2EventsCounter = base.tag2EventsCounter; - tag3EventsCounter = base.tag3EventsCounter; - closeCounter = base.closeCounter; - startTimestampMillis = System.currentTimeMillis(); + public void onReceive(String tag, long timestampMillis, MapValue data) { + if (tag.equals("foodb0.bartbl0")) { + tag0EventsCounter.incrementAndGet(); + } else if (tag.equals("foodb1.bartbl1")) { + tag1EventsCounter.incrementAndGet(); + } else if (tag.equals("foodb2.bartbl2")) { + tag2EventsCounter.incrementAndGet(); + } else if (tag.equals("foodb3.bartbl3")) { + tag3EventsCounter.incrementAndGet(); + } else { + throw new IllegalArgumentException("Unexpected tag: tag=" + tag); + } + assertTrue( + startTimestampMillis / 1000 <= timestampMillis / 1000 + && timestampMillis < startTimestampMillis + 60 * 1000); + + assertEquals(4, data.size()); + for (Map.Entry kv : data.entrySet()) { + String key = kv.getKey().asStringValue().toString(); + Value val = kv.getValue(); + if (key.equals("comment")) { + assertEquals("hello, world", val.toString()); + } else if (key.equals("rate")) { + assertEquals(1.23, val.asFloatValue().toFloat(), 0.000001); + } else if (key.equals("name")) { + nameEventsCounter.incrementAndGet(); + nameEventsLength.addAndGet(val.asRawValue().asString().length()); + } else if (key.equals("age")) { + ageEventsCounter.incrementAndGet(); + ageEventsSum.addAndGet(val.asIntegerValue().asInt()); + } + } } @Override - protected EventHandler getFluentdEventHandler() - { - return new EventHandler() - { - @Override - public void onConnect(Socket acceptSocket) - { - connectCounter.incrementAndGet(); - } - - @Override - public void onReceive(String tag, long timestampMillis, MapValue data) - { - if (tag.equals("foodb0.bartbl0")) { - tag0EventsCounter.incrementAndGet(); - } - else if (tag.equals("foodb1.bartbl1")) { - tag1EventsCounter.incrementAndGet(); - } - else if (tag.equals("foodb2.bartbl2")) { - tag2EventsCounter.incrementAndGet(); - } - else if (tag.equals("foodb3.bartbl3")) { - tag3EventsCounter.incrementAndGet(); - } - else { - throw new IllegalArgumentException("Unexpected tag: tag=" + tag); - } - assertTrue(startTimestampMillis / 1000 <= timestampMillis / 1000 && timestampMillis < startTimestampMillis + 60 * 1000); - - assertEquals(4, data.size()); - for (Map.Entry kv : data.entrySet()) { - String key = kv.getKey().asStringValue().toString(); - Value val = kv.getValue(); - if (key.equals("comment")) { - assertEquals("hello, world", val.toString()); - } - else if (key.equals("rate")) { - assertEquals(1.23, val.asFloatValue().toFloat(), 0.000001); - } - else if (key.equals("name")) { - nameEventsCounter.incrementAndGet(); - nameEventsLength.addAndGet(val.asRawValue().asString().length()); - } - else if (key.equals("age")) { - ageEventsCounter.incrementAndGet(); - ageEventsSum.addAndGet(val.asIntegerValue().asInt()); - } - } - } - - @Override - public void onClose(Socket accpetSocket) - { - closeCounter.incrementAndGet(); - } - }; + public void onClose(Socket accpetSocket) { + closeCounter.incrementAndGet(); } + }; } + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/MockTCPServer.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/MockTCPServer.java index f258808f..c8868608 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/MockTCPServer.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/MockTCPServer.java @@ -16,11 +16,6 @@ package org.komamitsu.fluency.fluentd; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.net.ssl.SSLHandshakeException; - import java.io.EOFException; import java.io.IOException; import java.net.ServerSocket; @@ -35,303 +30,283 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import javax.net.ssl.SSLHandshakeException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class MockTCPServer -{ - private static final Logger LOG = LoggerFactory.getLogger(MockTCPServer.class); - private final boolean sslEnabled; - private final AtomicLong lastEventTimeStampMilli = new AtomicLong(); - private final AtomicInteger threadSeqNum = new AtomicInteger(); - private ExecutorService executorService; - private ServerTask serverTask; - // TODO Make this class immutable - private List tasks = new ArrayList<>(); - - public MockTCPServer(boolean sslEnabled) - { - this.sslEnabled = sslEnabled; +public class MockTCPServer { + private static final Logger LOG = LoggerFactory.getLogger(MockTCPServer.class); + private final boolean sslEnabled; + private final AtomicLong lastEventTimeStampMilli = new AtomicLong(); + private final AtomicInteger threadSeqNum = new AtomicInteger(); + private ExecutorService executorService; + private ServerTask serverTask; + // TODO Make this class immutable + private List tasks = new ArrayList<>(); + + public MockTCPServer(boolean sslEnabled) { + this.sslEnabled = sslEnabled; + } + + protected EventHandler getEventHandler() { + return new EventHandler() { + @Override + public void onConnect(Socket acceptSocket) {} + + @Override + public void onReceive(Socket acceptSocket, int len, byte[] data) {} + + @Override + public void onClose(Socket acceptSocket) {} + }; + } + + public synchronized void start() throws Exception { + if (executorService == null) { + this.executorService = + Executors.newCachedThreadPool( + new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + return new Thread( + r, String.format("accepted-socket-worker-%d", threadSeqNum.getAndAdd(1))); + } + }); } - protected EventHandler getEventHandler() - { - return new EventHandler() - { - @Override - public void onConnect(Socket acceptSocket) - { - } - - @Override - public void onReceive(Socket acceptSocket, int len, byte[] data) - { - } + if (serverTask == null) { + serverTask = + new ServerTask( + executorService, + lastEventTimeStampMilli, + getEventHandler(), + sslEnabled ? SSLTestSocketFactories.createServerSocket() : new ServerSocket(), + tasks); + executorService.execute(serverTask); + tasks.add(serverTask); + } - @Override - public void onClose(Socket acceptSocket) - { - } - }; + for (int i = 0; i < 10; i++) { + int localPort = serverTask.getLocalPort(); + if (localPort > 0) { + return; + } + TimeUnit.MILLISECONDS.sleep(500); + } + throw new IllegalStateException("Local port open timeout"); + } + + public void waitUntilEventsStop() throws InterruptedException { + for (int i = 0; i < 20; i++) { + if (lastEventTimeStampMilli.get() + 2000 < System.currentTimeMillis()) { + return; + } + TimeUnit.MILLISECONDS.sleep(500); } + throw new IllegalStateException("Events didn't stop in the expected time"); + } - public synchronized void start() - throws Exception - { - if (executorService == null) { - this.executorService = Executors.newCachedThreadPool(new ThreadFactory() - { - @Override - public Thread newThread(Runnable r) - { - return new Thread(r, String.format("accepted-socket-worker-%d", threadSeqNum.getAndAdd(1))); - } - }); - } + @Override + public String toString() { + return "MockTCPServer{" + "serverTask=" + serverTask + ", sslEnabled=" + sslEnabled + '}'; + } - if (serverTask == null) { - serverTask = new ServerTask(executorService, lastEventTimeStampMilli, getEventHandler(), - sslEnabled ? SSLTestSocketFactories.createServerSocket() : new ServerSocket(), tasks); - executorService.execute(serverTask); - tasks.add(serverTask); - } + public int getLocalPort() { + return serverTask.getLocalPort(); + } - for (int i = 0; i < 10; i++) { - int localPort = serverTask.getLocalPort(); - if (localPort > 0) { - return; - } - TimeUnit.MILLISECONDS.sleep(500); - } - throw new IllegalStateException("Local port open timeout"); + public synchronized void stop() throws IOException { + stop(false); + } + + public synchronized void stop(boolean immediate) throws IOException { + if (executorService == null) { + return; + } + LOG.debug("Stopping MockTCPServer... {}", this); + executorService.shutdown(); + try { + if (!executorService.awaitTermination(1000, TimeUnit.MILLISECONDS)) { + LOG.debug("Shutting down MockTCPServer and child tasks... {}", this); + executorService.shutdownNow(); + } + } catch (InterruptedException e) { + LOG.warn("ExecutorService.shutdown() was failed: {}", this, e); + Thread.currentThread().interrupt(); } - public void waitUntilEventsStop() - throws InterruptedException - { - for (int i = 0; i < 20; i++) { - if (lastEventTimeStampMilli.get() + 2000 < System.currentTimeMillis()) { - return; - } - TimeUnit.MILLISECONDS.sleep(500); + if (immediate) { + LOG.debug("Closing related sockets {}", this); + for (Runnable runnable : tasks) { + if (runnable instanceof ServerTask) { + ((ServerTask) runnable).close(); + } else if (runnable instanceof ServerTask.AcceptTask) { + ((ServerTask.AcceptTask) runnable).close(); } - throw new IllegalStateException("Events didn't stop in the expected time"); + } } - @Override - public String toString() - { - return "MockTCPServer{" + - "serverTask=" + serverTask + - ", sslEnabled=" + sslEnabled + - '}'; + executorService = null; + serverTask = null; + } + + public interface EventHandler { + void onConnect(Socket acceptSocket); + + void onReceive(Socket acceptSocket, int len, byte[] data); + + void onClose(Socket acceptSocket); + } + + private static class ServerTask implements Runnable { + private final ServerSocket serverSocket; + private final ExecutorService serverExecutorService; + private final EventHandler eventHandler; + private final AtomicLong lastEventTimeStampMilli; + private final List tasks; + + private ServerTask( + ExecutorService executorService, + AtomicLong lastEventTimeStampMilli, + EventHandler eventHandler, + ServerSocket serverSocket, + List tasks) + throws IOException { + this.serverExecutorService = executorService; + this.lastEventTimeStampMilli = lastEventTimeStampMilli; + this.eventHandler = eventHandler; + this.serverSocket = serverSocket; + if (!serverSocket.isBound()) { + serverSocket.bind(null); + } + this.tasks = tasks; } - public int getLocalPort() - { - return serverTask.getLocalPort(); + public int getLocalPort() { + return serverSocket.getLocalPort(); } - public synchronized void stop() - throws IOException - { - stop(false); + @Override + public String toString() { + return "ServerTask{" + "serverSocket=" + serverSocket + '}'; } - public synchronized void stop(boolean immediate) - throws IOException - { - if (executorService == null) { - return; - } - LOG.debug("Stopping MockTCPServer... {}", this); - executorService.shutdown(); + @Override + public void run() { + while (!serverExecutorService.isShutdown()) { try { - if (!executorService.awaitTermination(1000, TimeUnit.MILLISECONDS)) { - LOG.debug("Shutting down MockTCPServer and child tasks... {}", this); - executorService.shutdownNow(); - } - } - catch (InterruptedException e) { - LOG.warn("ExecutorService.shutdown() was failed: {}", this, e); - Thread.currentThread().interrupt(); - } - - if (immediate) { - LOG.debug("Closing related sockets {}", this); - for (Runnable runnable : tasks) { - if (runnable instanceof ServerTask) { - ((ServerTask) runnable).close(); - } else if (runnable instanceof ServerTask.AcceptTask) { - ((ServerTask.AcceptTask) runnable).close(); - } - } + LOG.debug("ServerTask: accepting... this={}, local.port={}", this, getLocalPort()); + Socket acceptSocket = serverSocket.accept(); + LOG.debug( + "ServerTask: accepted. this={}, local.port={}, remote.port={}", + this, + getLocalPort(), + acceptSocket.getPort()); + AcceptTask acceptTask = + new AcceptTask( + serverExecutorService, lastEventTimeStampMilli, eventHandler, acceptSocket); + serverExecutorService.execute(acceptTask); + tasks.add(acceptTask); + } catch (RejectedExecutionException e) { + LOG.debug("ServerTask: ServerSocket.accept() failed[{}]: this={}", e.getMessage(), this); + } catch (ClosedByInterruptException e) { + LOG.debug("ServerTask: ServerSocket.accept() failed[{}]: this={}", e.getMessage(), this); + } catch (IOException e) { + LOG.warn("ServerTask: ServerSocket.accept() failed[{}]: this={}", e.getMessage(), this); } - - executorService = null; - serverTask = null; + } + try { + serverSocket.close(); + } catch (IOException e) { + LOG.warn("ServerTask: ServerSocketChannel.close() failed", e); + } + LOG.info("ServerTask: Finishing ServerTask...: this={}", this); } - public interface EventHandler - { - void onConnect(Socket acceptSocket); - - void onReceive(Socket acceptSocket, int len, byte[] data); - - void onClose(Socket acceptSocket); + private void close() throws IOException { + serverSocket.close(); } - private static class ServerTask - implements Runnable - { - private final ServerSocket serverSocket; - private final ExecutorService serverExecutorService; - private final EventHandler eventHandler; - private final AtomicLong lastEventTimeStampMilli; - private final List tasks; - - private ServerTask( - ExecutorService executorService, - AtomicLong lastEventTimeStampMilli, - EventHandler eventHandler, - ServerSocket serverSocket, - List tasks - ) - throws IOException - { - this.serverExecutorService = executorService; - this.lastEventTimeStampMilli = lastEventTimeStampMilli; - this.eventHandler = eventHandler; - this.serverSocket = serverSocket; - if (!serverSocket.isBound()) { - serverSocket.bind(null); - } - this.tasks = tasks; - } - - public int getLocalPort() - { - return serverSocket.getLocalPort(); - } - - @Override - public String toString() - { - return "ServerTask{" + - "serverSocket=" + serverSocket + - '}'; - } - - @Override - public void run() - { - while (!serverExecutorService.isShutdown()) { - try { - LOG.debug("ServerTask: accepting... this={}, local.port={}", this, getLocalPort()); - Socket acceptSocket = serverSocket.accept(); - LOG.debug("ServerTask: accepted. this={}, local.port={}, remote.port={}", this, getLocalPort(), acceptSocket.getPort()); - AcceptTask acceptTask = new AcceptTask(serverExecutorService, lastEventTimeStampMilli, eventHandler, acceptSocket); - serverExecutorService.execute(acceptTask); - tasks.add(acceptTask); - } - catch (RejectedExecutionException e) { - LOG.debug("ServerTask: ServerSocket.accept() failed[{}]: this={}", e.getMessage(), this); - } - catch (ClosedByInterruptException e) { - LOG.debug("ServerTask: ServerSocket.accept() failed[{}]: this={}", e.getMessage(), this); - } - catch (IOException e) { - LOG.warn("ServerTask: ServerSocket.accept() failed[{}]: this={}", e.getMessage(), this); - } - } + private static class AcceptTask implements Runnable { + private final Socket acceptSocket; + private final EventHandler eventHandler; + private final ExecutorService serverExecutorService; + private final AtomicLong lastEventTimeStampMilli; + + private AcceptTask( + ExecutorService serverExecutorService, + AtomicLong lastEventTimeStampMilli, + EventHandler eventHandler, + Socket acceptSocket) { + this.serverExecutorService = serverExecutorService; + this.lastEventTimeStampMilli = lastEventTimeStampMilli; + this.eventHandler = eventHandler; + this.acceptSocket = acceptSocket; + } + + private void close() throws IOException { + acceptSocket.close(); + } + + @Override + public void run() { + LOG.debug( + "AcceptTask: connected. this={}, local={}, remote={}", + this, + acceptSocket.getLocalPort(), + acceptSocket.getPort()); + try { + eventHandler.onConnect(acceptSocket); + byte[] byteBuf = new byte[512 * 1024]; + while (!serverExecutorService.isShutdown()) { try { - serverSocket.close(); - } - catch (IOException e) { - LOG.warn("ServerTask: ServerSocketChannel.close() failed", e); - } - LOG.info("ServerTask: Finishing ServerTask...: this={}", this); - } - - private void close() - throws IOException - { - serverSocket.close(); - } - - private static class AcceptTask - implements Runnable - { - private final Socket acceptSocket; - private final EventHandler eventHandler; - private final ExecutorService serverExecutorService; - private final AtomicLong lastEventTimeStampMilli; - - private AcceptTask(ExecutorService serverExecutorService, AtomicLong lastEventTimeStampMilli, EventHandler eventHandler, Socket acceptSocket) - { - this.serverExecutorService = serverExecutorService; - this.lastEventTimeStampMilli = lastEventTimeStampMilli; - this.eventHandler = eventHandler; - this.acceptSocket = acceptSocket; - } - - private void close() - throws IOException - { - acceptSocket.close(); - } - - @Override - public void run() - { - LOG.debug("AcceptTask: connected. this={}, local={}, remote={}", - this, acceptSocket.getLocalPort(), acceptSocket.getPort()); + int len = acceptSocket.getInputStream().read(byteBuf); + if (len <= 0) { + LOG.debug( + "AcceptTask: closed. this={}, local={}, remote={}", + this, + acceptSocket.getLocalPort(), + acceptSocket.getPort()); + eventHandler.onClose(acceptSocket); try { - eventHandler.onConnect(acceptSocket); - byte[] byteBuf = new byte[512 * 1024]; - while (!serverExecutorService.isShutdown()) { - try { - int len = acceptSocket.getInputStream().read(byteBuf); - if (len <= 0) { - LOG.debug("AcceptTask: closed. this={}, local={}, remote={}", - this, acceptSocket.getLocalPort(), acceptSocket.getPort()); - eventHandler.onClose(acceptSocket); - try { - acceptSocket.close(); - } - catch (IOException e) { - LOG.warn("AcceptTask: close() failed: this={}", this, e); - } - break; - } - else { - eventHandler.onReceive(acceptSocket, len, byteBuf); - lastEventTimeStampMilli.set(System.currentTimeMillis()); - } - } - catch (IOException e) { - LOG.warn("AcceptTask: recv() failed: this={}, message={}, cause={}", - this, e.getMessage(), e.getCause() == null ? "" : e.getCause().getMessage()); - if (e instanceof SSLHandshakeException && e.getCause() instanceof EOFException) { - eventHandler.onClose(acceptSocket); - break; - } - if (acceptSocket.isClosed()) { - eventHandler.onClose(acceptSocket); - throw new RuntimeException(e); - } - } - } - } - finally { - try { - LOG.debug("AcceptTask: Finished. Closing... this={}, local={}, remote={}", - this, acceptSocket.getLocalPort(), acceptSocket.getPort()); - acceptSocket.close(); - } - catch (IOException e) { - LOG.warn("AcceptTask: close() failed", e); - } + acceptSocket.close(); + } catch (IOException e) { + LOG.warn("AcceptTask: close() failed: this={}", this, e); } + break; + } else { + eventHandler.onReceive(acceptSocket, len, byteBuf); + lastEventTimeStampMilli.set(System.currentTimeMillis()); + } + } catch (IOException e) { + LOG.warn( + "AcceptTask: recv() failed: this={}, message={}, cause={}", + this, + e.getMessage(), + e.getCause() == null ? "" : e.getCause().getMessage()); + if (e instanceof SSLHandshakeException && e.getCause() instanceof EOFException) { + eventHandler.onClose(acceptSocket); + break; + } + if (acceptSocket.isClosed()) { + eventHandler.onClose(acceptSocket); + throw new RuntimeException(e); + } } + } + } finally { + try { + LOG.debug( + "AcceptTask: Finished. Closing... this={}, local={}, remote={}", + this, + acceptSocket.getLocalPort(), + acceptSocket.getPort()); + acceptSocket.close(); + } catch (IOException e) { + LOG.warn("AcceptTask: close() failed", e); + } } + } } + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/MockTCPServerWithMetrics.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/MockTCPServerWithMetrics.java index 5f7d9c20..b18d299f 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/MockTCPServerWithMetrics.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/MockTCPServerWithMetrics.java @@ -16,55 +16,48 @@ package org.komamitsu.fluency.fluentd; -import org.komamitsu.fluency.util.Tuple; - import java.net.Socket; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; +import org.komamitsu.fluency.util.Tuple; -public class MockTCPServerWithMetrics - extends MockTCPServer -{ - private final List> events = new CopyOnWriteArrayList>(); - private final EventHandler eventHandler = new EventHandler() - { +public class MockTCPServerWithMetrics extends MockTCPServer { + private final List> events = + new CopyOnWriteArrayList>(); + private final EventHandler eventHandler = + new EventHandler() { @Override - public void onConnect(Socket acceptSocket) - { - events.add(new Tuple(Type.CONNECT, null)); + public void onConnect(Socket acceptSocket) { + events.add(new Tuple(Type.CONNECT, null)); } @Override - public void onReceive(Socket acceptSocket, int len, byte[] data) - { - events.add(new Tuple(Type.RECEIVE, len)); + public void onReceive(Socket acceptSocket, int len, byte[] data) { + events.add(new Tuple(Type.RECEIVE, len)); } @Override - public void onClose(Socket acceptSocket) - { - events.add(new Tuple(Type.CLOSE, null)); + public void onClose(Socket acceptSocket) { + events.add(new Tuple(Type.CLOSE, null)); } - }; - - public MockTCPServerWithMetrics(boolean sslEnabled) - { - super(sslEnabled); - } - - @Override - protected EventHandler getEventHandler() - { - return eventHandler; - } - - public List> getEvents() - { - return events; - } - - public enum Type - { - CONNECT, RECEIVE, CLOSE; - } + }; + + public MockTCPServerWithMetrics(boolean sslEnabled) { + super(sslEnabled); + } + + @Override + protected EventHandler getEventHandler() { + return eventHandler; + } + + public List> getEvents() { + return events; + } + + public enum Type { + CONNECT, + RECEIVE, + CLOSE; + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/SSLTestSocketFactories.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/SSLTestSocketFactories.java index f341caf1..47c04d76 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/SSLTestSocketFactories.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/SSLTestSocketFactories.java @@ -16,70 +16,72 @@ package org.komamitsu.fluency.fluentd; -import org.komamitsu.fluency.fluentd.ingester.sender.SSLSocketBuilder; - -import javax.net.ssl.*; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.net.InetSocketAddress; import java.security.*; import java.security.cert.CertificateException; +import javax.net.ssl.*; +import org.komamitsu.fluency.fluentd.ingester.sender.SSLSocketBuilder; -public class SSLTestSocketFactories -{ - private static final String KEYSTORE_PASSWORD = "keypassword"; - private static final String KEY_PASSWORD = "keypassword"; - private static final String TRUSTSTORE_PASSWORD = "trustpassword"; +public class SSLTestSocketFactories { + private static final String KEYSTORE_PASSWORD = "keypassword"; + private static final String KEY_PASSWORD = "keypassword"; + private static final String TRUSTSTORE_PASSWORD = "trustpassword"; - public static final SSLSocketFactory SSL_CLIENT_SOCKET_FACTORY = createClientSocketFactory(); + public static final SSLSocketFactory SSL_CLIENT_SOCKET_FACTORY = createClientSocketFactory(); - private static SSLSocketFactory createClientSocketFactory() - { - String trustStorePath = SSLSocketBuilder.class.getClassLoader().getResource("truststore.jks").getFile(); + private static SSLSocketFactory createClientSocketFactory() { + String trustStorePath = + SSLSocketBuilder.class.getClassLoader().getResource("truststore.jks").getFile(); - try (InputStream keystoreStream = new FileInputStream(trustStorePath)) { - KeyStore keystore = KeyStore.getInstance("JKS"); + try (InputStream keystoreStream = new FileInputStream(trustStorePath)) { + KeyStore keystore = KeyStore.getInstance("JKS"); - keystore.load(keystoreStream, TRUSTSTORE_PASSWORD.toCharArray()); - TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + keystore.load(keystoreStream, TRUSTSTORE_PASSWORD.toCharArray()); + TrustManagerFactory trustManagerFactory = + TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); - trustManagerFactory.init(keystore); + trustManagerFactory.init(keystore); - SSLContext sslContext = SSLContext.getInstance("TLSv1.3"); - sslContext.init(null, trustManagerFactory.getTrustManagers(), new SecureRandom()); + SSLContext sslContext = SSLContext.getInstance("TLSv1.3"); + sslContext.init(null, trustManagerFactory.getTrustManagers(), new SecureRandom()); - return sslContext.getSocketFactory(); - } - catch (Throwable e) { - throw new RuntimeException("Failed to create SSLSocketFactory", e); - } + return sslContext.getSocketFactory(); + } catch (Throwable e) { + throw new RuntimeException("Failed to create SSLSocketFactory", e); } + } - public static SSLServerSocket createServerSocket() - throws IOException, CertificateException, NoSuchAlgorithmException, UnrecoverableKeyException, KeyStoreException, KeyManagementException - { - String trustStorePath = SSLSocketBuilder.class.getClassLoader().getResource("truststore.jks").getFile(); - System.getProperties().setProperty("javax.net.ssl.trustStore", trustStorePath); + public static SSLServerSocket createServerSocket() + throws IOException, CertificateException, NoSuchAlgorithmException, UnrecoverableKeyException, + KeyStoreException, KeyManagementException { + String trustStorePath = + SSLSocketBuilder.class.getClassLoader().getResource("truststore.jks").getFile(); + System.getProperties().setProperty("javax.net.ssl.trustStore", trustStorePath); - String keyStorePath = SSLSocketBuilder.class.getClassLoader().getResource("keystore.jks").getFile(); + String keyStorePath = + SSLSocketBuilder.class.getClassLoader().getResource("keystore.jks").getFile(); - try (InputStream keystoreStream = new FileInputStream(keyStorePath)) { - KeyStore keystore = KeyStore.getInstance("JKS"); + try (InputStream keystoreStream = new FileInputStream(keyStorePath)) { + KeyStore keystore = KeyStore.getInstance("JKS"); - keystore.load(keystoreStream, KEYSTORE_PASSWORD.toCharArray()); - KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keystore.load(keystoreStream, KEYSTORE_PASSWORD.toCharArray()); + KeyManagerFactory keyManagerFactory = + KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - keyManagerFactory.init(keystore, KEY_PASSWORD.toCharArray()); + keyManagerFactory.init(keystore, KEY_PASSWORD.toCharArray()); - SSLContext sslContext = SSLContext.getInstance("TLSv1.2"); - sslContext.init(keyManagerFactory.getKeyManagers(), null, new SecureRandom()); + SSLContext sslContext = SSLContext.getInstance("TLSv1.2"); + sslContext.init(keyManagerFactory.getKeyManagers(), null, new SecureRandom()); - SSLServerSocket serverSocket = (SSLServerSocket) sslContext.getServerSocketFactory().createServerSocket(); - serverSocket.setEnabledCipherSuites(serverSocket.getSupportedCipherSuites()); - serverSocket.bind(new InetSocketAddress(0)); + SSLServerSocket serverSocket = + (SSLServerSocket) sslContext.getServerSocketFactory().createServerSocket(); + serverSocket.setEnabledCipherSuites(serverSocket.getSupportedCipherSuites()); + serverSocket.bind(new InetSocketAddress(0)); - return serverSocket; - } + return serverSocket; } -} \ No newline at end of file + } +} diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/FluentdIngesterTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/FluentdIngesterTest.java index 7926ce56..a1cac1d2 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/FluentdIngesterTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/FluentdIngesterTest.java @@ -16,6 +16,18 @@ package org.komamitsu.fluency.fluentd.ingester; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.komamitsu.fluency.fluentd.ingester.sender.FluentdSender; @@ -29,110 +41,85 @@ import org.msgpack.value.Value; import org.msgpack.value.ValueFactory; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -class FluentdIngesterTest -{ - private static final String TAG = "foo.bar"; - private static final byte[] DATA = "hello, world".getBytes(StandardCharsets.UTF_8); - @Captor - public ArgumentCaptor> byteBuffersArgumentCaptor; - private FluentdSender fluentdSender; - - @BeforeEach - void setUp() - throws Exception - { - MockitoAnnotations.initMocks(this); - fluentdSender = mock(FluentdSender.class); - } - - private byte[] getIngestedData(List byteBuffers) - throws IOException - { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - for (ByteBuffer byteBuffer : byteBuffers) { - outputStream.write(byteBuffer.array()); - } - return outputStream.toByteArray(); - } - - @Test - void ingestWithoutAck() - throws IOException - { - Ingester ingester = new FluentdIngester(new FluentdIngester.Config(), fluentdSender); - ingester.ingest(TAG, ByteBuffer.wrap(DATA)); - - verify(fluentdSender, times(1)).send(byteBuffersArgumentCaptor.capture()); - List byteBuffers = byteBuffersArgumentCaptor.getAllValues().get(0); - byte[] ingested = getIngestedData(byteBuffers); - - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(ingested); - ImmutableArrayValue arrayValue = unpacker.unpackValue().asArrayValue(); - assertEquals(3, arrayValue.size()); - assertEquals(TAG, arrayValue.get(0).asStringValue().asString()); - assertArrayEquals(DATA, arrayValue.get(1).asRawValue().asByteArray()); - Map options = arrayValue.get(2).asMapValue().map(); - assertEquals(1, options.size()); - assertEquals(DATA.length, - options.get(ValueFactory.newString("size")).asIntegerValue().asInt()); - } - - @Test - void ingestWithAck() - throws IOException - { - FluentdIngester.Config config = new FluentdIngester.Config(); - config.setAckResponseMode(true); - Ingester ingester = new FluentdIngester(config, fluentdSender); - ingester.ingest(TAG, ByteBuffer.wrap(DATA)); - - ArgumentCaptor ackTokenArgumentCaptor = ArgumentCaptor.forClass(String.class); - verify(fluentdSender, times(1)) - .sendWithAck(byteBuffersArgumentCaptor.capture(), ackTokenArgumentCaptor.capture()); - List byteBuffers = byteBuffersArgumentCaptor.getAllValues().get(0); - byte[] ingested = getIngestedData(byteBuffers); - - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(ingested); - ImmutableArrayValue arrayValue = unpacker.unpackValue().asArrayValue(); - assertEquals(3, arrayValue.size()); - assertEquals(TAG, arrayValue.get(0).asStringValue().asString()); - assertArrayEquals(DATA, arrayValue.get(1).asRawValue().asByteArray()); - Map options = arrayValue.get(2).asMapValue().map(); - assertEquals(2, options.size()); - assertEquals(DATA.length, options.get(ValueFactory.newString("size")).asIntegerValue().asInt()); - String ackToken = options.get(ValueFactory.newString("chunk")).asRawValue().asString(); - - List ackTokenArgumentCaptorAllValues = ackTokenArgumentCaptor.getAllValues(); - assertEquals(1, ackTokenArgumentCaptorAllValues.size()); - assertEquals(ackToken, ackTokenArgumentCaptorAllValues.get(0)); - } - - @Test - void getSender() - { - assertEquals(fluentdSender, new FluentdIngester(new FluentdIngester.Config(), fluentdSender).getSender()); - } - - @Test - void close() - throws IOException - { - Ingester ingester = new FluentdIngester(new FluentdIngester.Config(), fluentdSender); - ingester.close(); - - verify(fluentdSender, times(1)).close(); +class FluentdIngesterTest { + private static final String TAG = "foo.bar"; + private static final byte[] DATA = "hello, world".getBytes(StandardCharsets.UTF_8); + @Captor public ArgumentCaptor> byteBuffersArgumentCaptor; + private FluentdSender fluentdSender; + + @BeforeEach + void setUp() throws Exception { + MockitoAnnotations.initMocks(this); + fluentdSender = mock(FluentdSender.class); + } + + private byte[] getIngestedData(List byteBuffers) throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + for (ByteBuffer byteBuffer : byteBuffers) { + outputStream.write(byteBuffer.array()); } + return outputStream.toByteArray(); + } + + @Test + void ingestWithoutAck() throws IOException { + Ingester ingester = new FluentdIngester(new FluentdIngester.Config(), fluentdSender); + ingester.ingest(TAG, ByteBuffer.wrap(DATA)); + + verify(fluentdSender, times(1)).send(byteBuffersArgumentCaptor.capture()); + List byteBuffers = byteBuffersArgumentCaptor.getAllValues().get(0); + byte[] ingested = getIngestedData(byteBuffers); + + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(ingested); + ImmutableArrayValue arrayValue = unpacker.unpackValue().asArrayValue(); + assertEquals(3, arrayValue.size()); + assertEquals(TAG, arrayValue.get(0).asStringValue().asString()); + assertArrayEquals(DATA, arrayValue.get(1).asRawValue().asByteArray()); + Map options = arrayValue.get(2).asMapValue().map(); + assertEquals(1, options.size()); + assertEquals(DATA.length, options.get(ValueFactory.newString("size")).asIntegerValue().asInt()); + } + + @Test + void ingestWithAck() throws IOException { + FluentdIngester.Config config = new FluentdIngester.Config(); + config.setAckResponseMode(true); + Ingester ingester = new FluentdIngester(config, fluentdSender); + ingester.ingest(TAG, ByteBuffer.wrap(DATA)); + + ArgumentCaptor ackTokenArgumentCaptor = ArgumentCaptor.forClass(String.class); + verify(fluentdSender, times(1)) + .sendWithAck(byteBuffersArgumentCaptor.capture(), ackTokenArgumentCaptor.capture()); + List byteBuffers = byteBuffersArgumentCaptor.getAllValues().get(0); + byte[] ingested = getIngestedData(byteBuffers); + + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(ingested); + ImmutableArrayValue arrayValue = unpacker.unpackValue().asArrayValue(); + assertEquals(3, arrayValue.size()); + assertEquals(TAG, arrayValue.get(0).asStringValue().asString()); + assertArrayEquals(DATA, arrayValue.get(1).asRawValue().asByteArray()); + Map options = arrayValue.get(2).asMapValue().map(); + assertEquals(2, options.size()); + assertEquals(DATA.length, options.get(ValueFactory.newString("size")).asIntegerValue().asInt()); + String ackToken = options.get(ValueFactory.newString("chunk")).asRawValue().asString(); + + List ackTokenArgumentCaptorAllValues = ackTokenArgumentCaptor.getAllValues(); + assertEquals(1, ackTokenArgumentCaptorAllValues.size()); + assertEquals(ackToken, ackTokenArgumentCaptorAllValues.get(0)); + } + + @Test + void getSender() { + assertEquals( + fluentdSender, + new FluentdIngester(new FluentdIngester.Config(), fluentdSender).getSender()); + } + + @Test + void close() throws IOException { + Ingester ingester = new FluentdIngester(new FluentdIngester.Config(), fluentdSender); + ingester.close(); + + verify(fluentdSender, times(1)).close(); + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/ConfigTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/ConfigTest.java index 3f152e86..57ae7657 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/ConfigTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/ConfigTest.java @@ -16,68 +16,54 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.jupiter.api.Test; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.fail; - -class ConfigTest -{ - @Test - void errorHandler() - throws IOException - { - final AtomicBoolean errorOccurred = new AtomicBoolean(); - FluentdSender.Config config = new FluentdSender.Config(); - config.setErrorHandler(e -> errorOccurred.set(true)); +class ConfigTest { + @Test + void errorHandler() throws IOException { + final AtomicBoolean errorOccurred = new AtomicBoolean(); + FluentdSender.Config config = new FluentdSender.Config(); + config.setErrorHandler(e -> errorOccurred.set(true)); - new DummySender(config, false).send(ByteBuffer.allocate(8)); - assertThat(errorOccurred.get()).isEqualTo(false); + new DummySender(config, false).send(ByteBuffer.allocate(8)); + assertThat(errorOccurred.get()).isEqualTo(false); - try { - new DummySender(config, true).send(ByteBuffer.allocate(8)); - fail(); - } - catch (Exception e) { - assertThat(errorOccurred.get()).isEqualTo(true); - } + try { + new DummySender(config, true).send(ByteBuffer.allocate(8)); + fail(); + } catch (Exception e) { + assertThat(errorOccurred.get()).isEqualTo(true); } + } - static class DummySender - extends FluentdSender - { - private final boolean shouldFail; - - protected DummySender(Config config, boolean shouldFail) - { - super(config); - this.shouldFail = shouldFail; - } + static class DummySender extends FluentdSender { + private final boolean shouldFail; - @Override - public boolean isAvailable() - { - return true; - } + protected DummySender(Config config, boolean shouldFail) { + super(config); + this.shouldFail = shouldFail; + } - @Override - protected void sendInternal(List buffers, String ackToken) - throws IOException - { - if (shouldFail) { - throw new RuntimeException("Unexpected scheduled error occurred"); - } - } + @Override + public boolean isAvailable() { + return true; + } - @Override - public void close() - throws IOException - { - } + @Override + protected void sendInternal(List buffers, String ackToken) throws IOException { + if (shouldFail) { + throw new RuntimeException("Unexpected scheduled error occurred"); + } } + + @Override + public void close() throws IOException {} + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/MockMultiTCPServerWithMetrics.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/MockMultiTCPServerWithMetrics.java index c250a610..573c6175 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/MockMultiTCPServerWithMetrics.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/MockMultiTCPServerWithMetrics.java @@ -16,9 +16,7 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.komamitsu.fluency.fluentd.MockTCPServerWithMetrics; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; import java.net.InetSocketAddress; @@ -29,77 +27,66 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import org.komamitsu.fluency.fluentd.MockTCPServerWithMetrics; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class MockMultiTCPServerWithMetrics - extends MockTCPServerWithMetrics -{ - private static final Logger LOG = LoggerFactory.getLogger(MockMultiTCPServerWithMetrics.class); - private ExecutorService executorService; - private DatagramChannel channel; +public class MockMultiTCPServerWithMetrics extends MockTCPServerWithMetrics { + private static final Logger LOG = LoggerFactory.getLogger(MockMultiTCPServerWithMetrics.class); + private ExecutorService executorService; + private DatagramChannel channel; - MockMultiTCPServerWithMetrics(boolean sslEnabled) - throws IOException - { - super(sslEnabled); - } + MockMultiTCPServerWithMetrics(boolean sslEnabled) throws IOException { + super(sslEnabled); + } - @Override - public synchronized void start() - throws Exception - { - super.start(); + @Override + public synchronized void start() throws Exception { + super.start(); - if (executorService != null) { - return; - } + if (executorService != null) { + return; + } - executorService = Executors.newSingleThreadExecutor(); - channel = DatagramChannel.open(); - channel.socket().bind(new InetSocketAddress(getLocalPort())); + executorService = Executors.newSingleThreadExecutor(); + channel = DatagramChannel.open(); + channel.socket().bind(new InetSocketAddress(getLocalPort())); - executorService.execute(new Runnable() - { - @Override - public void run() - { - while (executorService != null && !executorService.isTerminated()) { - try { - ByteBuffer buffer = ByteBuffer.allocate(8); - SocketAddress socketAddress = channel.receive(buffer); - assertEquals(0, buffer.position()); - channel.send(buffer, socketAddress); - } - catch (ClosedByInterruptException e) { - // Expected - } - catch (IOException e) { - LOG.warn("Failed to receive or send heartbeat", e); - } - } + executorService.execute( + new Runnable() { + @Override + public void run() { + while (executorService != null && !executorService.isTerminated()) { + try { + ByteBuffer buffer = ByteBuffer.allocate(8); + SocketAddress socketAddress = channel.receive(buffer); + assertEquals(0, buffer.position()); + channel.send(buffer, socketAddress); + } catch (ClosedByInterruptException e) { + // Expected + } catch (IOException e) { + LOG.warn("Failed to receive or send heartbeat", e); + } } + } }); - } + } - @Override - public synchronized void stop() - throws IOException - { - super.stop(); + @Override + public synchronized void stop() throws IOException { + super.stop(); - if (executorService == null) { - return; - } + if (executorService == null) { + return; + } - executorService.shutdown(); - try { - executorService.awaitTermination(2, TimeUnit.SECONDS); - } - catch (InterruptedException e) { - LOG.warn("stop(): Interrupted while shutting down", e); - } - executorService.shutdownNow(); - executorService = null; + executorService.shutdown(); + try { + executorService.awaitTermination(2, TimeUnit.SECONDS); + } catch (InterruptedException e) { + LOG.warn("stop(): Interrupted while shutting down", e); } + executorService.shutdownNow(); + executorService = null; + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/MockTCPSender.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/MockTCPSender.java index aa8dba68..a458c2e6 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/MockTCPSender.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/MockTCPSender.java @@ -16,101 +16,89 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class MockTCPSender - extends TCPSender -{ - private static final Logger LOG = LoggerFactory.getLogger(MockTCPSender.class); - - private final List events = new ArrayList(); - private final AtomicInteger closeCount = new AtomicInteger(); - - public MockTCPSender(TCPSender.Config config) - throws IOException - { - super(config); +public class MockTCPSender extends TCPSender { + private static final Logger LOG = LoggerFactory.getLogger(MockTCPSender.class); + + private final List events = new ArrayList(); + private final AtomicInteger closeCount = new AtomicInteger(); + + public MockTCPSender(TCPSender.Config config) throws IOException { + super(config); + } + + @Override + protected synchronized void sendInternal(List buffers, String ackToken) + throws IOException { + assertEquals(3, buffers.size()); + Event event = new Event(); + int i = 0; + for (ByteBuffer data : buffers) { + ByteBuffer byteBuffer = ByteBuffer.allocateDirect(data.capacity()); + byteBuffer.put(data); + byteBuffer.flip(); + switch (i) { + case 0: + event.header = byteBuffer; + break; + case 1: + event.data = byteBuffer; + break; + case 2: + event.option = byteBuffer; + break; + default: + throw new IllegalStateException("Shouldn't reach here"); + } + i++; } + events.add(event); + } - @Override - protected synchronized void sendInternal(List buffers, String ackToken) - throws IOException - { - assertEquals(3, buffers.size()); - Event event = new Event(); - int i = 0; - for (ByteBuffer data : buffers) { - ByteBuffer byteBuffer = ByteBuffer.allocateDirect(data.capacity()); - byteBuffer.put(data); - byteBuffer.flip(); - switch (i) { - case 0: - event.header = byteBuffer; - break; - case 1: - event.data = byteBuffer; - break; - case 2: - event.option = byteBuffer; - break; - default: - throw new IllegalStateException("Shouldn't reach here"); - } - i++; - } - events.add(event); - } + @Override + public synchronized void close() throws IOException { + closeCount.incrementAndGet(); + } - @Override - public synchronized void close() - throws IOException - { - closeCount.incrementAndGet(); - } + public List getEvents() { + return events; + } - public List getEvents() - { - return events; - } - - public AtomicInteger getCloseCount() - { - return closeCount; - } + public AtomicInteger getCloseCount() { + return closeCount; + } - public static class Event - { - ByteBuffer header; - ByteBuffer data; - ByteBuffer option; + public static class Event { + ByteBuffer header; + ByteBuffer data; + ByteBuffer option; - public byte[] getAllBytes() - { - byte[] bytes = new byte[header.limit() + data.limit() + option.limit()]; + public byte[] getAllBytes() { + byte[] bytes = new byte[header.limit() + data.limit() + option.limit()]; - int pos = 0; - int len = header.limit(); - header.get(bytes, pos, len); - pos += len; + int pos = 0; + int len = header.limit(); + header.get(bytes, pos, len); + pos += len; - len = data.limit(); - data.get(bytes, pos, len); - pos += len; + len = data.limit(); + data.get(bytes, pos, len); + pos += len; - len = option.limit(); - option.get(bytes, pos, len); - pos += len; + len = option.limit(); + option.get(bytes, pos, len); + pos += len; - return bytes; - } + return bytes; } + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/MultiSenderTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/MultiSenderTest.java index 3f5169c4..02a4a2a2 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/MultiSenderTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/MultiSenderTest.java @@ -16,16 +16,11 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import com.google.common.collect.ImmutableList; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.fluentd.MockTCPServerWithMetrics; -import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; -import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.PhiAccrualFailureDetectStrategy; -import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.*; -import org.komamitsu.fluency.util.Tuple; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.*; +import static org.komamitsu.fluency.fluentd.SSLTestSocketFactories.SSL_CLIENT_SOCKET_FACTORY; +import com.google.common.collect.ImmutableList; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.Charset; @@ -35,283 +30,284 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.fluentd.MockTCPServerWithMetrics; +import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; +import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.PhiAccrualFailureDetectStrategy; +import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.*; +import org.komamitsu.fluency.util.Tuple; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.*; -import static org.komamitsu.fluency.fluentd.SSLTestSocketFactories.SSL_CLIENT_SOCKET_FACTORY; - -class MultiSenderTest -{ - private static final Logger LOG = LoggerFactory.getLogger(MultiSenderTest.class); - - private FailureDetector createFailureDetector(Heartbeater hb) - { - return new FailureDetector(new PhiAccrualFailureDetectStrategy(), hb); - } - - @Test - void testConstructorForTCPSender() - throws IOException - { - MultiSender multiSender = null; - try { - TCPSender.Config senderConfig0 = new TCPSender.Config(); - senderConfig0.setPort(24225); - - TCPHeartbeater.Config hbConfig0 = new TCPHeartbeater.Config(); - hbConfig0.setPort(24225); - - TCPSender.Config senderConfig1 = new TCPSender.Config(); - senderConfig1.setHost("0.0.0.0"); - senderConfig1.setPort(24226); - - TCPHeartbeater.Config hbConfig1 = new TCPHeartbeater.Config(); - hbConfig1.setHost("0.0.0.0"); - hbConfig1.setPort(24226); - - multiSender = new MultiSender(new MultiSender.Config(), - Arrays.asList( - new TCPSender(senderConfig0, - createFailureDetector(new TCPHeartbeater(hbConfig0))), - new TCPSender(senderConfig1, - createFailureDetector(new TCPHeartbeater(hbConfig1))))); - - assertThat(multiSender.toString().length()).isGreaterThan(0); - - assertEquals(2, multiSender.getSenders().size()); - - TCPSender tcpSender = (TCPSender) multiSender.getSenders().get(0); - assertEquals("127.0.0.1", tcpSender.getHost()); - assertEquals(24225, tcpSender.getPort()); - assertTrue(tcpSender.getFailureDetector().getHeartbeater() instanceof InetSocketHeartbeater); - { - InetSocketHeartbeater hb = (InetSocketHeartbeater) tcpSender.getFailureDetector().getHeartbeater(); - assertEquals("127.0.0.1", hb.getHost()); - assertEquals(24225, hb.getPort()); - } - - tcpSender = (TCPSender) multiSender.getSenders().get(1); - assertEquals("0.0.0.0", tcpSender.getHost()); - assertEquals(24226, tcpSender.getPort()); - assertTrue(tcpSender.getFailureDetector().getHeartbeater() instanceof InetSocketHeartbeater); - { - InetSocketHeartbeater hb = (InetSocketHeartbeater) tcpSender.getFailureDetector().getHeartbeater(); - assertEquals("0.0.0.0", hb.getHost()); - assertEquals(24226, hb.getPort()); - } - } - finally { - if (multiSender != null) { - multiSender.close(); - } - } - } - - @Test - void testConstructorForSSLSender() - throws IOException - { - MultiSender multiSender = null; - try { - - SSLSender.Config senderConfig0 = new SSLSender.Config(); - senderConfig0.setPort(24225); - senderConfig0.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - - SSLHeartbeater.Config hbConfig0 = new SSLHeartbeater.Config(); - hbConfig0.setPort(24225); - - SSLSender.Config senderConfig1 = new SSLSender.Config(); - senderConfig1.setHost("0.0.0.0"); - senderConfig1.setPort(24226); - senderConfig1.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - - SSLHeartbeater.Config hbConfig1 = new SSLHeartbeater.Config(); - hbConfig1.setHost("0.0.0.0"); - hbConfig1.setPort(24226); - - multiSender = new MultiSender(new MultiSender.Config(), - Arrays.asList( - new SSLSender(senderConfig0, - createFailureDetector(new SSLHeartbeater(hbConfig0))), - new SSLSender(senderConfig1, - createFailureDetector(new SSLHeartbeater(hbConfig1))))); - - assertThat(multiSender.toString().length()).isGreaterThan(0); - - assertEquals(2, multiSender.getSenders().size()); - - SSLSender sslSender = (SSLSender) multiSender.getSenders().get(0); - assertEquals("127.0.0.1", sslSender.getHost()); - assertEquals(24225, sslSender.getPort()); - assertTrue(sslSender.getFailureDetector().getHeartbeater() instanceof InetSocketHeartbeater); - { - InetSocketHeartbeater hb = (InetSocketHeartbeater) sslSender.getFailureDetector().getHeartbeater(); - assertEquals("127.0.0.1", hb.getHost()); - assertEquals(24225, hb.getPort()); - } - - sslSender = (SSLSender) multiSender.getSenders().get(1); - assertEquals("0.0.0.0", sslSender.getHost()); - assertEquals(24226, sslSender.getPort()); - assertTrue(sslSender.getFailureDetector().getHeartbeater() instanceof InetSocketHeartbeater); - { - InetSocketHeartbeater hb = (InetSocketHeartbeater) sslSender.getFailureDetector().getHeartbeater(); - assertEquals("0.0.0.0", hb.getHost()); - assertEquals(24226, hb.getPort()); - } - } - finally { - if (multiSender != null) { - multiSender.close(); - } - } +class MultiSenderTest { + private static final Logger LOG = LoggerFactory.getLogger(MultiSenderTest.class); + + private FailureDetector createFailureDetector(Heartbeater hb) { + return new FailureDetector(new PhiAccrualFailureDetectStrategy(), hb); + } + + @Test + void testConstructorForTCPSender() throws IOException { + MultiSender multiSender = null; + try { + TCPSender.Config senderConfig0 = new TCPSender.Config(); + senderConfig0.setPort(24225); + + TCPHeartbeater.Config hbConfig0 = new TCPHeartbeater.Config(); + hbConfig0.setPort(24225); + + TCPSender.Config senderConfig1 = new TCPSender.Config(); + senderConfig1.setHost("0.0.0.0"); + senderConfig1.setPort(24226); + + TCPHeartbeater.Config hbConfig1 = new TCPHeartbeater.Config(); + hbConfig1.setHost("0.0.0.0"); + hbConfig1.setPort(24226); + + multiSender = + new MultiSender( + new MultiSender.Config(), + Arrays.asList( + new TCPSender( + senderConfig0, createFailureDetector(new TCPHeartbeater(hbConfig0))), + new TCPSender( + senderConfig1, createFailureDetector(new TCPHeartbeater(hbConfig1))))); + + assertThat(multiSender.toString().length()).isGreaterThan(0); + + assertEquals(2, multiSender.getSenders().size()); + + TCPSender tcpSender = (TCPSender) multiSender.getSenders().get(0); + assertEquals("127.0.0.1", tcpSender.getHost()); + assertEquals(24225, tcpSender.getPort()); + assertTrue(tcpSender.getFailureDetector().getHeartbeater() instanceof InetSocketHeartbeater); + { + InetSocketHeartbeater hb = + (InetSocketHeartbeater) tcpSender.getFailureDetector().getHeartbeater(); + assertEquals("127.0.0.1", hb.getHost()); + assertEquals(24225, hb.getPort()); + } + + tcpSender = (TCPSender) multiSender.getSenders().get(1); + assertEquals("0.0.0.0", tcpSender.getHost()); + assertEquals(24226, tcpSender.getPort()); + assertTrue(tcpSender.getFailureDetector().getHeartbeater() instanceof InetSocketHeartbeater); + { + InetSocketHeartbeater hb = + (InetSocketHeartbeater) tcpSender.getFailureDetector().getHeartbeater(); + assertEquals("0.0.0.0", hb.getHost()); + assertEquals(24226, hb.getPort()); + } + } finally { + if (multiSender != null) { + multiSender.close(); + } } - - @Test - void testTCPSend() - throws Exception - { - testSend(false); + } + + @Test + void testConstructorForSSLSender() throws IOException { + MultiSender multiSender = null; + try { + + SSLSender.Config senderConfig0 = new SSLSender.Config(); + senderConfig0.setPort(24225); + senderConfig0.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + + SSLHeartbeater.Config hbConfig0 = new SSLHeartbeater.Config(); + hbConfig0.setPort(24225); + + SSLSender.Config senderConfig1 = new SSLSender.Config(); + senderConfig1.setHost("0.0.0.0"); + senderConfig1.setPort(24226); + senderConfig1.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + + SSLHeartbeater.Config hbConfig1 = new SSLHeartbeater.Config(); + hbConfig1.setHost("0.0.0.0"); + hbConfig1.setPort(24226); + + multiSender = + new MultiSender( + new MultiSender.Config(), + Arrays.asList( + new SSLSender( + senderConfig0, createFailureDetector(new SSLHeartbeater(hbConfig0))), + new SSLSender( + senderConfig1, createFailureDetector(new SSLHeartbeater(hbConfig1))))); + + assertThat(multiSender.toString().length()).isGreaterThan(0); + + assertEquals(2, multiSender.getSenders().size()); + + SSLSender sslSender = (SSLSender) multiSender.getSenders().get(0); + assertEquals("127.0.0.1", sslSender.getHost()); + assertEquals(24225, sslSender.getPort()); + assertTrue(sslSender.getFailureDetector().getHeartbeater() instanceof InetSocketHeartbeater); + { + InetSocketHeartbeater hb = + (InetSocketHeartbeater) sslSender.getFailureDetector().getHeartbeater(); + assertEquals("127.0.0.1", hb.getHost()); + assertEquals(24225, hb.getPort()); + } + + sslSender = (SSLSender) multiSender.getSenders().get(1); + assertEquals("0.0.0.0", sslSender.getHost()); + assertEquals(24226, sslSender.getPort()); + assertTrue(sslSender.getFailureDetector().getHeartbeater() instanceof InetSocketHeartbeater); + { + InetSocketHeartbeater hb = + (InetSocketHeartbeater) sslSender.getFailureDetector().getHeartbeater(); + assertEquals("0.0.0.0", hb.getHost()); + assertEquals(24226, hb.getPort()); + } + } finally { + if (multiSender != null) { + multiSender.close(); + } } - - @Test - void testSSLSend() - throws Exception - { - testSend(true); + } + + @Test + void testTCPSend() throws Exception { + testSend(false); + } + + @Test + void testSSLSend() throws Exception { + testSend(true); + } + + private void testSend(boolean sslEnabled) throws Exception { + final MockMultiTCPServerWithMetrics server0 = new MockMultiTCPServerWithMetrics(sslEnabled); + server0.start(); + final MockMultiTCPServerWithMetrics server1 = new MockMultiTCPServerWithMetrics(sslEnabled); + server1.start(); + + int concurency = 20; + final int reqNum = 5000; + final CountDownLatch latch = new CountDownLatch(concurency); + + final MultiSender sender; + + if (sslEnabled) { + SSLSender.Config senderConfig0 = new SSLSender.Config(); + senderConfig0.setPort(server0.getLocalPort()); + senderConfig0.setReadTimeoutMilli(500); + senderConfig0.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + + UDPHeartbeater.Config hbConfig0 = new UDPHeartbeater.Config(); + hbConfig0.setPort(server0.getLocalPort()); + + SSLSender.Config senderConfig1 = new SSLSender.Config(); + senderConfig1.setPort(server1.getLocalPort()); + senderConfig1.setReadTimeoutMilli(500); + senderConfig1.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + + UDPHeartbeater.Config hbConfig1 = new UDPHeartbeater.Config(); + hbConfig1.setPort(server1.getLocalPort()); + + sender = + new MultiSender( + new MultiSender.Config(), + ImmutableList.of( + new SSLSender( + senderConfig0, createFailureDetector(new UDPHeartbeater(hbConfig0))), + new SSLSender( + senderConfig1, createFailureDetector(new UDPHeartbeater(hbConfig1))))); + } else { + TCPSender.Config senderConfig0 = new TCPSender.Config(); + senderConfig0.setPort(server0.getLocalPort()); + senderConfig0.setReadTimeoutMilli(500); + + UDPHeartbeater.Config hbConfig0 = new UDPHeartbeater.Config(); + hbConfig0.setPort(server0.getLocalPort()); + + TCPSender.Config senderConfig1 = new TCPSender.Config(); + senderConfig1.setPort(server1.getLocalPort()); + senderConfig1.setReadTimeoutMilli(500); + + UDPHeartbeater.Config hbConfig1 = new UDPHeartbeater.Config(); + hbConfig1.setPort(server0.getLocalPort()); + + sender = + new MultiSender( + new MultiSender.Config(), + ImmutableList.of( + new TCPSender( + senderConfig0, createFailureDetector(new UDPHeartbeater(hbConfig0))), + new TCPSender( + senderConfig1, createFailureDetector(new UDPHeartbeater(hbConfig0))))); } - private void testSend(boolean sslEnabled) - throws Exception - { - final MockMultiTCPServerWithMetrics server0 = new MockMultiTCPServerWithMetrics(sslEnabled); - server0.start(); - final MockMultiTCPServerWithMetrics server1 = new MockMultiTCPServerWithMetrics(sslEnabled); - server1.start(); - - int concurency = 20; - final int reqNum = 5000; - final CountDownLatch latch = new CountDownLatch(concurency); - - final MultiSender sender; - - if (sslEnabled) { - SSLSender.Config senderConfig0 = new SSLSender.Config(); - senderConfig0.setPort(server0.getLocalPort()); - senderConfig0.setReadTimeoutMilli(500); - senderConfig0.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - - UDPHeartbeater.Config hbConfig0 = new UDPHeartbeater.Config(); - hbConfig0.setPort(server0.getLocalPort()); - - SSLSender.Config senderConfig1 = new SSLSender.Config(); - senderConfig1.setPort(server1.getLocalPort()); - senderConfig1.setReadTimeoutMilli(500); - senderConfig1.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - - UDPHeartbeater.Config hbConfig1 = new UDPHeartbeater.Config(); - hbConfig1.setPort(server1.getLocalPort()); - - sender = new MultiSender(new MultiSender.Config(), - ImmutableList.of( - new SSLSender(senderConfig0, - createFailureDetector(new UDPHeartbeater(hbConfig0))), - new SSLSender(senderConfig1, - createFailureDetector(new UDPHeartbeater(hbConfig1))))); - } - else { - TCPSender.Config senderConfig0 = new TCPSender.Config(); - senderConfig0.setPort(server0.getLocalPort()); - senderConfig0.setReadTimeoutMilli(500); - - UDPHeartbeater.Config hbConfig0 = new UDPHeartbeater.Config(); - hbConfig0.setPort(server0.getLocalPort()); - - TCPSender.Config senderConfig1 = new TCPSender.Config(); - senderConfig1.setPort(server1.getLocalPort()); - senderConfig1.setReadTimeoutMilli(500); - - UDPHeartbeater.Config hbConfig1 = new UDPHeartbeater.Config(); - hbConfig1.setPort(server0.getLocalPort()); - - sender = new MultiSender(new MultiSender.Config(), - ImmutableList.of( - new TCPSender(senderConfig0, - createFailureDetector(new UDPHeartbeater(hbConfig0))), - new TCPSender(senderConfig1, - createFailureDetector(new UDPHeartbeater(hbConfig0))))); - } - - final ExecutorService senderExecutorService = Executors.newCachedThreadPool(); - final AtomicBoolean shouldFailOver = new AtomicBoolean(true); - for (int i = 0; i < concurency; i++) { - senderExecutorService.execute(new Runnable() - { - @Override - public void run() - { - try { - byte[] bytes = "0123456789".getBytes(Charset.forName("UTF-8")); - - for (int j = 0; j < reqNum; j++) { - if (j == reqNum / 2) { - if (shouldFailOver.getAndSet(false)) { - TimeUnit.MILLISECONDS.sleep(100); - LOG.info("Failing over..."); - server0.stop(); - } - } - sender.send(ByteBuffer.wrap(bytes)); - } - latch.countDown(); - } - catch (Exception e) { - LOG.error("Failed to send", e); + final ExecutorService senderExecutorService = Executors.newCachedThreadPool(); + final AtomicBoolean shouldFailOver = new AtomicBoolean(true); + for (int i = 0; i < concurency; i++) { + senderExecutorService.execute( + new Runnable() { + @Override + public void run() { + try { + byte[] bytes = "0123456789".getBytes(Charset.forName("UTF-8")); + + for (int j = 0; j < reqNum; j++) { + if (j == reqNum / 2) { + if (shouldFailOver.getAndSet(false)) { + TimeUnit.MILLISECONDS.sleep(100); + LOG.info("Failing over..."); + server0.stop(); } + } + sender.send(ByteBuffer.wrap(bytes)); } - }); - } + latch.countDown(); + } catch (Exception e) { + LOG.error("Failed to send", e); + } + } + }); + } - if (!latch.await(60, TimeUnit.SECONDS)) { - fail("Sending all requests is timed out"); - } + if (!latch.await(60, TimeUnit.SECONDS)) { + fail("Sending all requests is timed out"); + } - sender.close(); - - server1.waitUntilEventsStop(); - server1.stop(); - - int connectCount = 0; - int closeCount = 0; - long recvCount = 0; - long recvLen = 0; - for (MockMultiTCPServerWithMetrics server : Arrays.asList(server0, server1)) { - for (Tuple event : server.getEvents()) { - switch (event.getFirst()) { - case CONNECT: - connectCount++; - break; - case CLOSE: - closeCount++; - break; - case RECEIVE: - recvCount++; - recvLen += event.getSecond(); - break; - } - } + sender.close(); + + server1.waitUntilEventsStop(); + server1.stop(); + + int connectCount = 0; + int closeCount = 0; + long recvCount = 0; + long recvLen = 0; + for (MockMultiTCPServerWithMetrics server : Arrays.asList(server0, server1)) { + for (Tuple event : server.getEvents()) { + switch (event.getFirst()) { + case CONNECT: + connectCount++; + break; + case CLOSE: + closeCount++; + break; + case RECEIVE: + recvCount++; + recvLen += event.getSecond(); + break; } - LOG.debug("recvCount={}", recvCount); - LOG.debug("recvLen={}", recvLen); - - assertEquals(2, connectCount); - // This test doesn't use actual PackedForward format so that it can simply test MultiSender itself. - // But w/o ack responses, Sender can't detect dropped requests. So some margin for expected result is allowed here. - long minExpectedRecvLen = ((long) (concurency * (sslEnabled ? 0.5 : 0.8)) * reqNum) * 10; - long maxExpectedRecvLen = ((long) concurency * reqNum) * 10; - assertThat(recvLen).isGreaterThanOrEqualTo(minExpectedRecvLen); - assertThat(recvLen).isLessThanOrEqualTo(maxExpectedRecvLen); - assertEquals(1, closeCount); + } } + LOG.debug("recvCount={}", recvCount); + LOG.debug("recvLen={}", recvLen); + + assertEquals(2, connectCount); + // This test doesn't use actual PackedForward format so that it can simply test MultiSender + // itself. + // But w/o ack responses, Sender can't detect dropped requests. So some margin for expected + // result is allowed here. + long minExpectedRecvLen = ((long) (concurency * (sslEnabled ? 0.5 : 0.8)) * reqNum) * 10; + long maxExpectedRecvLen = ((long) concurency * reqNum) * 10; + assertThat(recvLen).isGreaterThanOrEqualTo(minExpectedRecvLen); + assertThat(recvLen).isLessThanOrEqualTo(maxExpectedRecvLen); + assertEquals(1, closeCount); + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/RetryableSenderTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/RetryableSenderTest.java index f1307f21..33652c6c 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/RetryableSenderTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/RetryableSenderTest.java @@ -16,90 +16,81 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.fluentd.ingester.sender.retry.ExponentialBackOffRetryStrategy; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.fluentd.ingester.sender.retry.ExponentialBackOffRetryStrategy; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertThrows; - -class RetryableSenderTest -{ - @Test - void testSend() - throws IOException - { - ExponentialBackOffRetryStrategy.Config retryStrategyConfig = - new ExponentialBackOffRetryStrategy.Config(); - retryStrategyConfig.setMaxRetryCount(3); - - RetryableSender.Config senderConfig = new RetryableSender.Config(); - RetryableSender sender = new RetryableSender(senderConfig, - new FailurableSender(3), new ExponentialBackOffRetryStrategy(retryStrategyConfig)); - - FailurableSender baseSender = (FailurableSender) sender.getBaseSender(); - assertThat(baseSender.getRetry()).isEqualTo(0); - sender.send(ByteBuffer.allocate(64)); - assertThat(baseSender.getRetry()).isEqualTo(3); +class RetryableSenderTest { + @Test + void testSend() throws IOException { + ExponentialBackOffRetryStrategy.Config retryStrategyConfig = + new ExponentialBackOffRetryStrategy.Config(); + retryStrategyConfig.setMaxRetryCount(3); + + RetryableSender.Config senderConfig = new RetryableSender.Config(); + RetryableSender sender = + new RetryableSender( + senderConfig, + new FailurableSender(3), + new ExponentialBackOffRetryStrategy(retryStrategyConfig)); + + FailurableSender baseSender = (FailurableSender) sender.getBaseSender(); + assertThat(baseSender.getRetry()).isEqualTo(0); + sender.send(ByteBuffer.allocate(64)); + assertThat(baseSender.getRetry()).isEqualTo(3); + } + + @Test + void testSendRetryOver() throws IOException { + ExponentialBackOffRetryStrategy.Config retryStrategyConfig = + new ExponentialBackOffRetryStrategy.Config(); + retryStrategyConfig.setMaxRetryCount(2); + + RetryableSender.Config senderConfig = new RetryableSender.Config(); + RetryableSender sender = + new RetryableSender( + senderConfig, + new FailurableSender(3), + new ExponentialBackOffRetryStrategy(retryStrategyConfig)); + + FailurableSender baseSender = (FailurableSender) sender.getBaseSender(); + assertThat(baseSender.getRetry()).isEqualTo(0); + assertThrows( + RetryableSender.RetryOverException.class, () -> sender.send(ByteBuffer.allocate(64))); + } + + static class FailurableSender extends FluentdSender { + private final int maxFailures; + private int retry; + + FailurableSender(int maxFailures) { + super(new FluentdSender.Config()); + this.maxFailures = maxFailures; } - @Test - void testSendRetryOver() - throws IOException - { - ExponentialBackOffRetryStrategy.Config retryStrategyConfig = - new ExponentialBackOffRetryStrategy.Config(); - retryStrategyConfig.setMaxRetryCount(2); - - RetryableSender.Config senderConfig = new RetryableSender.Config(); - RetryableSender sender = new RetryableSender(senderConfig, - new FailurableSender(3), new ExponentialBackOffRetryStrategy(retryStrategyConfig)); - - FailurableSender baseSender = (FailurableSender) sender.getBaseSender(); - assertThat(baseSender.getRetry()).isEqualTo(0); - assertThrows(RetryableSender.RetryOverException.class, - () -> sender.send(ByteBuffer.allocate(64))); + @Override + public boolean isAvailable() { + return true; } - static class FailurableSender - extends FluentdSender - { - private final int maxFailures; - private int retry; - - FailurableSender(int maxFailures) - { - super(new FluentdSender.Config()); - this.maxFailures = maxFailures; - } - - @Override - public boolean isAvailable() - { - return true; - } - - @Override - protected void sendInternal(List buffers, String ackToken) - throws IOException - { - if (retry < maxFailures) { - retry++; - throw new IOException("Something is wrong..."); - } - } - - int getRetry() - { - return retry; - } + @Override + protected void sendInternal(List buffers, String ackToken) throws IOException { + if (retry < maxFailures) { + retry++; + throw new IOException("Something is wrong..."); + } + } - @Override - public void close() - { - } + int getRetry() { + return retry; } + + @Override + public void close() {} + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSenderTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSenderTest.java index c6191667..f9008358 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSenderTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSenderTest.java @@ -16,15 +16,12 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.fluentd.MockTCPServer; -import org.komamitsu.fluency.fluentd.MockTCPServerWithMetrics; -import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; -import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.PhiAccrualFailureDetectStrategy; -import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.SSLHeartbeater; -import org.komamitsu.fluency.util.Tuple; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.komamitsu.fluency.fluentd.SSLTestSocketFactories.SSL_CLIENT_SOCKET_FACTORY; import java.io.IOException; import java.net.SocketException; @@ -36,241 +33,232 @@ import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.fluentd.MockTCPServer; +import org.komamitsu.fluency.fluentd.MockTCPServerWithMetrics; +import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; +import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.PhiAccrualFailureDetectStrategy; +import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.SSLHeartbeater; +import org.komamitsu.fluency.util.Tuple; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.komamitsu.fluency.fluentd.SSLTestSocketFactories.SSL_CLIENT_SOCKET_FACTORY; - -class SSLSenderTest -{ - private static final Logger LOG = LoggerFactory.getLogger(SSLSenderTest.class); - - @Test - void testSend() - throws Exception - { - testSendBase(port -> { - SSLSender.Config config = new SSLSender.Config(); - config.setPort(port); - config.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - return new SSLSender(config); - }, - count -> assertThat(count).isEqualTo(1), - count -> assertThat(count).isEqualTo(1)); +class SSLSenderTest { + private static final Logger LOG = LoggerFactory.getLogger(SSLSenderTest.class); + + @Test + void testSend() throws Exception { + testSendBase( + port -> { + SSLSender.Config config = new SSLSender.Config(); + config.setPort(port); + config.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + return new SSLSender(config); + }, + count -> assertThat(count).isEqualTo(1), + count -> assertThat(count).isEqualTo(1)); + } + + @Test + void testSendWithHeartbeart() throws Exception { + testSendBase( + port -> { + SSLHeartbeater.Config hbConfig = new SSLHeartbeater.Config(); + hbConfig.setPort(port); + hbConfig.setIntervalMillis(400); + SSLSender.Config senderConfig = new SSLSender.Config(); + senderConfig.setPort(port); + senderConfig.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + return new SSLSender( + senderConfig, + new FailureDetector( + new PhiAccrualFailureDetectStrategy(), new SSLHeartbeater(hbConfig))); + }, + count -> assertThat(count).isGreaterThan(1), + count -> assertThat(count).isGreaterThan(1)); + } + + private void testSendBase( + SSLSenderCreator sslSenderCreator, + Consumer connectCountAssertion, + Consumer closeCountAssertion) + throws Exception { + MockTCPServerWithMetrics server = new MockTCPServerWithMetrics(true); + server.start(); + + int concurency = 20; + final int reqNum = 5000; + final CountDownLatch latch = new CountDownLatch(concurency); + SSLSender sender = sslSenderCreator.create(server.getLocalPort()); + + // To receive heartbeat at least once + TimeUnit.MILLISECONDS.sleep(500); + + final ExecutorService senderExecutorService = Executors.newCachedThreadPool(); + for (int i = 0; i < concurency; i++) { + senderExecutorService.execute( + () -> { + try { + byte[] bytes = "0123456789".getBytes(Charset.forName("UTF-8")); + + for (int j = 0; j < reqNum; j++) { + sender.send(ByteBuffer.wrap(bytes)); + } + latch.countDown(); + } catch (IOException e) { + e.printStackTrace(); + } + }); } - @Test - void testSendWithHeartbeart() - throws Exception - { - testSendBase(port -> { - SSLHeartbeater.Config hbConfig = new SSLHeartbeater.Config(); - hbConfig.setPort(port); - hbConfig.setIntervalMillis(400); - SSLSender.Config senderConfig = new SSLSender.Config(); - senderConfig.setPort(port); - senderConfig.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - return new SSLSender(senderConfig, - new FailureDetector( - new PhiAccrualFailureDetectStrategy(), - new SSLHeartbeater(hbConfig))); - }, - count -> assertThat(count).isGreaterThan(1), - count -> assertThat(count).isGreaterThan(1)); + if (!latch.await(30, TimeUnit.SECONDS)) { + fail("Sending all requests timed out"); } - - private void testSendBase( - SSLSenderCreator sslSenderCreator, - Consumer connectCountAssertion, - Consumer closeCountAssertion) - throws Exception - { - MockTCPServerWithMetrics server = new MockTCPServerWithMetrics(true); - server.start(); - - int concurency = 20; - final int reqNum = 5000; - final CountDownLatch latch = new CountDownLatch(concurency); - SSLSender sender = sslSenderCreator.create(server.getLocalPort()); - - // To receive heartbeat at least once - TimeUnit.MILLISECONDS.sleep(500); - - final ExecutorService senderExecutorService = Executors.newCachedThreadPool(); - for (int i = 0; i < concurency; i++) { - senderExecutorService.execute(() -> { - try { - byte[] bytes = "0123456789".getBytes(Charset.forName("UTF-8")); - - for (int j = 0; j < reqNum; j++) { - sender.send(ByteBuffer.wrap(bytes)); - } - latch.countDown(); - } - catch (IOException e) { - e.printStackTrace(); - } - }); - } - - if (!latch.await(30, TimeUnit.SECONDS)) { - fail("Sending all requests timed out"); - } - sender.close(); - - server.waitUntilEventsStop(); - server.stop(); - - int connectCount = 0; - int closeCount = 0; - long recvCount = 0; - long recvLen = 0; - for (Tuple event : server.getEvents()) { - switch (event.getFirst()) { - case CONNECT: - connectCount++; - break; - case CLOSE: - closeCount++; - break; - case RECEIVE: - recvCount++; - recvLen += event.getSecond(); - break; - } - } - LOG.debug("recvCount={}", recvCount); - - connectCountAssertion.accept(connectCount); - assertThat(recvLen).isEqualTo((long) concurency * reqNum * 10); - closeCountAssertion.accept(closeCount); + sender.close(); + + server.waitUntilEventsStop(); + server.stop(); + + int connectCount = 0; + int closeCount = 0; + long recvCount = 0; + long recvLen = 0; + for (Tuple event : server.getEvents()) { + switch (event.getFirst()) { + case CONNECT: + connectCount++; + break; + case CLOSE: + closeCount++; + break; + case RECEIVE: + recvCount++; + recvLen += event.getSecond(); + break; + } } - - @Test - void testConnectionTimeout() - throws IOException, InterruptedException - { - final CountDownLatch latch = new CountDownLatch(1); - ExecutorService executorService = Executors.newSingleThreadExecutor(); - executorService.execute(() -> { + LOG.debug("recvCount={}", recvCount); + + connectCountAssertion.accept(connectCount); + assertThat(recvLen).isEqualTo((long) concurency * reqNum * 10); + closeCountAssertion.accept(closeCount); + } + + @Test + void testConnectionTimeout() throws IOException, InterruptedException { + final CountDownLatch latch = new CountDownLatch(1); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + executorService.execute( + () -> { + SSLSender.Config senderConfig = new SSLSender.Config(); + senderConfig.setHost("192.0.2.0"); + senderConfig.setConnectionTimeoutMilli(1000); + senderConfig.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + SSLSender sender = new SSLSender(senderConfig); + try { + sender.send(ByteBuffer.wrap("hello, world".getBytes("UTF-8"))); + } catch (Throwable e) { + if (e instanceof SocketTimeoutException) { + latch.countDown(); + } else { + throw new RuntimeException(e); + } + } + }); + assertTrue(latch.await(2000, TimeUnit.MILLISECONDS)); + } + + @Test + void testReadTimeout() throws Exception { + final MockTCPServer server = new MockTCPServer(true); + server.start(); + + try { + final CountDownLatch latch = new CountDownLatch(1); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + executorService.execute( + () -> { SSLSender.Config senderConfig = new SSLSender.Config(); - senderConfig.setHost("192.0.2.0"); - senderConfig.setConnectionTimeoutMilli(1000); + senderConfig.setPort(server.getLocalPort()); + senderConfig.setReadTimeoutMilli(1000); senderConfig.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); SSLSender sender = new SSLSender(senderConfig); try { - sender.send(ByteBuffer.wrap("hello, world".getBytes("UTF-8"))); - } - catch (Throwable e) { - if (e instanceof SocketTimeoutException) { - latch.countDown(); - } - else { - throw new RuntimeException(e); - } + sender.sendWithAck( + Arrays.asList(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))), + "Waiting ack forever"); + } catch (Throwable e) { + if (e instanceof SocketTimeoutException) { + latch.countDown(); + } else { + throw new RuntimeException(e); + } } - }); - assertTrue(latch.await(2000, TimeUnit.MILLISECONDS)); + }); + assertTrue(latch.await(2000, TimeUnit.MILLISECONDS)); + } finally { + server.stop(); } + } - @Test - void testReadTimeout() - throws Exception - { - final MockTCPServer server = new MockTCPServer(true); - server.start(); - - try { - final CountDownLatch latch = new CountDownLatch(1); - ExecutorService executorService = Executors.newSingleThreadExecutor(); - executorService.execute(() -> { - SSLSender.Config senderConfig = new SSLSender.Config(); - senderConfig.setPort(server.getLocalPort()); - senderConfig.setReadTimeoutMilli(1000); - senderConfig.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - SSLSender sender = new SSLSender(senderConfig); - try { - sender.sendWithAck(Arrays.asList(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))), "Waiting ack forever"); - } - catch (Throwable e) { - if (e instanceof SocketTimeoutException) { - latch.countDown(); - } - else { - throw new RuntimeException(e); - } - } - }); - assertTrue(latch.await(2000, TimeUnit.MILLISECONDS)); - } - finally { - server.stop(); - } - } - - private Throwable extractRootCause(Throwable exception) - { - Throwable e = exception; - while (e.getCause() != null) { - e = e.getCause(); - } - return e; + private Throwable extractRootCause(Throwable exception) { + Throwable e = exception; + while (e.getCause() != null) { + e = e.getCause(); } + return e; + } + + @Test + void testDisconnBeforeRecv() throws Exception { + final MockTCPServer server = new MockTCPServer(true); + server.start(); + + try { + final CountDownLatch latch = new CountDownLatch(1); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + executorService.execute( + () -> { + SSLSender.Config senderConfig = new SSLSender.Config(); + senderConfig.setPort(server.getLocalPort()); + senderConfig.setReadTimeoutMilli(4000); + senderConfig.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + SSLSender sender = new SSLSender(senderConfig); + try { + sender.sendWithAck( + Arrays.asList(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))), + "Waiting ack forever"); + } catch (Throwable e) { + Throwable rootCause = extractRootCause(e); + if (rootCause instanceof SocketException + && rootCause.getMessage().toLowerCase().contains("disconnected")) { + latch.countDown(); + } else { + throw new RuntimeException(e); + } + } + }); - @Test - void testDisconnBeforeRecv() - throws Exception - { - final MockTCPServer server = new MockTCPServer(true); - server.start(); - - try { - final CountDownLatch latch = new CountDownLatch(1); - ExecutorService executorService = Executors.newSingleThreadExecutor(); - executorService.execute(() -> { - SSLSender.Config senderConfig = new SSLSender.Config(); - senderConfig.setPort(server.getLocalPort()); - senderConfig.setReadTimeoutMilli(4000); - senderConfig.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - SSLSender sender = new SSLSender(senderConfig); - try { - sender.sendWithAck(Arrays.asList(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))), "Waiting ack forever"); - } - catch (Throwable e) { - Throwable rootCause = extractRootCause(e); - if (rootCause instanceof SocketException && rootCause.getMessage().toLowerCase().contains("disconnected")) { - latch.countDown(); - } - else { - throw new RuntimeException(e); - } - } - }); - - TimeUnit.MILLISECONDS.sleep(1000); - server.stop(true); + TimeUnit.MILLISECONDS.sleep(1000); + server.stop(true); - assertTrue(latch.await(8000, TimeUnit.MILLISECONDS)); - } - finally { - server.stop(); - } + assertTrue(latch.await(8000, TimeUnit.MILLISECONDS)); + } finally { + server.stop(); } - - @Test - void testClose() - throws Exception - { - final MockTCPServer server = new MockTCPServer(true); - server.start(); - - try { - final AtomicLong duration = new AtomicLong(); - ExecutorService executorService = Executors.newSingleThreadExecutor(); - Future future = executorService.submit(() -> { + } + + @Test + void testClose() throws Exception { + final MockTCPServer server = new MockTCPServer(true); + server.start(); + + try { + final AtomicLong duration = new AtomicLong(); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + Future future = + executorService.submit( + () -> { SSLSender.Config senderConfig = new SSLSender.Config(); senderConfig.setPort(server.getLocalPort()); senderConfig.setWaitBeforeCloseMilli(1500); @@ -278,57 +266,52 @@ void testClose() SSLSender sender = new SSLSender(senderConfig); long start; try { - sender.send(Arrays.asList(ByteBuffer.wrap("hello, world".getBytes("UTF-8")))); - start = System.currentTimeMillis(); - sender.close(); - duration.set(System.currentTimeMillis() - start); - } - catch (Exception e) { - LOG.error("Unexpected exception", e); + sender.send(Arrays.asList(ByteBuffer.wrap("hello, world".getBytes("UTF-8")))); + start = System.currentTimeMillis(); + sender.close(); + duration.set(System.currentTimeMillis() - start); + } catch (Exception e) { + LOG.error("Unexpected exception", e); } return null; - }); - future.get(3000, TimeUnit.MILLISECONDS); - assertTrue(duration.get() > 1000 && duration.get() < 2000); - } - finally { - server.stop(); - } + }); + future.get(3000, TimeUnit.MILLISECONDS); + assertTrue(duration.get() > 1000 && duration.get() < 2000); + } finally { + server.stop(); } + } + + @Test + void testConfig() { + SSLSender.Config config = new SSLSender.Config(); + assertEquals(1000, config.getWaitBeforeCloseMilli()); + // TODO: Add others later + } - @Test - void testConfig() + @Test + void validateConfig() { { - SSLSender.Config config = new SSLSender.Config(); - assertEquals(1000, config.getWaitBeforeCloseMilli()); - // TODO: Add others later + SSLSender.Config config = new SSLSender.Config(); + config.setConnectionTimeoutMilli(9); + assertThrows(IllegalArgumentException.class, () -> new SSLSender(config)); } - @Test - void validateConfig() { - { - SSLSender.Config config = new SSLSender.Config(); - config.setConnectionTimeoutMilli(9); - assertThrows(IllegalArgumentException.class, () -> new SSLSender(config)); - } - - { - SSLSender.Config config = new SSLSender.Config(); - config.setReadTimeoutMilli(9); - assertThrows(IllegalArgumentException.class, () -> new SSLSender(config)); - } - - { - SSLSender.Config config = new SSLSender.Config(); - config.setWaitBeforeCloseMilli(-1); - assertThrows(IllegalArgumentException.class, () -> new SSLSender(config)); - } + SSLSender.Config config = new SSLSender.Config(); + config.setReadTimeoutMilli(9); + assertThrows(IllegalArgumentException.class, () -> new SSLSender(config)); } - interface SSLSenderCreator { - SSLSender create(int port); + SSLSender.Config config = new SSLSender.Config(); + config.setWaitBeforeCloseMilli(-1); + assertThrows(IllegalArgumentException.class, () -> new SSLSender(config)); } -} \ No newline at end of file + } + + interface SSLSenderCreator { + SSLSender create(int port); + } +} diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSocketBuilderTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSocketBuilderTest.java index c3fd8650..0b744e01 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSocketBuilderTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSocketBuilderTest.java @@ -16,13 +16,8 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.fluentd.SSLTestSocketFactories; - -import javax.net.ssl.SSLServerSocket; -import javax.net.ssl.SSLSocket; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.komamitsu.fluency.fluentd.SSLTestSocketFactories.SSL_CLIENT_SOCKET_FACTORY; import java.io.IOException; import java.io.OutputStream; @@ -40,65 +35,65 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; +import javax.net.ssl.SSLServerSocket; +import javax.net.ssl.SSLSocket; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.fluentd.SSLTestSocketFactories; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.komamitsu.fluency.fluentd.SSLTestSocketFactories.SSL_CLIENT_SOCKET_FACTORY; - -class SSLSocketBuilderTest -{ - private SSLServerSocket serverSocket; +class SSLSocketBuilderTest { + private SSLServerSocket serverSocket; - @BeforeEach - void setUp() - throws IOException, CertificateException, NoSuchAlgorithmException, UnrecoverableKeyException, KeyStoreException, KeyManagementException - { - serverSocket = SSLTestSocketFactories.createServerSocket(); - } + @BeforeEach + void setUp() + throws IOException, CertificateException, NoSuchAlgorithmException, UnrecoverableKeyException, + KeyStoreException, KeyManagementException { + serverSocket = SSLTestSocketFactories.createServerSocket(); + } - @AfterEach - void tearDown() - throws IOException - { - if (serverSocket != null) { - serverSocket.close(); - } + @AfterEach + void tearDown() throws IOException { + if (serverSocket != null) { + serverSocket.close(); } + } - @Test - void testWithServer() - throws IOException, InterruptedException, ExecutionException, TimeoutException - { - final AtomicInteger readLen = new AtomicInteger(); - final byte[] buf = new byte[256]; + @Test + void testWithServer() + throws IOException, InterruptedException, ExecutionException, TimeoutException { + final AtomicInteger readLen = new AtomicInteger(); + final byte[] buf = new byte[256]; - ExecutorService executorService = Executors.newSingleThreadExecutor(); - Future future = executorService.submit(new Callable() - { - @Override - public Void call() - throws Exception - { + ExecutorService executorService = Executors.newSingleThreadExecutor(); + Future future = + executorService.submit( + new Callable() { + @Override + public Void call() throws Exception { Socket clientSocket = serverSocket.accept(); readLen.set(clientSocket.getInputStream().read(buf)); return null; - } - }); + } + }); - SSLSocket sslSocket = new SSLSocketBuilder("localhost", serverSocket.getLocalPort(), 5000, 5000, SSL_CLIENT_SOCKET_FACTORY).build(); + SSLSocket sslSocket = + new SSLSocketBuilder( + "localhost", serverSocket.getLocalPort(), 5000, 5000, SSL_CLIENT_SOCKET_FACTORY) + .build(); - try { - OutputStream outputStream = sslSocket.getOutputStream(); - outputStream.write("hello".getBytes("ASCII")); - outputStream.flush(); - } - finally { - sslSocket.close(); - } + try { + OutputStream outputStream = sslSocket.getOutputStream(); + outputStream.write("hello".getBytes("ASCII")); + outputStream.flush(); + } finally { + sslSocket.close(); + } - future.get(10, TimeUnit.SECONDS); - executorService.shutdown(); - executorService.awaitTermination(10, TimeUnit.SECONDS); + future.get(10, TimeUnit.SECONDS); + executorService.shutdown(); + executorService.awaitTermination(10, TimeUnit.SECONDS); - assertEquals(new String(buf, 0, readLen.get(), "ASCII"), "hello"); - } + assertEquals(new String(buf, 0, readLen.get(), "ASCII"), "hello"); + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/TCPSenderTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/TCPSenderTest.java index cd6bfa92..6826c554 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/TCPSenderTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/TCPSenderTest.java @@ -16,15 +16,10 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.fluentd.MockTCPServer; -import org.komamitsu.fluency.fluentd.MockTCPServerWithMetrics; -import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; -import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.PhiAccrualFailureDetectStrategy; -import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.TCPHeartbeater; -import org.komamitsu.fluency.util.Tuple; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.net.SocketException; @@ -40,291 +35,279 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.fluentd.MockTCPServer; +import org.komamitsu.fluency.fluentd.MockTCPServerWithMetrics; +import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; +import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.PhiAccrualFailureDetectStrategy; +import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.TCPHeartbeater; +import org.komamitsu.fluency.util.Tuple; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class TCPSenderTest -{ - private static final Logger LOG = LoggerFactory.getLogger(TCPSenderTest.class); - - @Test - void testSend() - throws Exception - { - testSendBase(port -> { - TCPSender.Config senderConfig = new TCPSender.Config(); - senderConfig.setPort(port); - return new TCPSender(senderConfig); - }, - count -> assertThat(count).isEqualTo(1), - count -> assertThat(count).isEqualTo(1)); +class TCPSenderTest { + private static final Logger LOG = LoggerFactory.getLogger(TCPSenderTest.class); + + @Test + void testSend() throws Exception { + testSendBase( + port -> { + TCPSender.Config senderConfig = new TCPSender.Config(); + senderConfig.setPort(port); + return new TCPSender(senderConfig); + }, + count -> assertThat(count).isEqualTo(1), + count -> assertThat(count).isEqualTo(1)); + } + + @Test + void testSendWithHeartbeart() throws Exception { + testSendBase( + port -> { + TCPHeartbeater.Config hbConfig = new TCPHeartbeater.Config(); + hbConfig.setPort(port); + hbConfig.setIntervalMillis(400); + + TCPSender.Config senderConfig = new TCPSender.Config(); + senderConfig.setPort(port); + + return new TCPSender( + senderConfig, + new FailureDetector( + new PhiAccrualFailureDetectStrategy(), new TCPHeartbeater(hbConfig))); + }, + count -> assertThat(count).isGreaterThan(1), + count -> assertThat(count).isGreaterThan(1)); + } + + private void testSendBase( + TCPSenderCreater senderCreater, + Consumer connectCountAssertion, + Consumer closeCountAssertion) + throws Exception { + MockTCPServerWithMetrics server = new MockTCPServerWithMetrics(false); + server.start(); + + int concurency = 20; + final int reqNum = 5000; + final CountDownLatch latch = new CountDownLatch(concurency); + TCPSender sender = senderCreater.create(server.getLocalPort()); + + // To receive heartbeat at least once + TimeUnit.MILLISECONDS.sleep(500); + + final ExecutorService senderExecutorService = Executors.newCachedThreadPool(); + for (int i = 0; i < concurency; i++) { + senderExecutorService.execute( + () -> { + try { + byte[] bytes = "0123456789".getBytes(Charset.forName("UTF-8")); + + for (int j = 0; j < reqNum; j++) { + sender.send(ByteBuffer.wrap(bytes)); + } + latch.countDown(); + } catch (IOException e) { + e.printStackTrace(); + } + }); } - @Test - void testSendWithHeartbeart() - throws Exception - { - testSendBase(port -> { - TCPHeartbeater.Config hbConfig = new TCPHeartbeater.Config(); - hbConfig.setPort(port); - hbConfig.setIntervalMillis(400); - - TCPSender.Config senderConfig = new TCPSender.Config(); - senderConfig.setPort(port); - - return new TCPSender(senderConfig, - new FailureDetector( - new PhiAccrualFailureDetectStrategy(), - new TCPHeartbeater(hbConfig))); - }, - count -> assertThat(count).isGreaterThan(1), - count -> assertThat(count).isGreaterThan(1)); + assertTrue(latch.await(4, TimeUnit.SECONDS)); + sender.close(); + + server.waitUntilEventsStop(); + server.stop(); + + int connectCount = 0; + int closeCount = 0; + long recvCount = 0; + long recvLen = 0; + for (Tuple event : server.getEvents()) { + switch (event.getFirst()) { + case CONNECT: + connectCount++; + break; + case CLOSE: + closeCount++; + break; + case RECEIVE: + recvCount++; + recvLen += event.getSecond(); + break; + } } - - private void testSendBase( - TCPSenderCreater senderCreater, - Consumer connectCountAssertion, - Consumer closeCountAssertion) - throws Exception - { - MockTCPServerWithMetrics server = new MockTCPServerWithMetrics(false); - server.start(); - - int concurency = 20; - final int reqNum = 5000; - final CountDownLatch latch = new CountDownLatch(concurency); - TCPSender sender = senderCreater.create(server.getLocalPort()); - - // To receive heartbeat at least once - TimeUnit.MILLISECONDS.sleep(500); - - final ExecutorService senderExecutorService = Executors.newCachedThreadPool(); - for (int i = 0; i < concurency; i++) { - senderExecutorService.execute(() -> { - try { - byte[] bytes = "0123456789".getBytes(Charset.forName("UTF-8")); - - for (int j = 0; j < reqNum; j++) { - sender.send(ByteBuffer.wrap(bytes)); - } - latch.countDown(); - } - catch (IOException e) { - e.printStackTrace(); - } - }); - } - - assertTrue(latch.await(4, TimeUnit.SECONDS)); - sender.close(); - - server.waitUntilEventsStop(); - server.stop(); - - int connectCount = 0; - int closeCount = 0; - long recvCount = 0; - long recvLen = 0; - for (Tuple event : server.getEvents()) { - switch (event.getFirst()) { - case CONNECT: - connectCount++; - break; - case CLOSE: - closeCount++; - break; - case RECEIVE: - recvCount++; - recvLen += event.getSecond(); - break; + LOG.debug("recvCount={}", recvCount); + + connectCountAssertion.accept(connectCount); + assertThat(recvLen).isEqualTo((long) concurency * reqNum * 10); + closeCountAssertion.accept(closeCount); + } + + @Test + void testConnectionTimeout() throws InterruptedException { + final CountDownLatch latch = new CountDownLatch(1); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + executorService.execute( + () -> { + TCPSender.Config senderConfig = new TCPSender.Config(); + senderConfig.setHost("192.0.2.0"); + senderConfig.setConnectionTimeoutMilli(1000); + TCPSender sender = new TCPSender(senderConfig); + try { + sender.send(ByteBuffer.wrap("hello, world".getBytes("UTF-8"))); + } catch (Throwable e) { + if (e instanceof SocketTimeoutException) { + latch.countDown(); + } else { + throw new RuntimeException(e); } - } - LOG.debug("recvCount={}", recvCount); - - connectCountAssertion.accept(connectCount); - assertThat(recvLen).isEqualTo((long) concurency * reqNum * 10); - closeCountAssertion.accept(closeCount); - } - - @Test - void testConnectionTimeout() - throws InterruptedException - { - final CountDownLatch latch = new CountDownLatch(1); - ExecutorService executorService = Executors.newSingleThreadExecutor(); - executorService.execute(() -> { + } + }); + assertTrue(latch.await(2000, TimeUnit.MILLISECONDS)); + } + + @Test + void testReadTimeout() throws Exception { + final MockTCPServer server = new MockTCPServer(false); + server.start(); + + try { + final CountDownLatch latch = new CountDownLatch(1); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + executorService.execute( + () -> { TCPSender.Config senderConfig = new TCPSender.Config(); - senderConfig.setHost("192.0.2.0"); - senderConfig.setConnectionTimeoutMilli(1000); + senderConfig.setPort(server.getLocalPort()); + senderConfig.setReadTimeoutMilli(1000); TCPSender sender = new TCPSender(senderConfig); try { - sender.send(ByteBuffer.wrap("hello, world".getBytes("UTF-8"))); + sender.sendWithAck( + Arrays.asList(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))), + "Waiting ack forever"); + } catch (Throwable e) { + if (e instanceof SocketTimeoutException) { + latch.countDown(); + } else { + throw new RuntimeException(e); + } } - catch (Throwable e) { - if (e instanceof SocketTimeoutException) { - latch.countDown(); - } - else { - throw new RuntimeException(e); - } - } - }); - assertTrue(latch.await(2000, TimeUnit.MILLISECONDS)); + }); + assertTrue(latch.await(2000, TimeUnit.MILLISECONDS)); + } finally { + server.stop(); } + } - @Test - void testReadTimeout() - throws Exception - { - final MockTCPServer server = new MockTCPServer(false); - server.start(); - - try { - final CountDownLatch latch = new CountDownLatch(1); - ExecutorService executorService = Executors.newSingleThreadExecutor(); - executorService.execute(() -> { - TCPSender.Config senderConfig = new TCPSender.Config(); - senderConfig.setPort(server.getLocalPort()); - senderConfig.setReadTimeoutMilli(1000); - TCPSender sender = new TCPSender(senderConfig); - try { - sender.sendWithAck(Arrays.asList(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))), "Waiting ack forever"); - } - catch (Throwable e) { - if (e instanceof SocketTimeoutException) { - latch.countDown(); - } - else { - throw new RuntimeException(e); - } - } - }); - assertTrue(latch.await(2000, TimeUnit.MILLISECONDS)); - } - finally { - server.stop(); - } + private Throwable extractRootCause(Throwable exception) { + Throwable e = exception; + while (e.getCause() != null) { + e = e.getCause(); } + return e; + } + + @Test + void testDisconnBeforeRecv() throws Exception { + final MockTCPServer server = new MockTCPServer(false); + server.start(); + + try { + final CountDownLatch latch = new CountDownLatch(1); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + executorService.execute( + () -> { + TCPSender.Config senderConfig = new TCPSender.Config(); + senderConfig.setPort(server.getLocalPort()); + senderConfig.setReadTimeoutMilli(4000); + TCPSender sender = new TCPSender(senderConfig); + try { + sender.sendWithAck( + Arrays.asList(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))), + "Waiting ack forever"); + } catch (Throwable e) { + Throwable rootCause = extractRootCause(e); + if (rootCause instanceof SocketException + && rootCause.getMessage().toLowerCase().contains("disconnected")) { + latch.countDown(); + } else { + throw new RuntimeException(e); + } + } + }); - private Throwable extractRootCause(Throwable exception) - { - Throwable e = exception; - while (e.getCause() != null) { - e = e.getCause(); - } - return e; - } - - @Test - void testDisconnBeforeRecv() - throws Exception - { - final MockTCPServer server = new MockTCPServer(false); - server.start(); - - try { - final CountDownLatch latch = new CountDownLatch(1); - ExecutorService executorService = Executors.newSingleThreadExecutor(); - executorService.execute(() -> { - TCPSender.Config senderConfig = new TCPSender.Config(); - senderConfig.setPort(server.getLocalPort()); - senderConfig.setReadTimeoutMilli(4000); - TCPSender sender = new TCPSender(senderConfig); - try { - sender.sendWithAck(Arrays.asList(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))), "Waiting ack forever"); - } - catch (Throwable e) { - Throwable rootCause = extractRootCause(e); - if (rootCause instanceof SocketException && rootCause.getMessage().toLowerCase().contains("disconnected")) { - latch.countDown(); - } - else { - throw new RuntimeException(e); - } - } - }); - - TimeUnit.MILLISECONDS.sleep(1000); - server.stop(true); + TimeUnit.MILLISECONDS.sleep(1000); + server.stop(true); - assertTrue(latch.await(8000, TimeUnit.MILLISECONDS)); - } - finally { - server.stop(); - } + assertTrue(latch.await(8000, TimeUnit.MILLISECONDS)); + } finally { + server.stop(); } - - @Test - void testClose() - throws Exception - { - final MockTCPServer server = new MockTCPServer(false); - server.start(); - - try { - final AtomicLong duration = new AtomicLong(); - ExecutorService executorService = Executors.newSingleThreadExecutor(); - Future future = executorService.submit(() -> { + } + + @Test + void testClose() throws Exception { + final MockTCPServer server = new MockTCPServer(false); + server.start(); + + try { + final AtomicLong duration = new AtomicLong(); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + Future future = + executorService.submit( + () -> { TCPSender.Config senderConfig = new TCPSender.Config(); senderConfig.setPort(server.getLocalPort()); senderConfig.setWaitBeforeCloseMilli(1500); TCPSender sender = new TCPSender(senderConfig); long start; try { - sender.send(Arrays.asList(ByteBuffer.wrap("hello, world".getBytes("UTF-8")))); - start = System.currentTimeMillis(); - sender.close(); - duration.set(System.currentTimeMillis() - start); - } - catch (Exception e) { - LOG.error("Unexpected exception", e); + sender.send(Arrays.asList(ByteBuffer.wrap("hello, world".getBytes("UTF-8")))); + start = System.currentTimeMillis(); + sender.close(); + duration.set(System.currentTimeMillis() - start); + } catch (Exception e) { + LOG.error("Unexpected exception", e); } return null; - }); - future.get(3000, TimeUnit.MILLISECONDS); - assertTrue(duration.get() > 1000 && duration.get() < 2000); - } - finally { - server.stop(); - } + }); + future.get(3000, TimeUnit.MILLISECONDS); + assertTrue(duration.get() > 1000 && duration.get() < 2000); + } finally { + server.stop(); } + } - @Test - void testConfig() + @Test + void testConfig() { + TCPSender.Config config = new TCPSender.Config(); + assertEquals(1000, config.getWaitBeforeCloseMilli()); + // TODO: Add others later + } + + @Test + void validateConfig() { { - TCPSender.Config config = new TCPSender.Config(); - assertEquals(1000, config.getWaitBeforeCloseMilli()); - // TODO: Add others later + TCPSender.Config config = new TCPSender.Config(); + config.setConnectionTimeoutMilli(9); + assertThrows(IllegalArgumentException.class, () -> new TCPSender(config)); } - @Test - void validateConfig() { - { - TCPSender.Config config = new TCPSender.Config(); - config.setConnectionTimeoutMilli(9); - assertThrows(IllegalArgumentException.class, () -> new TCPSender(config)); - } - - { - TCPSender.Config config = new TCPSender.Config(); - config.setReadTimeoutMilli(9); - assertThrows(IllegalArgumentException.class, () -> new TCPSender(config)); - } - - { - TCPSender.Config config = new TCPSender.Config(); - config.setWaitBeforeCloseMilli(-1); - assertThrows(IllegalArgumentException.class, () -> new TCPSender(config)); - } + TCPSender.Config config = new TCPSender.Config(); + config.setReadTimeoutMilli(9); + assertThrows(IllegalArgumentException.class, () -> new TCPSender(config)); } - interface TCPSenderCreater { - TCPSender create(int port); + TCPSender.Config config = new TCPSender.Config(); + config.setWaitBeforeCloseMilli(-1); + assertThrows(IllegalArgumentException.class, () -> new TCPSender(config)); } -} \ No newline at end of file + } + + interface TCPSenderCreater { + TCPSender create(int port); + } +} diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/FailureDetectorTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/FailureDetectorTest.java index 57344fa8..da276c2a 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/FailureDetectorTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/FailureDetectorTest.java @@ -16,11 +16,10 @@ package org.komamitsu.fluency.fluentd.ingester.sender.failuredetect; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.Heartbeater; -import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.TCPHeartbeater; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; import java.io.IOException; import java.nio.channels.ServerSocketChannel; @@ -28,77 +27,74 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.Heartbeater; +import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.TCPHeartbeater; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; - -class FailureDetectorTest -{ - private static final Logger LOG = LoggerFactory.getLogger(FailureDetectorTest.class); +class FailureDetectorTest { + private static final Logger LOG = LoggerFactory.getLogger(FailureDetectorTest.class); - @Test - void testIsAvailable() - throws IOException, InterruptedException - { - final ServerSocketChannel serverSocketChannel = ServerSocketChannel.open(); - serverSocketChannel.socket().bind(null); - int localPort = serverSocketChannel.socket().getLocalPort(); + @Test + void testIsAvailable() throws IOException, InterruptedException { + final ServerSocketChannel serverSocketChannel = ServerSocketChannel.open(); + serverSocketChannel.socket().bind(null); + int localPort = serverSocketChannel.socket().getLocalPort(); - final ExecutorService executorService = Executors.newSingleThreadExecutor(); - Runnable serverTask = () -> { - while (!executorService.isShutdown()) { - try (SocketChannel accept = serverSocketChannel.accept()) { - LOG.debug("Accepted: {}", accept); - } - catch (IOException e) { - LOG.warn("Stab TCP server got an error", e); - } - } - try { - serverSocketChannel.close(); - } - catch (IOException e) { - LOG.warn("Failed to close serverSocketChannel", e); + final ExecutorService executorService = Executors.newSingleThreadExecutor(); + Runnable serverTask = + () -> { + while (!executorService.isShutdown()) { + try (SocketChannel accept = serverSocketChannel.accept()) { + LOG.debug("Accepted: {}", accept); + } catch (IOException e) { + LOG.warn("Stab TCP server got an error", e); } + } + try { + serverSocketChannel.close(); + } catch (IOException e) { + LOG.warn("Failed to close serverSocketChannel", e); + } }; - executorService.execute(serverTask); + executorService.execute(serverTask); - TCPHeartbeater.Config heartbeaterConfig = new TCPHeartbeater.Config(); - heartbeaterConfig.setPort(localPort); + TCPHeartbeater.Config heartbeaterConfig = new TCPHeartbeater.Config(); + heartbeaterConfig.setPort(localPort); - PhiAccrualFailureDetectStrategy.Config failureDetectorConfig = new PhiAccrualFailureDetectStrategy.Config(); - try (FailureDetector failureDetector = new FailureDetector( - new PhiAccrualFailureDetectStrategy(failureDetectorConfig), - new TCPHeartbeater(heartbeaterConfig))) { + PhiAccrualFailureDetectStrategy.Config failureDetectorConfig = + new PhiAccrualFailureDetectStrategy.Config(); + try (FailureDetector failureDetector = + new FailureDetector( + new PhiAccrualFailureDetectStrategy(failureDetectorConfig), + new TCPHeartbeater(heartbeaterConfig))) { - assertTrue(failureDetector.isAvailable()); - TimeUnit.SECONDS.sleep(4); - assertTrue(failureDetector.isAvailable()); + assertTrue(failureDetector.isAvailable()); + TimeUnit.SECONDS.sleep(4); + assertTrue(failureDetector.isAvailable()); - executorService.shutdownNow(); - for (int i = 0; i < 20; i++) { - if (!failureDetector.isAvailable()) { - break; - } - TimeUnit.MILLISECONDS.sleep(500); - } - assertFalse(failureDetector.isAvailable()); + executorService.shutdownNow(); + for (int i = 0; i < 20; i++) { + if (!failureDetector.isAvailable()) { + break; } + TimeUnit.MILLISECONDS.sleep(500); + } + assertFalse(failureDetector.isAvailable()); } + } - @Test - void validateConfig() - { - FailureDetector.Config config = new FailureDetector.Config(); - config.setFailureIntervalMillis(-1); + @Test + void validateConfig() { + FailureDetector.Config config = new FailureDetector.Config(); + config.setFailureIntervalMillis(-1); - assertThrows(IllegalArgumentException.class, () -> new FailureDetector( - mock(FailureDetectStrategy.class), - mock(Heartbeater.class), - config - )); - } -} \ No newline at end of file + assertThrows( + IllegalArgumentException.class, + () -> + new FailureDetector( + mock(FailureDetectStrategy.class), mock(Heartbeater.class), config)); + } +} diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/PhiAccrualFailureDetectStrategyTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/PhiAccrualFailureDetectStrategyTest.java index 0b9d3fde..0b4f06dc 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/PhiAccrualFailureDetectStrategyTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/PhiAccrualFailureDetectStrategyTest.java @@ -16,25 +16,25 @@ package org.komamitsu.fluency.fluentd.ingester.sender.failuredetect; -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.*; -class PhiAccrualFailureDetectStrategyTest -{ - @Test - void validateConfig() +import org.junit.jupiter.api.Test; + +class PhiAccrualFailureDetectStrategyTest { + @Test + void validateConfig() { { - { - PhiAccrualFailureDetectStrategy.Config config = new PhiAccrualFailureDetectStrategy.Config(); - config.setPhiThreshold(0); - assertThrows(IllegalArgumentException.class, () -> new PhiAccrualFailureDetectStrategy(config)); - } + PhiAccrualFailureDetectStrategy.Config config = new PhiAccrualFailureDetectStrategy.Config(); + config.setPhiThreshold(0); + assertThrows( + IllegalArgumentException.class, () -> new PhiAccrualFailureDetectStrategy(config)); + } - { - PhiAccrualFailureDetectStrategy.Config config = new PhiAccrualFailureDetectStrategy.Config(); - config.setArrivalWindowSize(0); - assertThrows(IllegalArgumentException.class, () -> new PhiAccrualFailureDetectStrategy(config)); - } + { + PhiAccrualFailureDetectStrategy.Config config = new PhiAccrualFailureDetectStrategy.Config(); + config.setArrivalWindowSize(0); + assertThrows( + IllegalArgumentException.class, () -> new PhiAccrualFailureDetectStrategy(config)); } -} \ No newline at end of file + } +} diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/HeartbeaterTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/HeartbeaterTest.java index 99546d66..922c8721 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/HeartbeaterTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/HeartbeaterTest.java @@ -16,91 +16,77 @@ package org.komamitsu.fluency.fluentd.ingester.sender.heartbeat; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.util.Tuple3; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -public class HeartbeaterTest -{ - private TestableHeartbeater.Config config; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.util.Tuple3; - @BeforeEach - public void setup() - { - config = new TestableHeartbeater.Config(); - config.setHost("dummy-hostname"); - config.setPort(123456); - config.setIntervalMillis(300); - } +public class HeartbeaterTest { + private TestableHeartbeater.Config config; - @Test - public void testHeartbeater() - throws InterruptedException - { - try (TestableHeartbeater heartbeater = new TestableHeartbeater(config)) { - heartbeater.start(); - TimeUnit.SECONDS.sleep(1); - heartbeater.close(); - TimeUnit.MILLISECONDS.sleep(500); - assertTrue(1 < heartbeater.pongedRecords.size() && heartbeater.pongedRecords.size() < 4); - Tuple3 firstPong = heartbeater.pongedRecords.get(0); - Tuple3 secondPong = heartbeater.pongedRecords.get(1); - long diff = secondPong.getFirst() - firstPong.getFirst(); - assertTrue(100 < diff && diff < 1000); - for (Tuple3 pongedRecord : heartbeater.pongedRecords) { - assertEquals("dummy-hostname", pongedRecord.getSecond()); - assertEquals((Integer) 123456, pongedRecord.getThird()); - } - } - } + @BeforeEach + public void setup() { + config = new TestableHeartbeater.Config(); + config.setHost("dummy-hostname"); + config.setPort(123456); + config.setIntervalMillis(300); + } - @Test - void validateConfig() - { - TestableHeartbeater.Config invalidConfig = new TestableHeartbeater.Config(); - invalidConfig.setIntervalMillis(99); - assertThrows(IllegalArgumentException.class, () -> new TestableHeartbeater(invalidConfig)); + @Test + public void testHeartbeater() throws InterruptedException { + try (TestableHeartbeater heartbeater = new TestableHeartbeater(config)) { + heartbeater.start(); + TimeUnit.SECONDS.sleep(1); + heartbeater.close(); + TimeUnit.MILLISECONDS.sleep(500); + assertTrue(1 < heartbeater.pongedRecords.size() && heartbeater.pongedRecords.size() < 4); + Tuple3 firstPong = heartbeater.pongedRecords.get(0); + Tuple3 secondPong = heartbeater.pongedRecords.get(1); + long diff = secondPong.getFirst() - firstPong.getFirst(); + assertTrue(100 < diff && diff < 1000); + for (Tuple3 pongedRecord : heartbeater.pongedRecords) { + assertEquals("dummy-hostname", pongedRecord.getSecond()); + assertEquals((Integer) 123456, pongedRecord.getThird()); + } } + } - private static class TestableHeartbeater - extends InetSocketHeartbeater - { - private final Config config; - private List> pongedRecords = new ArrayList<>(); + @Test + void validateConfig() { + TestableHeartbeater.Config invalidConfig = new TestableHeartbeater.Config(); + invalidConfig.setIntervalMillis(99); + assertThrows(IllegalArgumentException.class, () -> new TestableHeartbeater(invalidConfig)); + } - public TestableHeartbeater() - { - this(new Config()); - } + private static class TestableHeartbeater extends InetSocketHeartbeater { + private final Config config; + private List> pongedRecords = new ArrayList<>(); - public TestableHeartbeater(Config config) - { - super(config); - this.config = config; - } + public TestableHeartbeater() { + this(new Config()); + } - public List> getPongedRecords() - { - return pongedRecords; - } + public TestableHeartbeater(Config config) { + super(config); + this.config = config; + } - @Override - protected void invoke() - { - pongedRecords.add(new Tuple3<>(System.currentTimeMillis(), config.getHost(), config.getPort())); - } + public List> getPongedRecords() { + return pongedRecords; + } - private static class Config - extends InetSocketHeartbeater.Config - { - } + @Override + protected void invoke() { + pongedRecords.add( + new Tuple3<>(System.currentTimeMillis(), config.getHost(), config.getPort())); } -} \ No newline at end of file + + private static class Config extends InetSocketHeartbeater.Config {} + } +} diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/SSLHeartbeaterTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/SSLHeartbeaterTest.java index ea324a15..30080194 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/SSLHeartbeaterTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/SSLHeartbeaterTest.java @@ -16,12 +16,9 @@ package org.komamitsu.fluency.fluentd.ingester.sender.heartbeat; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.fluentd.SSLTestSocketFactories; - -import javax.net.ssl.SSLServerSocket; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.security.KeyManagementException; @@ -33,136 +30,124 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import javax.net.ssl.SSLServerSocket; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.fluentd.SSLTestSocketFactories; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class SSLHeartbeaterTest -{ - private SSLServerSocket sslServerSocket; - private SSLHeartbeater heartbeater; +public class SSLHeartbeaterTest { + private SSLServerSocket sslServerSocket; + private SSLHeartbeater heartbeater; + + @BeforeEach + void setUp() + throws CertificateException, UnrecoverableKeyException, NoSuchAlgorithmException, IOException, + KeyManagementException, KeyStoreException { + sslServerSocket = SSLTestSocketFactories.createServerSocket(); + + SSLHeartbeater.Config config = new SSLHeartbeater.Config(); + config.setPort(sslServerSocket.getLocalPort()); + config.setIntervalMillis(500); + heartbeater = new SSLHeartbeater(config); + } + + @AfterEach + void tearDown() throws IOException { + if (sslServerSocket != null && !sslServerSocket.isClosed()) { + sslServerSocket.close(); + } + } + + @Test + void testHeartbeaterUp() throws IOException, InterruptedException { + final CountDownLatch latch = new CountDownLatch(2); + Executors.newSingleThreadExecutor() + .execute( + () -> { + try { + sslServerSocket.accept(); + latch.countDown(); + } catch (IOException e) { + e.printStackTrace(); + } + }); - @BeforeEach - void setUp() - throws CertificateException, UnrecoverableKeyException, NoSuchAlgorithmException, IOException, KeyManagementException, KeyStoreException - { - sslServerSocket = SSLTestSocketFactories.createServerSocket(); + final AtomicInteger pongCounter = new AtomicInteger(); + final AtomicInteger failureCounter = new AtomicInteger(); + try { + heartbeater.setCallback( + new Heartbeater.Callback() { + @Override + public void onHeartbeat() { + pongCounter.incrementAndGet(); + latch.countDown(); + } - SSLHeartbeater.Config config = new SSLHeartbeater.Config(); - config.setPort(sslServerSocket.getLocalPort()); - config.setIntervalMillis(500); - heartbeater = new SSLHeartbeater(config); + @Override + public void onFailure(Throwable cause) { + failureCounter.incrementAndGet(); + } + }); + heartbeater.start(); + assertTrue(latch.await(10, TimeUnit.SECONDS)); + assertTrue(0 < pongCounter.get() && pongCounter.get() < 3); + assertEquals(0, failureCounter.get()); + } finally { + if (heartbeater != null) { + heartbeater.close(); + } } + } + + @Test + public void testHeartbeaterDown() throws IOException, InterruptedException { + sslServerSocket.close(); + + final AtomicInteger pongCounter = new AtomicInteger(); + final AtomicInteger failureCounter = new AtomicInteger(); + try { + heartbeater.setCallback( + new Heartbeater.Callback() { + @Override + public void onHeartbeat() { + pongCounter.incrementAndGet(); + } - @AfterEach - void tearDown() - throws IOException - { - if (sslServerSocket != null && !sslServerSocket.isClosed()) { - sslServerSocket.close(); - } + @Override + public void onFailure(Throwable cause) { + failureCounter.incrementAndGet(); + } + }); + heartbeater.start(); + TimeUnit.SECONDS.sleep(1); + assertEquals(0, pongCounter.get()); + assertTrue(failureCounter.get() > 0); + } finally { + if (heartbeater != null) { + heartbeater.close(); + } } + } - @Test - void testHeartbeaterUp() - throws IOException, InterruptedException + @Test + void validateConfig() { { - final CountDownLatch latch = new CountDownLatch(2); - Executors.newSingleThreadExecutor().execute(() -> { - try { - sslServerSocket.accept(); - latch.countDown(); - } - catch (IOException e) { - e.printStackTrace(); - } - }); - - final AtomicInteger pongCounter = new AtomicInteger(); - final AtomicInteger failureCounter = new AtomicInteger(); - try { - heartbeater.setCallback(new Heartbeater.Callback() - { - @Override - public void onHeartbeat() - { - pongCounter.incrementAndGet(); - latch.countDown(); - } - - @Override - public void onFailure(Throwable cause) - { - failureCounter.incrementAndGet(); - } - }); - heartbeater.start(); - assertTrue(latch.await(10, TimeUnit.SECONDS)); - assertTrue(0 < pongCounter.get() && pongCounter.get() < 3); - assertEquals(0, failureCounter.get()); - } - finally { - if (heartbeater != null) { - heartbeater.close(); - } - } + SSLHeartbeater.Config config = new SSLHeartbeater.Config(); + config.setIntervalMillis(99); + assertThrows(IllegalArgumentException.class, () -> new SSLHeartbeater(config)); } - @Test - public void testHeartbeaterDown() - throws IOException, InterruptedException { - sslServerSocket.close(); - - final AtomicInteger pongCounter = new AtomicInteger(); - final AtomicInteger failureCounter = new AtomicInteger(); - try { - heartbeater.setCallback(new Heartbeater.Callback() - { - @Override - public void onHeartbeat() - { - pongCounter.incrementAndGet(); - } - - @Override - public void onFailure(Throwable cause) - { - failureCounter.incrementAndGet(); - } - }); - heartbeater.start(); - TimeUnit.SECONDS.sleep(1); - assertEquals(0, pongCounter.get()); - assertTrue(failureCounter.get() > 0); - } - finally { - if (heartbeater != null) { - heartbeater.close(); - } - } + SSLHeartbeater.Config config = new SSLHeartbeater.Config(); + config.setConnectionTimeoutMilli(9); + assertThrows(IllegalArgumentException.class, () -> new SSLHeartbeater(config)); } - @Test - void validateConfig() { - { - SSLHeartbeater.Config config = new SSLHeartbeater.Config(); - config.setIntervalMillis(99); - assertThrows(IllegalArgumentException.class, () -> new SSLHeartbeater(config)); - } - - { - SSLHeartbeater.Config config = new SSLHeartbeater.Config(); - config.setConnectionTimeoutMilli(9); - assertThrows(IllegalArgumentException.class, () -> new SSLHeartbeater(config)); - } - - { - SSLHeartbeater.Config config = new SSLHeartbeater.Config(); - config.setReadTimeoutMilli(9); - assertThrows(IllegalArgumentException.class, () -> new SSLHeartbeater(config)); - } + SSLHeartbeater.Config config = new SSLHeartbeater.Config(); + config.setReadTimeoutMilli(9); + assertThrows(IllegalArgumentException.class, () -> new SSLHeartbeater(config)); } -} \ No newline at end of file + } +} diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/TCPHeartbeaterTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/TCPHeartbeaterTest.java index a121860f..111f8abc 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/TCPHeartbeaterTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/TCPHeartbeaterTest.java @@ -16,7 +16,8 @@ package org.komamitsu.fluency.fluentd.ingester.sender.heartbeat; -import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.nio.channels.ServerSocketChannel; @@ -24,90 +25,80 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class TCPHeartbeaterTest -{ - @Test - void testTCPHeartbeaterUp() - throws IOException, InterruptedException - { - final CountDownLatch latch = new CountDownLatch(2); - final ServerSocketChannel serverSocketChannel = ServerSocketChannel.open(); - serverSocketChannel.socket().bind(null); - Executors.newSingleThreadExecutor().execute(() -> { - try { +class TCPHeartbeaterTest { + @Test + void testTCPHeartbeaterUp() throws IOException, InterruptedException { + final CountDownLatch latch = new CountDownLatch(2); + final ServerSocketChannel serverSocketChannel = ServerSocketChannel.open(); + serverSocketChannel.socket().bind(null); + Executors.newSingleThreadExecutor() + .execute( + () -> { + try { serverSocketChannel.accept(); latch.countDown(); - } - catch (IOException e) { + } catch (IOException e) { e.printStackTrace(); - } - }); + } + }); - TCPHeartbeater.Config config = new TCPHeartbeater.Config(); - config.setPort(serverSocketChannel.socket().getLocalPort()); - config.setIntervalMillis(500); - try (TCPHeartbeater heartbeater = new TCPHeartbeater(config)) { - final AtomicInteger pongCounter = new AtomicInteger(); - final AtomicInteger failureCounter = new AtomicInteger(); - heartbeater.setCallback(new Heartbeater.Callback() - { - @Override - public void onHeartbeat() - { - pongCounter.incrementAndGet(); - latch.countDown(); - } + TCPHeartbeater.Config config = new TCPHeartbeater.Config(); + config.setPort(serverSocketChannel.socket().getLocalPort()); + config.setIntervalMillis(500); + try (TCPHeartbeater heartbeater = new TCPHeartbeater(config)) { + final AtomicInteger pongCounter = new AtomicInteger(); + final AtomicInteger failureCounter = new AtomicInteger(); + heartbeater.setCallback( + new Heartbeater.Callback() { + @Override + public void onHeartbeat() { + pongCounter.incrementAndGet(); + latch.countDown(); + } - @Override - public void onFailure(Throwable cause) - { - failureCounter.incrementAndGet(); - } - }); - heartbeater.start(); - assertTrue(latch.await(5, TimeUnit.SECONDS)); - assertTrue(0 < pongCounter.get() && pongCounter.get() < 3); - assertEquals(0, failureCounter.get()); - } + @Override + public void onFailure(Throwable cause) { + failureCounter.incrementAndGet(); + } + }); + heartbeater.start(); + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(0 < pongCounter.get() && pongCounter.get() < 3); + assertEquals(0, failureCounter.get()); } + } - @Test - void testTCPHeartbeaterDown() - throws IOException, InterruptedException - { - final ServerSocketChannel serverSocketChannel = ServerSocketChannel.open(); - serverSocketChannel.socket().bind(null); - int localPort = serverSocketChannel.socket().getLocalPort(); - serverSocketChannel.close(); + @Test + void testTCPHeartbeaterDown() throws IOException, InterruptedException { + final ServerSocketChannel serverSocketChannel = ServerSocketChannel.open(); + serverSocketChannel.socket().bind(null); + int localPort = serverSocketChannel.socket().getLocalPort(); + serverSocketChannel.close(); - TCPHeartbeater.Config config = new TCPHeartbeater.Config(); - config.setPort(localPort); - config.setIntervalMillis(500); - try (TCPHeartbeater heartbeater = new TCPHeartbeater(config)) { - final AtomicInteger pongCounter = new AtomicInteger(); - final AtomicInteger failureCounter = new AtomicInteger(); - heartbeater.setCallback(new Heartbeater.Callback() - { - @Override - public void onHeartbeat() - { - pongCounter.incrementAndGet(); - } + TCPHeartbeater.Config config = new TCPHeartbeater.Config(); + config.setPort(localPort); + config.setIntervalMillis(500); + try (TCPHeartbeater heartbeater = new TCPHeartbeater(config)) { + final AtomicInteger pongCounter = new AtomicInteger(); + final AtomicInteger failureCounter = new AtomicInteger(); + heartbeater.setCallback( + new Heartbeater.Callback() { + @Override + public void onHeartbeat() { + pongCounter.incrementAndGet(); + } - @Override - public void onFailure(Throwable cause) - { - failureCounter.incrementAndGet(); - } - }); - heartbeater.start(); - TimeUnit.SECONDS.sleep(1); - assertEquals(0, pongCounter.get()); - assertTrue(failureCounter.get() > 0); - } + @Override + public void onFailure(Throwable cause) { + failureCounter.incrementAndGet(); + } + }); + heartbeater.start(); + TimeUnit.SECONDS.sleep(1); + assertEquals(0, pongCounter.get()); + assertTrue(failureCounter.get() > 0); } + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UDPHeartbeaterTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UDPHeartbeaterTest.java index 1e299bdf..5d92e322 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UDPHeartbeaterTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UDPHeartbeaterTest.java @@ -16,7 +16,8 @@ package org.komamitsu.fluency.fluentd.ingester.sender.heartbeat; -import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.net.SocketAddress; @@ -26,89 +27,78 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class UDPHeartbeaterTest -{ - @Test - void testUDPHeartbeaterUp() - throws IOException, InterruptedException - { - final CountDownLatch latch = new CountDownLatch(2); - final DatagramChannel datagramChannel = DatagramChannel.open(); - datagramChannel.socket().bind(null); - Executors.newSingleThreadExecutor().execute(() -> { - try { +class UDPHeartbeaterTest { + @Test + void testUDPHeartbeaterUp() throws IOException, InterruptedException { + final CountDownLatch latch = new CountDownLatch(2); + final DatagramChannel datagramChannel = DatagramChannel.open(); + datagramChannel.socket().bind(null); + Executors.newSingleThreadExecutor() + .execute( + () -> { + try { ByteBuffer byteBuffer = ByteBuffer.allocate(8); SocketAddress socketAddress = datagramChannel.receive(byteBuffer); datagramChannel.send(ByteBuffer.allocate(0), socketAddress); latch.countDown(); - } - catch (IOException e) { + } catch (IOException e) { e.printStackTrace(); - } - }); + } + }); - UDPHeartbeater.Config config = new UDPHeartbeater.Config(); - config.setPort(datagramChannel.socket().getLocalPort()); - config.setIntervalMillis(500); - try (UDPHeartbeater heartbeater = new UDPHeartbeater(config)) { - final AtomicInteger pongCounter = new AtomicInteger(); - final AtomicInteger failureCounter = new AtomicInteger(); - heartbeater.setCallback(new Heartbeater.Callback() - { - @Override - public void onHeartbeat() - { - pongCounter.incrementAndGet(); - latch.countDown(); - } + UDPHeartbeater.Config config = new UDPHeartbeater.Config(); + config.setPort(datagramChannel.socket().getLocalPort()); + config.setIntervalMillis(500); + try (UDPHeartbeater heartbeater = new UDPHeartbeater(config)) { + final AtomicInteger pongCounter = new AtomicInteger(); + final AtomicInteger failureCounter = new AtomicInteger(); + heartbeater.setCallback( + new Heartbeater.Callback() { + @Override + public void onHeartbeat() { + pongCounter.incrementAndGet(); + latch.countDown(); + } - @Override - public void onFailure(Throwable cause) - { - failureCounter.incrementAndGet(); - } - }); - heartbeater.start(); - assertTrue(latch.await(5, TimeUnit.SECONDS)); - assertTrue(0 < pongCounter.get() && pongCounter.get() < 3); - assertEquals(0, failureCounter.get()); - } + @Override + public void onFailure(Throwable cause) { + failureCounter.incrementAndGet(); + } + }); + heartbeater.start(); + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(0 < pongCounter.get() && pongCounter.get() < 3); + assertEquals(0, failureCounter.get()); } + } - @Test - void testUDPHeartbeaterDown() - throws IOException, InterruptedException - { - final DatagramChannel datagramChannel = DatagramChannel.open(); - datagramChannel.socket().bind(null); - int localPort = datagramChannel.socket().getLocalPort(); - datagramChannel.close(); + @Test + void testUDPHeartbeaterDown() throws IOException, InterruptedException { + final DatagramChannel datagramChannel = DatagramChannel.open(); + datagramChannel.socket().bind(null); + int localPort = datagramChannel.socket().getLocalPort(); + datagramChannel.close(); - UDPHeartbeater.Config config = new UDPHeartbeater.Config(); - config.setPort(localPort); - config.setIntervalMillis(500); - try (UDPHeartbeater heartbeater = new UDPHeartbeater(config)) { - final AtomicInteger pongCounter = new AtomicInteger(); - heartbeater.setCallback(new Heartbeater.Callback() - { - @Override - public void onHeartbeat() - { - pongCounter.incrementAndGet(); - } + UDPHeartbeater.Config config = new UDPHeartbeater.Config(); + config.setPort(localPort); + config.setIntervalMillis(500); + try (UDPHeartbeater heartbeater = new UDPHeartbeater(config)) { + final AtomicInteger pongCounter = new AtomicInteger(); + heartbeater.setCallback( + new Heartbeater.Callback() { + @Override + public void onHeartbeat() { + pongCounter.incrementAndGet(); + } - @Override - public void onFailure(Throwable cause) - { - } - }); - heartbeater.start(); - TimeUnit.SECONDS.sleep(1); - assertEquals(0, pongCounter.get()); - } + @Override + public void onFailure(Throwable cause) {} + }); + heartbeater.start(); + TimeUnit.SECONDS.sleep(1); + assertEquals(0, pongCounter.get()); } + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ConstantRetryStrategyTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ConstantRetryStrategyTest.java index 99d1c38c..51d7975e 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ConstantRetryStrategyTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ConstantRetryStrategyTest.java @@ -16,38 +16,34 @@ package org.komamitsu.fluency.fluentd.ingester.sender.retry; - -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -class ConstantRetryStrategyTest -{ - @Test - void testGetNextIntervalMillis() - { - ConstantRetryStrategy.Config config = new ConstantRetryStrategy.Config(); - config.setRetryIntervalMillis(600); - config.setMaxRetryCount(6); - RetryStrategy strategy = new ConstantRetryStrategy(config); +import org.junit.jupiter.api.Test; + +class ConstantRetryStrategyTest { + @Test + void testGetNextIntervalMillis() { + ConstantRetryStrategy.Config config = new ConstantRetryStrategy.Config(); + config.setRetryIntervalMillis(600); + config.setMaxRetryCount(6); + RetryStrategy strategy = new ConstantRetryStrategy(config); - assertEquals(600, strategy.getNextIntervalMillis(0)); - assertEquals(600, strategy.getNextIntervalMillis(1)); - assertEquals(600, strategy.getNextIntervalMillis(2)); - assertEquals(600, strategy.getNextIntervalMillis(6)); - assertFalse(strategy.isRetriedOver(6)); - assertEquals(600, strategy.getNextIntervalMillis(7)); - assertTrue(strategy.isRetriedOver(7)); - } + assertEquals(600, strategy.getNextIntervalMillis(0)); + assertEquals(600, strategy.getNextIntervalMillis(1)); + assertEquals(600, strategy.getNextIntervalMillis(2)); + assertEquals(600, strategy.getNextIntervalMillis(6)); + assertFalse(strategy.isRetriedOver(6)); + assertEquals(600, strategy.getNextIntervalMillis(7)); + assertTrue(strategy.isRetriedOver(7)); + } - @Test - void validateConfig() - { - ConstantRetryStrategy.Config config = new ConstantRetryStrategy.Config(); - config.setRetryIntervalMillis(9); - assertThrows(IllegalArgumentException.class, () -> new ConstantRetryStrategy(config)); - } -} \ No newline at end of file + @Test + void validateConfig() { + ConstantRetryStrategy.Config config = new ConstantRetryStrategy.Config(); + config.setRetryIntervalMillis(9); + assertThrows(IllegalArgumentException.class, () -> new ConstantRetryStrategy(config)); + } +} diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ExponentialBackOffRetryStrategyTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ExponentialBackOffRetryStrategyTest.java index 9aa1fdc0..9be162f8 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ExponentialBackOffRetryStrategyTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ExponentialBackOffRetryStrategyTest.java @@ -16,54 +16,54 @@ package org.komamitsu.fluency.fluentd.ingester.sender.retry; -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -class ExponentialBackOffRetryStrategyTest -{ - @Test - void testGetNextIntervalMillis() - { - ExponentialBackOffRetryStrategy.Config config = new ExponentialBackOffRetryStrategy.Config(); - config.setBaseIntervalMillis(400); - config.setMaxIntervalMillis(30000); - config.setMaxRetryCount(7); - RetryStrategy strategy = new ExponentialBackOffRetryStrategy(config); +import org.junit.jupiter.api.Test; - assertEquals(400, strategy.getNextIntervalMillis(0)); - assertEquals(800, strategy.getNextIntervalMillis(1)); - assertEquals(1600, strategy.getNextIntervalMillis(2)); - assertEquals(3200, strategy.getNextIntervalMillis(3)); - assertEquals(25600, strategy.getNextIntervalMillis(6)); - assertEquals(30000, strategy.getNextIntervalMillis(7)); - assertFalse(strategy.isRetriedOver(7)); - assertEquals(30000, strategy.getNextIntervalMillis(8)); - assertTrue(strategy.isRetriedOver(8)); - } +class ExponentialBackOffRetryStrategyTest { + @Test + void testGetNextIntervalMillis() { + ExponentialBackOffRetryStrategy.Config config = new ExponentialBackOffRetryStrategy.Config(); + config.setBaseIntervalMillis(400); + config.setMaxIntervalMillis(30000); + config.setMaxRetryCount(7); + RetryStrategy strategy = new ExponentialBackOffRetryStrategy(config); - @Test - void validateConfig() + assertEquals(400, strategy.getNextIntervalMillis(0)); + assertEquals(800, strategy.getNextIntervalMillis(1)); + assertEquals(1600, strategy.getNextIntervalMillis(2)); + assertEquals(3200, strategy.getNextIntervalMillis(3)); + assertEquals(25600, strategy.getNextIntervalMillis(6)); + assertEquals(30000, strategy.getNextIntervalMillis(7)); + assertFalse(strategy.isRetriedOver(7)); + assertEquals(30000, strategy.getNextIntervalMillis(8)); + assertTrue(strategy.isRetriedOver(8)); + } + + @Test + void validateConfig() { { - { - ExponentialBackOffRetryStrategy.Config config = new ExponentialBackOffRetryStrategy.Config(); - config.setMaxRetryCount(-1); - assertThrows(IllegalArgumentException.class, () -> new ExponentialBackOffRetryStrategy(config)); - } + ExponentialBackOffRetryStrategy.Config config = new ExponentialBackOffRetryStrategy.Config(); + config.setMaxRetryCount(-1); + assertThrows( + IllegalArgumentException.class, () -> new ExponentialBackOffRetryStrategy(config)); + } - { - ExponentialBackOffRetryStrategy.Config config = new ExponentialBackOffRetryStrategy.Config(); - config.setBaseIntervalMillis(9); - assertThrows(IllegalArgumentException.class, () -> new ExponentialBackOffRetryStrategy(config)); - } + { + ExponentialBackOffRetryStrategy.Config config = new ExponentialBackOffRetryStrategy.Config(); + config.setBaseIntervalMillis(9); + assertThrows( + IllegalArgumentException.class, () -> new ExponentialBackOffRetryStrategy(config)); + } - { - ExponentialBackOffRetryStrategy.Config config = new ExponentialBackOffRetryStrategy.Config(); - config.setMaxIntervalMillis(9); - assertThrows(IllegalArgumentException.class, () -> new ExponentialBackOffRetryStrategy(config)); - } + { + ExponentialBackOffRetryStrategy.Config config = new ExponentialBackOffRetryStrategy.Config(); + config.setMaxIntervalMillis(9); + assertThrows( + IllegalArgumentException.class, () -> new ExponentialBackOffRetryStrategy(config)); } -} \ No newline at end of file + } +} diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/integration/WithRealFluentd.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/integration/WithRealFluentd.java index e832bb54..b957563a 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/integration/WithRealFluentd.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/integration/WithRealFluentd.java @@ -16,12 +16,10 @@ package org.komamitsu.fluency.integration; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.Fluency; -import org.komamitsu.fluency.fluentd.FluencyBuilderForFluentd; - import java.io.IOException; import java.net.InetSocketAddress; import java.util.ArrayList; @@ -36,203 +34,191 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.Fluency; +import org.komamitsu.fluency.fluentd.FluencyBuilderForFluentd; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -class WithRealFluentd -{ - private final ObjectMapper objectMapper = new ObjectMapper(); +class WithRealFluentd { + private final ObjectMapper objectMapper = new ObjectMapper(); - WithRealFluentd.Config getConfig() - throws IOException - { - String conf = System.getenv("WITH_FLUENTD"); - if (conf == null) { - return null; - } - - return objectMapper.readValue(conf, WithRealFluentd.Config.class); + WithRealFluentd.Config getConfig() throws IOException { + String conf = System.getenv("WITH_FLUENTD"); + if (conf == null) { + return null; } - @Test - void testWithRealFluentd() - throws Exception - { - WithRealFluentd.Config config = getConfig(); - assumeTrue(config != null); - - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSslEnabled(config.sslEnabled); - - try (Fluency fluency = builder.build(config.host, config.port)) { - Map data = new HashMap<>(); - data.put("name", "komamitsu"); - data.put("age", 42); - data.put("comment", "hello, world"); - ExecutorService executorService = Executors.newCachedThreadPool(); - List> futures = new ArrayList<>(); - for (int i = 0; i < config.concurrency; i++) { - futures.add(executorService.submit(new EmitTask(fluency, config.tag, data, config.requests))); - } - for (Future future : futures) { - future.get(config.waitSeconds, TimeUnit.SECONDS); - } - } + return objectMapper.readValue(conf, WithRealFluentd.Config.class); + } + + @Test + void testWithRealFluentd() throws Exception { + WithRealFluentd.Config config = getConfig(); + assumeTrue(config != null); + + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSslEnabled(config.sslEnabled); + + try (Fluency fluency = builder.build(config.host, config.port)) { + Map data = new HashMap<>(); + data.put("name", "komamitsu"); + data.put("age", 42); + data.put("comment", "hello, world"); + ExecutorService executorService = Executors.newCachedThreadPool(); + List> futures = new ArrayList<>(); + for (int i = 0; i < config.concurrency; i++) { + futures.add( + executorService.submit(new EmitTask(fluency, config.tag, data, config.requests))); + } + for (Future future : futures) { + future.get(config.waitSeconds, TimeUnit.SECONDS); + } } - - @Test - void testWithRealMultipleFluentd() - throws IOException, InterruptedException, TimeoutException, ExecutionException - { - WithRealFluentd.Config config = getConfig(); - assumeTrue(config != null); - assumeTrue(config.anotherPort != null); - - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSslEnabled(config.sslEnabled); - builder.setAckResponseMode(true); - - try (Fluency fluency = builder.build( - Arrays.asList( - new InetSocketAddress(config.host, config.port), - new InetSocketAddress(config.host, config.anotherPort)))) { - - Map data = new HashMap<>(); - data.put("name", "komamitsu"); - data.put("age", 42); - data.put("comment", "hello, world"); - ExecutorService executorService = Executors.newCachedThreadPool(); - List> futures = new ArrayList<>(); - for (int i = 0; i < config.concurrency; i++) { - futures.add(executorService.submit(new EmitTask(fluency, config.tag, data, config.requests))); - } - for (Future future : futures) { - future.get(config.waitSeconds, TimeUnit.SECONDS); - } - } + } + + @Test + void testWithRealMultipleFluentd() + throws IOException, InterruptedException, TimeoutException, ExecutionException { + WithRealFluentd.Config config = getConfig(); + assumeTrue(config != null); + assumeTrue(config.anotherPort != null); + + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSslEnabled(config.sslEnabled); + builder.setAckResponseMode(true); + + try (Fluency fluency = + builder.build( + Arrays.asList( + new InetSocketAddress(config.host, config.port), + new InetSocketAddress(config.host, config.anotherPort)))) { + + Map data = new HashMap<>(); + data.put("name", "komamitsu"); + data.put("age", 42); + data.put("comment", "hello, world"); + ExecutorService executorService = Executors.newCachedThreadPool(); + List> futures = new ArrayList<>(); + for (int i = 0; i < config.concurrency; i++) { + futures.add( + executorService.submit(new EmitTask(fluency, config.tag, data, config.requests))); + } + for (Future future : futures) { + future.get(config.waitSeconds, TimeUnit.SECONDS); + } } - - @Test - void testWithRealFluentdWithFileBackup() - throws ExecutionException, TimeoutException, IOException, InterruptedException - { - WithRealFluentd.Config config = getConfig(); - assumeTrue(config != null); - - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSslEnabled(config.sslEnabled); - // Fluency might use a lot of buffer for loaded backup files. - // So it'd better increase max buffer size - builder.setMaxBufferSize(512 * 1024 * 1024L); - builder.setFileBackupDir(System.getProperty("java.io.tmpdir")); - - try (Fluency fluency = new FluencyBuilderForFluentd().build(config.host, config.port)) { - Map data = new HashMap<>(); - data.put("name", "komamitsu"); - data.put("age", 42); - data.put("comment", "hello, world"); - ExecutorService executorService = Executors.newCachedThreadPool(); - List> futures = new ArrayList<>(); - for (int i = 0; i < config.concurrency; i++) { - futures.add(executorService.submit(new EmitTask(fluency, config.tag, data, config.requests))); - } - for (Future future : futures) { - future.get(config.waitSeconds, TimeUnit.SECONDS); - } - } + } + + @Test + void testWithRealFluentdWithFileBackup() + throws ExecutionException, TimeoutException, IOException, InterruptedException { + WithRealFluentd.Config config = getConfig(); + assumeTrue(config != null); + + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSslEnabled(config.sslEnabled); + // Fluency might use a lot of buffer for loaded backup files. + // So it'd better increase max buffer size + builder.setMaxBufferSize(512 * 1024 * 1024L); + builder.setFileBackupDir(System.getProperty("java.io.tmpdir")); + + try (Fluency fluency = new FluencyBuilderForFluentd().build(config.host, config.port)) { + Map data = new HashMap<>(); + data.put("name", "komamitsu"); + data.put("age", 42); + data.put("comment", "hello, world"); + ExecutorService executorService = Executors.newCachedThreadPool(); + List> futures = new ArrayList<>(); + for (int i = 0; i < config.concurrency; i++) { + futures.add( + executorService.submit(new EmitTask(fluency, config.tag, data, config.requests))); + } + for (Future future : futures) { + future.get(config.waitSeconds, TimeUnit.SECONDS); + } } - - public static class EmitTask - implements Callable - { - private final Fluency fluency; - private final String tag; - private final Map data; - private final int count; - - private EmitTask(Fluency fluency, String tag, Map data, int count) - { - this.fluency = fluency; - this.tag = tag; - this.data = data; - this.count = count; - } - - @Override - public Void call() - { - for (int i = 0; i < count; i++) { - try { - fluency.emit(tag, data); - } - catch (IOException e) { - e.printStackTrace(); - try { - TimeUnit.MILLISECONDS.sleep(500); - } - catch (InterruptedException e1) { - e1.printStackTrace(); - } - } - } - return null; - } + } + + public static class EmitTask implements Callable { + private final Fluency fluency; + private final String tag; + private final Map data; + private final int count; + + private EmitTask(Fluency fluency, String tag, Map data, int count) { + this.fluency = fluency; + this.tag = tag; + this.data = data; + this.count = count; } - public static class Config - { - @JsonProperty("host") - public final String host; - @JsonProperty("port") - public final Integer port; - @JsonProperty("another_host") - public final String anotherHost; - @JsonProperty("another_port") - public final Integer anotherPort; - @JsonProperty("tag") - public final String tag; - @JsonProperty("requests") - public final int requests; - @JsonProperty("concurrency") - public final int concurrency; - @JsonProperty("wait_seconds") - public final int waitSeconds; - @JsonProperty("ssl_enabled") - public final boolean sslEnabled; - - public Config( - @JsonProperty("host") - String host, - @JsonProperty("port") - Integer port, - @JsonProperty("another_host") - String anotherHost, - @JsonProperty("another_port") - Integer anotherPort, - @JsonProperty("tag") - String tag, - @JsonProperty("requests") - Integer requests, - @JsonProperty("concurrency") - Integer concurrency, - @JsonProperty("wait_seconds") - Integer waitSeconds, - @JsonProperty("ssl_enabled") - Boolean sslEnabled - ) - { - this.host = host == null ? "127.0.0.1" : host; - this.port = port == null ? Integer.valueOf(24224) : port; - - this.anotherHost = anotherHost == null ? "127.0.0.1" : anotherHost; - // Nullable - this.anotherPort = anotherPort; - - this.tag = tag == null ? "foodb.bartbl" : tag; - this.requests = requests == null ? 1000000 : requests; - this.concurrency = concurrency == null ? 4 : concurrency; - this.waitSeconds = waitSeconds == null ? 60 : waitSeconds; - this.sslEnabled = sslEnabled == null ? false : sslEnabled; + @Override + public Void call() { + for (int i = 0; i < count; i++) { + try { + fluency.emit(tag, data); + } catch (IOException e) { + e.printStackTrace(); + try { + TimeUnit.MILLISECONDS.sleep(500); + } catch (InterruptedException e1) { + e1.printStackTrace(); + } } + } + return null; + } + } + + public static class Config { + @JsonProperty("host") + public final String host; + + @JsonProperty("port") + public final Integer port; + + @JsonProperty("another_host") + public final String anotherHost; + + @JsonProperty("another_port") + public final Integer anotherPort; + + @JsonProperty("tag") + public final String tag; + + @JsonProperty("requests") + public final int requests; + + @JsonProperty("concurrency") + public final int concurrency; + + @JsonProperty("wait_seconds") + public final int waitSeconds; + + @JsonProperty("ssl_enabled") + public final boolean sslEnabled; + + public Config( + @JsonProperty("host") String host, + @JsonProperty("port") Integer port, + @JsonProperty("another_host") String anotherHost, + @JsonProperty("another_port") Integer anotherPort, + @JsonProperty("tag") String tag, + @JsonProperty("requests") Integer requests, + @JsonProperty("concurrency") Integer concurrency, + @JsonProperty("wait_seconds") Integer waitSeconds, + @JsonProperty("ssl_enabled") Boolean sslEnabled) { + this.host = host == null ? "127.0.0.1" : host; + this.port = port == null ? Integer.valueOf(24224) : port; + + this.anotherHost = anotherHost == null ? "127.0.0.1" : anotherHost; + // Nullable + this.anotherPort = anotherPort; + + this.tag = tag == null ? "foodb.bartbl" : tag; + this.requests = requests == null ? 1000000 : requests; + this.concurrency = concurrency == null ? 4 : concurrency; + this.waitSeconds = waitSeconds == null ? 60 : waitSeconds; + this.sslEnabled = sslEnabled == null ? false : sslEnabled; } + } } diff --git a/fluency-treasuredata/src/main/java/org/komamitsu/fluency/treasuredata/FluencyBuilderForTreasureData.java b/fluency-treasuredata/src/main/java/org/komamitsu/fluency/treasuredata/FluencyBuilderForTreasureData.java index dc8bd5bf..50e43f8e 100644 --- a/fluency-treasuredata/src/main/java/org/komamitsu/fluency/treasuredata/FluencyBuilderForTreasureData.java +++ b/fluency-treasuredata/src/main/java/org/komamitsu/fluency/treasuredata/FluencyBuilderForTreasureData.java @@ -18,152 +18,136 @@ import org.komamitsu.fluency.Fluency; import org.komamitsu.fluency.ingester.Ingester; -import org.komamitsu.fluency.recordformat.RecordFormatter; +import org.komamitsu.fluency.recordformat.MessagePackRecordFormatter; import org.komamitsu.fluency.treasuredata.ingester.TreasureDataIngester; import org.komamitsu.fluency.treasuredata.ingester.sender.TreasureDataSender; -import org.komamitsu.fluency.recordformat.MessagePackRecordFormatter; -public class FluencyBuilderForTreasureData - extends org.komamitsu.fluency.FluencyBuilder -{ - private Integer senderRetryMax; - private Integer senderRetryIntervalMillis; - private Integer senderMaxRetryIntervalMillis; - private Float senderRetryFactor; - private Integer senderWorkBufSize; - - public FluencyBuilderForTreasureData() - { - setBufferChunkRetentionTimeMillis(30 * 1000); - setBufferChunkInitialSize(4 * 1024 * 1024); - setBufferChunkRetentionSize(64 * 1024 * 1024); - } +public class FluencyBuilderForTreasureData extends org.komamitsu.fluency.FluencyBuilder { + private Integer senderRetryMax; + private Integer senderRetryIntervalMillis; + private Integer senderMaxRetryIntervalMillis; + private Float senderRetryFactor; + private Integer senderWorkBufSize; - public Integer getSenderRetryMax() - { - return senderRetryMax; - } + public FluencyBuilderForTreasureData() { + setBufferChunkRetentionTimeMillis(30 * 1000); + setBufferChunkInitialSize(4 * 1024 * 1024); + setBufferChunkRetentionSize(64 * 1024 * 1024); + } - public void setSenderRetryMax(Integer senderRetryMax) - { - this.senderRetryMax = senderRetryMax; - } + public Integer getSenderRetryMax() { + return senderRetryMax; + } - public Integer getSenderRetryIntervalMillis() - { - return senderRetryIntervalMillis; - } + public void setSenderRetryMax(Integer senderRetryMax) { + this.senderRetryMax = senderRetryMax; + } - public void setSenderRetryIntervalMillis(Integer senderRetryIntervalMillis) - { - this.senderRetryIntervalMillis = senderRetryIntervalMillis; - } + public Integer getSenderRetryIntervalMillis() { + return senderRetryIntervalMillis; + } - public Integer getSenderMaxRetryIntervalMillis() - { - return senderMaxRetryIntervalMillis; - } - - public void setSenderMaxRetryIntervalMillis(Integer senderMaxRetryIntervalMillis) - { - this.senderMaxRetryIntervalMillis = senderMaxRetryIntervalMillis; - } + public void setSenderRetryIntervalMillis(Integer senderRetryIntervalMillis) { + this.senderRetryIntervalMillis = senderRetryIntervalMillis; + } - public Float getSenderRetryFactor() - { - return senderRetryFactor; - } - - public void setSenderRetryFactor(Float senderRetryFactor) - { - this.senderRetryFactor = senderRetryFactor; - } + public Integer getSenderMaxRetryIntervalMillis() { + return senderMaxRetryIntervalMillis; + } - public Integer getSenderWorkBufSize() - { - return senderWorkBufSize; - } + public void setSenderMaxRetryIntervalMillis(Integer senderMaxRetryIntervalMillis) { + this.senderMaxRetryIntervalMillis = senderMaxRetryIntervalMillis; + } - public void setSenderWorkBufSize(Integer senderWorkBufSize) - { - this.senderWorkBufSize = senderWorkBufSize; - } + public Float getSenderRetryFactor() { + return senderRetryFactor; + } - public Fluency build(String apikey, String endpoint) - { - return buildFromIngester( - buildRecordFormatter(), - buildIngester(createSenderConfig(endpoint, apikey))); - } + public void setSenderRetryFactor(Float senderRetryFactor) { + this.senderRetryFactor = senderRetryFactor; + } - public Fluency build(String apikey) - { - return buildFromIngester( - buildRecordFormatter(), - buildIngester(createSenderConfig(null, apikey))); - } + public Integer getSenderWorkBufSize() { + return senderWorkBufSize; + } - private TreasureDataSender.Config createSenderConfig(String endpoint, String apikey) - { - if (apikey == null) { - throw new IllegalArgumentException("`apikey` should be set"); - } + public void setSenderWorkBufSize(Integer senderWorkBufSize) { + this.senderWorkBufSize = senderWorkBufSize; + } - TreasureDataSender.Config senderConfig = new TreasureDataSender.Config(); - senderConfig.setApikey(apikey); + public Fluency build(String apikey, String endpoint) { + return buildFromIngester( + buildRecordFormatter(), buildIngester(createSenderConfig(endpoint, apikey))); + } - if (endpoint != null) { - senderConfig.setEndpoint(endpoint); - } + public Fluency build(String apikey) { + return buildFromIngester( + buildRecordFormatter(), buildIngester(createSenderConfig(null, apikey))); + } - if (getSenderRetryMax() != null) { - senderConfig.setRetryMax(getSenderRetryMax()); - } + private TreasureDataSender.Config createSenderConfig(String endpoint, String apikey) { + if (apikey == null) { + throw new IllegalArgumentException("`apikey` should be set"); + } - if (getSenderRetryIntervalMillis() != null) { - senderConfig.setRetryIntervalMs(getSenderRetryIntervalMillis()); - } + TreasureDataSender.Config senderConfig = new TreasureDataSender.Config(); + senderConfig.setApikey(apikey); - if (getSenderMaxRetryIntervalMillis() != null) { - senderConfig.setMaxRetryIntervalMs(getSenderMaxRetryIntervalMillis()); - } + if (endpoint != null) { + senderConfig.setEndpoint(endpoint); + } - if (getSenderRetryFactor() != null) { - senderConfig.setRetryFactor(getSenderRetryFactor()); - } + if (getSenderRetryMax() != null) { + senderConfig.setRetryMax(getSenderRetryMax()); + } - if (getErrorHandler() != null) { - senderConfig.setErrorHandler(getErrorHandler()); - } + if (getSenderRetryIntervalMillis() != null) { + senderConfig.setRetryIntervalMs(getSenderRetryIntervalMillis()); + } - if (getSenderWorkBufSize() != null) { - senderConfig.setWorkBufSize(getSenderWorkBufSize()); - } + if (getSenderMaxRetryIntervalMillis() != null) { + senderConfig.setMaxRetryIntervalMs(getSenderMaxRetryIntervalMillis()); + } - return senderConfig; + if (getSenderRetryFactor() != null) { + senderConfig.setRetryFactor(getSenderRetryFactor()); } - @Override - public String toString() - { - return "FluencyBuilder{" + - "senderRetryMax=" + senderRetryMax + - ", senderRetryIntervalMillis=" + senderRetryIntervalMillis + - ", senderMaxRetryIntervalMillis=" + senderMaxRetryIntervalMillis + - ", senderRetryFactor=" + senderRetryFactor + - ", senderWorkBufSize=" + senderWorkBufSize + - "} " + super.toString(); + if (getErrorHandler() != null) { + senderConfig.setErrorHandler(getErrorHandler()); } - private MessagePackRecordFormatter buildRecordFormatter() - { - return new MessagePackRecordFormatter(); + if (getSenderWorkBufSize() != null) { + senderConfig.setWorkBufSize(getSenderWorkBufSize()); } - private Ingester buildIngester(TreasureDataSender.Config senderConfig) - { - TreasureDataSender sender = new TreasureDataSender(senderConfig); + return senderConfig; + } - return new TreasureDataIngester(sender); - } + @Override + public String toString() { + return "FluencyBuilder{" + + "senderRetryMax=" + + senderRetryMax + + ", senderRetryIntervalMillis=" + + senderRetryIntervalMillis + + ", senderMaxRetryIntervalMillis=" + + senderMaxRetryIntervalMillis + + ", senderRetryFactor=" + + senderRetryFactor + + ", senderWorkBufSize=" + + senderWorkBufSize + + "} " + + super.toString(); + } + + private MessagePackRecordFormatter buildRecordFormatter() { + return new MessagePackRecordFormatter(); + } + + private Ingester buildIngester(TreasureDataSender.Config senderConfig) { + TreasureDataSender sender = new TreasureDataSender(senderConfig); + + return new TreasureDataIngester(sender); + } } diff --git a/fluency-treasuredata/src/main/java/org/komamitsu/fluency/treasuredata/ingester/TreasureDataIngester.java b/fluency-treasuredata/src/main/java/org/komamitsu/fluency/treasuredata/ingester/TreasureDataIngester.java index 547f04ec..9bf4a261 100644 --- a/fluency-treasuredata/src/main/java/org/komamitsu/fluency/treasuredata/ingester/TreasureDataIngester.java +++ b/fluency-treasuredata/src/main/java/org/komamitsu/fluency/treasuredata/ingester/TreasureDataIngester.java @@ -16,43 +16,34 @@ package org.komamitsu.fluency.treasuredata.ingester; +import java.io.IOException; +import java.nio.ByteBuffer; import org.komamitsu.fluency.ingester.Ingester; import org.komamitsu.fluency.ingester.sender.Sender; import org.komamitsu.fluency.treasuredata.ingester.sender.TreasureDataSender; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.nio.ByteBuffer; - -public class TreasureDataIngester - implements Ingester -{ - private static final Logger LOG = LoggerFactory.getLogger(TreasureDataIngester.class); - private final TreasureDataSender sender; +public class TreasureDataIngester implements Ingester { + private static final Logger LOG = LoggerFactory.getLogger(TreasureDataIngester.class); + private final TreasureDataSender sender; - public TreasureDataIngester(TreasureDataSender sender) - { - this.sender = sender; - } + public TreasureDataIngester(TreasureDataSender sender) { + this.sender = sender; + } - @Override - public void ingest(String tag, ByteBuffer dataBuffer) - throws IOException - { - sender.send(tag, dataBuffer); - } + @Override + public void ingest(String tag, ByteBuffer dataBuffer) throws IOException { + sender.send(tag, dataBuffer); + } - @Override - public Sender getSender() - { - return sender; - } + @Override + public Sender getSender() { + return sender; + } - @Override - public void close() - throws IOException - { - sender.close(); - } + @Override + public void close() throws IOException { + sender.close(); + } } diff --git a/fluency-treasuredata/src/main/java/org/komamitsu/fluency/treasuredata/ingester/sender/TreasureDataSender.java b/fluency-treasuredata/src/main/java/org/komamitsu/fluency/treasuredata/ingester/sender/TreasureDataSender.java index 9457fa22..219e0256 100644 --- a/fluency-treasuredata/src/main/java/org/komamitsu/fluency/treasuredata/ingester/sender/TreasureDataSender.java +++ b/fluency-treasuredata/src/main/java/org/komamitsu/fluency/treasuredata/ingester/sender/TreasureDataSender.java @@ -21,19 +21,6 @@ import com.treasuredata.client.TDClientBuilder; import com.treasuredata.client.TDClientHttpException; import com.treasuredata.client.TDClientHttpNotFoundException; -import net.jodah.failsafe.Failsafe; -import net.jodah.failsafe.RetryPolicy; -import org.komamitsu.fluency.NonRetryableException; -import org.komamitsu.fluency.RetryableException; -import org.komamitsu.fluency.ingester.sender.ErrorHandler; -import org.komamitsu.fluency.ingester.sender.Sender; -import org.komamitsu.fluency.validation.Validatable; -import org.komamitsu.fluency.validation.annotation.DecimalMin; -import org.komamitsu.fluency.validation.annotation.Min; -import org.msgpack.core.annotations.VisibleForTesting; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.Closeable; import java.io.File; import java.io.IOException; @@ -48,388 +35,363 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.zip.GZIPOutputStream; +import net.jodah.failsafe.Failsafe; +import net.jodah.failsafe.RetryPolicy; +import org.komamitsu.fluency.NonRetryableException; +import org.komamitsu.fluency.RetryableException; +import org.komamitsu.fluency.ingester.sender.ErrorHandler; +import org.komamitsu.fluency.ingester.sender.Sender; +import org.komamitsu.fluency.validation.Validatable; +import org.komamitsu.fluency.validation.annotation.DecimalMin; +import org.komamitsu.fluency.validation.annotation.Min; +import org.msgpack.core.annotations.VisibleForTesting; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class TreasureDataSender - implements Closeable, Sender -{ - private static final Logger LOG = LoggerFactory.getLogger(TreasureDataSender.class); - private static final int RETRY_COUNT_FOR_DB_CREATE_DELETE_CONFLICT = 4; - private static final int RETRY_INTERVAL_MS_FOR_DB_CREATE_DELETE_CONFLICT = 1000; - private final Config config; - private final TDClient client; - private final RetryPolicy retryPolicy; - - public TreasureDataSender() - { - this(new Config()); - } - - public TreasureDataSender(Config config) - { - config.validateValues(); - this.config = config; - this.retryPolicy = - new RetryPolicy(). - handleIf(ex -> { - if (ex == null) { - // Success. Shouldn't retry. - return false; - } - - ErrorHandler errorHandler = config.getErrorHandler(); - - if (errorHandler != null) { - errorHandler.handle(ex); - } - - if (ex instanceof InterruptedException || ex instanceof NonRetryableException) { - return false; - } - - return true; - }). - withBackoff( - getRetryInternalMs(), - getMaxRetryInternalMs(), - ChronoUnit.MILLIS, - getRetryFactor()). - withMaxRetries(getRetryMax()); - - this.client = buildClient(); +public class TreasureDataSender implements Closeable, Sender { + private static final Logger LOG = LoggerFactory.getLogger(TreasureDataSender.class); + private static final int RETRY_COUNT_FOR_DB_CREATE_DELETE_CONFLICT = 4; + private static final int RETRY_INTERVAL_MS_FOR_DB_CREATE_DELETE_CONFLICT = 1000; + private final Config config; + private final TDClient client; + private final RetryPolicy retryPolicy; + + public TreasureDataSender() { + this(new Config()); + } + + public TreasureDataSender(Config config) { + config.validateValues(); + this.config = config; + this.retryPolicy = + new RetryPolicy() + .handleIf( + ex -> { + if (ex == null) { + // Success. Shouldn't retry. + return false; + } + + ErrorHandler errorHandler = config.getErrorHandler(); + + if (errorHandler != null) { + errorHandler.handle(ex); + } + + if (ex instanceof InterruptedException || ex instanceof NonRetryableException) { + return false; + } + + return true; + }) + .withBackoff( + getRetryInternalMs(), getMaxRetryInternalMs(), ChronoUnit.MILLIS, getRetryFactor()) + .withMaxRetries(getRetryMax()); + + this.client = buildClient(); + } + + @VisibleForTesting + protected TDClient buildClient() { + URI uri; + try { + uri = new URI(config.getEndpoint()); + } catch (URISyntaxException e) { + throw new NonRetryableException( + String.format("Invalid endpoint. %s", config.getEndpoint()), e); } - @VisibleForTesting - protected TDClient buildClient() - { - URI uri; - try { - uri = new URI(config.getEndpoint()); - } - catch (URISyntaxException e) { - throw new NonRetryableException( - String.format("Invalid endpoint. %s", config.getEndpoint()), e); - } + String host = uri.getHost() != null ? uri.getHost() : config.getEndpoint(); - String host = uri.getHost() != null ? uri.getHost() : config.getEndpoint(); - - TDClientBuilder builder = new TDClientBuilder(false) - .setEndpoint(host) - .setApiKey(config.getApikey()) - .setRetryLimit(config.getRetryMax()) - .setRetryInitialIntervalMillis(config.getRetryIntervalMs()) - .setRetryMaxIntervalMillis(config.getMaxRetryIntervalMs()) - .setRetryMultiplier(config.getRetryFactor()); - - if (uri.getScheme() != null && uri.getScheme().equals("http")) { - builder.setUseSSL(false); - } + TDClientBuilder builder = + new TDClientBuilder(false) + .setEndpoint(host) + .setApiKey(config.getApikey()) + .setRetryLimit(config.getRetryMax()) + .setRetryInitialIntervalMillis(config.getRetryIntervalMs()) + .setRetryMaxIntervalMillis(config.getMaxRetryIntervalMs()) + .setRetryMultiplier(config.getRetryFactor()); - if (uri.getPort() > 0) { - builder.setPort(uri.getPort()); - } - - return builder.build(); - } - - public TDClient getClient() - { - return client; + if (uri.getScheme() != null && uri.getScheme().equals("http")) { + builder.setUseSSL(false); } - public int getRetryInternalMs() - { - return config.getRetryIntervalMs(); + if (uri.getPort() > 0) { + builder.setPort(uri.getPort()); } - public int getMaxRetryInternalMs() - { - return config.getMaxRetryIntervalMs(); + return builder.build(); + } + + public TDClient getClient() { + return client; + } + + public int getRetryInternalMs() { + return config.getRetryIntervalMs(); + } + + public int getMaxRetryInternalMs() { + return config.getMaxRetryIntervalMs(); + } + + public float getRetryFactor() { + return config.getRetryFactor(); + } + + public int getRetryMax() { + return config.getRetryMax(); + } + + public int getWorkBufSize() { + return config.getWorkBufSize(); + } + + private void copyStreams(InputStream in, OutputStream out) throws IOException { + byte[] buf = new byte[getWorkBufSize()]; + while (true) { + int readLen = in.read(buf); + if (readLen < 0) { + break; + } + out.write(buf, 0, readLen); } + } - public float getRetryFactor() - { - return config.getRetryFactor(); + private boolean checkDatabaseAndWaitIfNeeded(String database) { + if (client.existsDatabase(database)) { + return true; } - public int getRetryMax() - { - return config.getRetryMax(); - } + LOG.warn("The database could be just removed or invisible. Retrying.... database={}", database); - public int getWorkBufSize() - { - return config.getWorkBufSize(); + try { + TimeUnit.MILLISECONDS.sleep(RETRY_INTERVAL_MS_FOR_DB_CREATE_DELETE_CONFLICT); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new NonRetryableException( + String.format("Failed to create database. database=%s", database), e); } - private void copyStreams(InputStream in, OutputStream out) - throws IOException - { - byte[] buf = new byte[getWorkBufSize()]; - while (true) { - int readLen = in.read(buf); - if (readLen < 0) { - break; + return false; + } + + private void createDatabase(String database) { + for (int i = 0; i < RETRY_COUNT_FOR_DB_CREATE_DELETE_CONFLICT; i++) { + try { + client.createDatabase(database); + LOG.info("Created database. database={}", database); + return; + } catch (TDClientHttpException e) { + switch (e.getStatusCode()) { + case 409: + LOG.info("The database already exists. database={}", database); + if (checkDatabaseAndWaitIfNeeded(database)) { + return; } - out.write(buf, 0, readLen); + // Retrying... + break; + case 401: + case 403: + case 404: + throw new NonRetryableException( + String.format("Failed to create database. database=%s", database), e); } + } catch (NonRetryableException e) { + throw e; + } catch (Throwable e) { + throw new RetryableException( + String.format("Failed to create database. database=%s", database), e); + } } - private boolean checkDatabaseAndWaitIfNeeded(String database) - { - if (client.existsDatabase(database)) { - return true; - } - - LOG.warn("The database could be just removed or invisible. Retrying.... database={}", database); - - try { - TimeUnit.MILLISECONDS.sleep(RETRY_INTERVAL_MS_FOR_DB_CREATE_DELETE_CONFLICT); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); + // Retry over + throw new NonRetryableException( + String.format( + "It seems you don't have a proper permission on the database. database=%s", database)); + } + + private void createTable(String database, String table) { + while (true) { + try { + client.createTable(database, table); + LOG.info("Created table. database={}, table={}", database, table); + return; + } catch (TDClientHttpException e) { + switch (e.getStatusCode()) { + case 409: + LOG.info("The table already exists. database={}, table={}", database, table); + return; + case 401: + case 403: throw new NonRetryableException( - String.format("Failed to create database. database=%s", database), e); + String.format("Failed to create table. database=%s, table=%s", database, table), e); + case 404: + createDatabase(database); + // Retry to create the table + break; + default: + throw new RetryableException( + String.format("Failed to create table. database=%s, table=%s", database, table), e); } - - return false; + } catch (NonRetryableException e) { + throw e; + } catch (Throwable e) { + throw new RetryableException( + String.format("Failed to create table. database=%s, table=%s", database, table), e); + } + } + } + + private void importData(String database, String table, String uniqueId, File file) { + LOG.debug( + "Importing data to TD table: database={}, table={}, uniqueId={}, fileSize={}", + database, + table, + uniqueId, + file.length()); + + while (true) { + try { + client.importFile(database, table, file, uniqueId); + return; + } catch (TDClientHttpNotFoundException e) { + createTable(database, table); + // Retry to create the table + } catch (NonRetryableException e) { + throw e; + } catch (Throwable e) { + throw new RetryableException( + String.format("Failed to import data. database=%s, table=%s", database, table), e); + } } + } + + public void send(String dbAndTableTag, ByteBuffer dataBuffer) throws IOException { + String[] dbAndTable = dbAndTableTag.split("\\."); + // TODO: Validation + String database = dbAndTable[0]; + String table = dbAndTable[1]; + + File file = File.createTempFile("tmp-fluency-", ".msgpack.gz"); + try { + try (InputStream in = new ByteBufferBackedInputStream(dataBuffer); + OutputStream out = + new GZIPOutputStream( + Files.newOutputStream(file.toPath(), StandardOpenOption.WRITE))) { + + copyStreams(in, out); + } + + String uniqueId = UUID.randomUUID().toString(); + Failsafe.with(retryPolicy).run(() -> importData(database, table, uniqueId, file)); + } finally { + if (!file.delete()) { + LOG.warn("Failed to delete a temp file: {}", file.getAbsolutePath()); + } + } + } - private void createDatabase(String database) - { - for (int i = 0; i < RETRY_COUNT_FOR_DB_CREATE_DELETE_CONFLICT; i++) { - try { - client.createDatabase(database); - LOG.info("Created database. database={}", database); - return; - } - catch (TDClientHttpException e) { - switch (e.getStatusCode()) { - case 409: - LOG.info("The database already exists. database={}", database); - if (checkDatabaseAndWaitIfNeeded(database)) { - return; - } - // Retrying... - break; - case 401: - case 403: - case 404: - throw new NonRetryableException( - String.format("Failed to create database. database=%s", database), e); - } - } - catch (NonRetryableException e) { - throw e; - } - catch (Throwable e) { - throw new RetryableException( - String.format("Failed to create database. database=%s", database), e); - } - } + @Override + public void close() throws IOException {} - // Retry over - throw new NonRetryableException( - String.format("It seems you don't have a proper permission on the database. database=%s", database)); - } + public static class Config extends Sender.Config implements Validatable { + private String endpoint = "https://api-import.treasuredata.com"; + private String apikey; - private void createTable(String database, String table) - { - while (true) { - try { - client.createTable(database, table); - LOG.info("Created table. database={}, table={}", database, table); - return; - } - catch (TDClientHttpException e) { - switch (e.getStatusCode()) { - case 409: - LOG.info("The table already exists. database={}, table={}", database, table); - return; - case 401: - case 403: - throw new NonRetryableException( - String.format("Failed to create table. database=%s, table=%s", database, table), e); - case 404: - createDatabase(database); - // Retry to create the table - break; - default: - throw new RetryableException( - String.format("Failed to create table. database=%s, table=%s", database, table), e); - } - } - catch (NonRetryableException e) { - throw e; - } - catch (Throwable e) { - throw new RetryableException( - String.format("Failed to create table. database=%s, table=%s", database, table), e); - } - } - } + @Min(10) + private int retryIntervalMs = 1000; - private void importData(String database, String table, String uniqueId, File file) - { - LOG.debug("Importing data to TD table: database={}, table={}, uniqueId={}, fileSize={}", - database, table, uniqueId, file.length()); + @Min(10) + private int maxRetryIntervalMs = 30000; - while (true) { - try { - client.importFile(database, table, file, uniqueId); - return; - } - catch (TDClientHttpNotFoundException e) { - createTable(database, table); - // Retry to create the table - } - catch (NonRetryableException e) { - throw e; - } - catch (Throwable e) { - throw new RetryableException( - String.format("Failed to import data. database=%s, table=%s", database, table), e); - } - } - } + @DecimalMin("1.0") + private float retryFactor = 2; - public void send(String dbAndTableTag, ByteBuffer dataBuffer) - throws IOException - { - String[] dbAndTable = dbAndTableTag.split("\\."); - // TODO: Validation - String database = dbAndTable[0]; - String table = dbAndTable[1]; - - File file = File.createTempFile("tmp-fluency-", ".msgpack.gz"); - try { - try (InputStream in = new ByteBufferBackedInputStream(dataBuffer); - OutputStream out = new GZIPOutputStream( - Files.newOutputStream( - file.toPath(), - StandardOpenOption.WRITE))) { - - copyStreams(in, out); - } + @Min(0) + private int retryMax = 10; - String uniqueId = UUID.randomUUID().toString(); - Failsafe.with(retryPolicy).run(() -> importData(database, table, uniqueId, file)); - } - finally { - if (!file.delete()) { - LOG.warn("Failed to delete a temp file: {}", file.getAbsolutePath()); - } - } - } + @Min(1024) + private int workBufSize = 8192; - @Override - public void close() - throws IOException - { + public String getEndpoint() { + return endpoint; } - public static class Config - extends Sender.Config - implements Validatable - { - private String endpoint = "https://api-import.treasuredata.com"; - private String apikey; - @Min(10) - private int retryIntervalMs = 1000; - @Min(10) - private int maxRetryIntervalMs = 30000; - @DecimalMin("1.0") - private float retryFactor = 2; - @Min(0) - private int retryMax = 10; - @Min(1024) - private int workBufSize = 8192; - - public String getEndpoint() - { - return endpoint; - } - - public void setEndpoint(String endpoint) - { - this.endpoint = endpoint; - } + public void setEndpoint(String endpoint) { + this.endpoint = endpoint; + } - public String getApikey() - { - return apikey; - } + public String getApikey() { + return apikey; + } - public void setApikey(String apikey) - { - this.apikey = apikey; - } + public void setApikey(String apikey) { + this.apikey = apikey; + } - public int getRetryIntervalMs() - { - return retryIntervalMs; - } + public int getRetryIntervalMs() { + return retryIntervalMs; + } - public void setRetryIntervalMs(int retryIntervalMs) - { - this.retryIntervalMs = retryIntervalMs; - } + public void setRetryIntervalMs(int retryIntervalMs) { + this.retryIntervalMs = retryIntervalMs; + } - public int getMaxRetryIntervalMs() - { - return maxRetryIntervalMs; - } + public int getMaxRetryIntervalMs() { + return maxRetryIntervalMs; + } - public void setMaxRetryIntervalMs(int maxRetryIntervalMs) - { - this.maxRetryIntervalMs = maxRetryIntervalMs; - } + public void setMaxRetryIntervalMs(int maxRetryIntervalMs) { + this.maxRetryIntervalMs = maxRetryIntervalMs; + } - public float getRetryFactor() - { - return retryFactor; - } + public float getRetryFactor() { + return retryFactor; + } - public void setRetryFactor(float retryFactor) - { - this.retryFactor = retryFactor; - } + public void setRetryFactor(float retryFactor) { + this.retryFactor = retryFactor; + } - public int getRetryMax() - { - return retryMax; - } + public int getRetryMax() { + return retryMax; + } - public void setRetryMax(int retryMax) - { - this.retryMax = retryMax; - } + public void setRetryMax(int retryMax) { + this.retryMax = retryMax; + } - public int getWorkBufSize() - { - return workBufSize; - } + public int getWorkBufSize() { + return workBufSize; + } - public void setWorkBufSize(int workBufSize) - { - this.workBufSize = workBufSize; - } + public void setWorkBufSize(int workBufSize) { + this.workBufSize = workBufSize; + } - @Override - public String toString() - { - return "Config{" + - "endpoint='" + endpoint + '\'' + - ", retryIntervalMs=" + retryIntervalMs + - ", maxRetryIntervalMs=" + maxRetryIntervalMs + - ", retryFactor=" + retryFactor + - ", retryMax=" + retryMax + - ", workBufSize=" + workBufSize + - "} " + super.toString(); - } + @Override + public String toString() { + return "Config{" + + "endpoint='" + + endpoint + + '\'' + + ", retryIntervalMs=" + + retryIntervalMs + + ", maxRetryIntervalMs=" + + maxRetryIntervalMs + + ", retryFactor=" + + retryFactor + + ", retryMax=" + + retryMax + + ", workBufSize=" + + workBufSize + + "} " + + super.toString(); + } - void validateValues() - { - validate(); - } + void validateValues() { + validate(); } + } } diff --git a/fluency-treasuredata/src/test/java/org/komamitsu/fluency/treasuredata/FluencyBuilderForTreasureDataTest.java b/fluency-treasuredata/src/test/java/org/komamitsu/fluency/treasuredata/FluencyBuilderForTreasureDataTest.java index 8bb67a04..65cce117 100644 --- a/fluency-treasuredata/src/test/java/org/komamitsu/fluency/treasuredata/FluencyBuilderForTreasureDataTest.java +++ b/fluency-treasuredata/src/test/java/org/komamitsu/fluency/treasuredata/FluencyBuilderForTreasureDataTest.java @@ -16,174 +16,167 @@ package org.komamitsu.fluency.treasuredata; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + import com.treasuredata.client.TDClient; import com.treasuredata.client.TDClientConfig; import com.treasuredata.client.TDHttpClient; +import java.io.IOException; +import java.lang.reflect.Field; import org.junit.jupiter.api.Test; import org.komamitsu.fluency.Fluency; import org.komamitsu.fluency.buffer.Buffer; import org.komamitsu.fluency.flusher.Flusher; import org.komamitsu.fluency.treasuredata.ingester.sender.TreasureDataSender; -import java.io.IOException; -import java.lang.reflect.Field; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class FluencyBuilderForTreasureDataTest -{ - private static final String APIKEY = "12345/1qaz2wsx3edc4rfv5tgb6yhn"; - - private void assertBuffer(Buffer buffer) - { - assertEquals(512 * 1024 * 1024L, buffer.getMaxBufferSize()); - assertNull(buffer.getFileBackupDir()); - assertEquals("packed_forward", buffer.bufferFormatType()); - assertEquals(2f, buffer.getChunkExpandRatio()); - assertEquals(64 * 1024 * 1024, buffer.getChunkRetentionSize()); - assertEquals(4 * 1024 * 1024, buffer.getChunkInitialSize()); - assertEquals(30000, buffer.getChunkRetentionTimeMillis()); - assertFalse(buffer.getJvmHeapBufferMode()); +class FluencyBuilderForTreasureDataTest { + private static final String APIKEY = "12345/1qaz2wsx3edc4rfv5tgb6yhn"; + + private void assertBuffer(Buffer buffer) { + assertEquals(512 * 1024 * 1024L, buffer.getMaxBufferSize()); + assertNull(buffer.getFileBackupDir()); + assertEquals("packed_forward", buffer.bufferFormatType()); + assertEquals(2f, buffer.getChunkExpandRatio()); + assertEquals(64 * 1024 * 1024, buffer.getChunkRetentionSize()); + assertEquals(4 * 1024 * 1024, buffer.getChunkInitialSize()); + assertEquals(30000, buffer.getChunkRetentionTimeMillis()); + assertFalse(buffer.getJvmHeapBufferMode()); + } + + private void assertFlusher(Flusher flusher) { + assertFalse(flusher.isTerminated()); + assertEquals(600, flusher.getFlushAttemptIntervalMillis()); + assertEquals(60, flusher.getWaitUntilBufferFlushed()); + assertEquals(60, flusher.getWaitUntilTerminated()); + } + + private void assertDefaultFluentdSender( + TreasureDataSender sender, String expectedEndpoint, boolean expectedUseSsl) + throws NoSuchFieldException, IllegalAccessException { + assertEquals(1000, sender.getRetryInternalMs()); + assertEquals(30000, sender.getMaxRetryInternalMs()); + assertEquals(2.0f, sender.getRetryFactor()); + assertEquals(10, sender.getRetryMax()); + assertEquals(8192, sender.getWorkBufSize()); + + Field httpClientField = TDClient.class.getDeclaredField("httpClient"); + httpClientField.setAccessible(true); + TDHttpClient tdHttpClient = (TDHttpClient) httpClientField.get(sender.getClient()); + + Field configField = TDHttpClient.class.getDeclaredField("config"); + configField.setAccessible(true); + TDClientConfig config = (TDClientConfig) configField.get(tdHttpClient); + + assertEquals(expectedEndpoint, config.endpoint); + assertEquals(expectedUseSsl, config.useSSL); + assertEquals(FluencyBuilderForTreasureDataTest.APIKEY, config.apiKey.get()); + } + + @Test + void build() throws IOException, NoSuchFieldException, IllegalAccessException { + try (Fluency fluency = new FluencyBuilderForTreasureData().build(APIKEY)) { + assertBuffer(fluency.getBuffer()); + assertFlusher(fluency.getFlusher()); + assertDefaultFluentdSender( + (TreasureDataSender) fluency.getFlusher().getIngester().getSender(), + "api-import.treasuredata.com", + true); } - - private void assertFlusher(Flusher flusher) - { - assertFalse(flusher.isTerminated()); - assertEquals(600, flusher.getFlushAttemptIntervalMillis()); - assertEquals(60, flusher.getWaitUntilBufferFlushed()); - assertEquals(60, flusher.getWaitUntilTerminated()); + } + + @Test + void buildWithCustomHttpsEndpoint() + throws IOException, NoSuchFieldException, IllegalAccessException { + try (Fluency fluency = + new FluencyBuilderForTreasureData().build(APIKEY, "https://custom.endpoint.org")) { + assertBuffer(fluency.getBuffer()); + assertFlusher(fluency.getFlusher()); + assertDefaultFluentdSender( + (TreasureDataSender) fluency.getFlusher().getIngester().getSender(), + "custom.endpoint.org", + true); } - - private void assertDefaultFluentdSender( - TreasureDataSender sender, - String expectedEndpoint, - boolean expectedUseSsl) - throws NoSuchFieldException, IllegalAccessException - { - assertEquals(1000, sender.getRetryInternalMs()); - assertEquals(30000, sender.getMaxRetryInternalMs()); - assertEquals(2.0f, sender.getRetryFactor()); - assertEquals(10, sender.getRetryMax()); - assertEquals(8192, sender.getWorkBufSize()); - - Field httpClientField = TDClient.class.getDeclaredField("httpClient"); - httpClientField.setAccessible(true); - TDHttpClient tdHttpClient = (TDHttpClient) httpClientField.get(sender.getClient()); - - Field configField = TDHttpClient.class.getDeclaredField("config"); - configField.setAccessible(true); - TDClientConfig config = (TDClientConfig) configField.get(tdHttpClient); - - assertEquals(expectedEndpoint, config.endpoint); - assertEquals(expectedUseSsl, config.useSSL); - assertEquals(FluencyBuilderForTreasureDataTest.APIKEY, config.apiKey.get()); + } + + @Test + void buildWithCustomHttpsEndpointWithoutScheme() + throws IOException, NoSuchFieldException, IllegalAccessException { + try (Fluency fluency = + new FluencyBuilderForTreasureData().build(APIKEY, "custom.endpoint.org")) { + assertBuffer(fluency.getBuffer()); + assertFlusher(fluency.getFlusher()); + assertDefaultFluentdSender( + (TreasureDataSender) fluency.getFlusher().getIngester().getSender(), + "custom.endpoint.org", + true); } - - @Test - void build() - throws IOException, NoSuchFieldException, IllegalAccessException - { - try (Fluency fluency = new FluencyBuilderForTreasureData().build(APIKEY)) { - assertBuffer(fluency.getBuffer()); - assertFlusher(fluency.getFlusher()); - assertDefaultFluentdSender( - (TreasureDataSender) fluency.getFlusher().getIngester().getSender(), - "api-import.treasuredata.com", true); - } + } + + @Test + void buildWithCustomHttpEndpoint() + throws IOException, NoSuchFieldException, IllegalAccessException { + try (Fluency fluency = + new FluencyBuilderForTreasureData().build(APIKEY, "http://custom.endpoint.org")) { + assertBuffer(fluency.getBuffer()); + assertFlusher(fluency.getFlusher()); + assertDefaultFluentdSender( + (TreasureDataSender) fluency.getFlusher().getIngester().getSender(), + "custom.endpoint.org", + false); } - - @Test - void buildWithCustomHttpsEndpoint() - throws IOException, NoSuchFieldException, IllegalAccessException - { - try (Fluency fluency = new FluencyBuilderForTreasureData().build(APIKEY, "https://custom.endpoint.org")) { - assertBuffer(fluency.getBuffer()); - assertFlusher(fluency.getFlusher()); - assertDefaultFluentdSender( - (TreasureDataSender) fluency.getFlusher().getIngester().getSender(), - "custom.endpoint.org", true); - } - } - - @Test - void buildWithCustomHttpsEndpointWithoutScheme() - throws IOException, NoSuchFieldException, IllegalAccessException - { - try (Fluency fluency = new FluencyBuilderForTreasureData().build(APIKEY, "custom.endpoint.org")) { - assertBuffer(fluency.getBuffer()); - assertFlusher(fluency.getFlusher()); - assertDefaultFluentdSender( - (TreasureDataSender) fluency.getFlusher().getIngester().getSender(), - "custom.endpoint.org", true); - } - } - - @Test - void buildWithCustomHttpEndpoint() - throws IOException, NoSuchFieldException, IllegalAccessException - { - try (Fluency fluency = new FluencyBuilderForTreasureData().build(APIKEY, "http://custom.endpoint.org")) { - assertBuffer(fluency.getBuffer()); - assertFlusher(fluency.getFlusher()); - assertDefaultFluentdSender( - (TreasureDataSender) fluency.getFlusher().getIngester().getSender(), - "custom.endpoint.org", false); - } - } - - @Test - void buildWithAllCustomConfig() - throws IOException - { - String tmpdir = System.getProperty("java.io.tmpdir"); - assertNotNull(tmpdir); - - FluencyBuilderForTreasureData builder = new FluencyBuilderForTreasureData(); - builder.setFlushAttemptIntervalMillis(200); - builder.setMaxBufferSize(Long.MAX_VALUE); - builder.setBufferChunkInitialSize(7 * 1024 * 1024); - builder.setBufferChunkRetentionSize(13 * 1024 * 1024); - builder.setBufferChunkRetentionTimeMillis(19 * 1000); - builder.setJvmHeapBufferMode(true); - builder.setWaitUntilBufferFlushed(42); - builder.setWaitUntilFlusherTerminated(24); - builder.setFileBackupDir(tmpdir); - builder.setSenderRetryIntervalMillis(1234); - builder.setSenderMaxRetryIntervalMillis(345678); - builder.setSenderRetryFactor(3.14f); - builder.setSenderRetryMax(17); - builder.setSenderWorkBufSize(123456); - ; - - try (Fluency fluency = builder.build(APIKEY)) { - assertEquals(Buffer.class, fluency.getBuffer().getClass()); - Buffer buffer = fluency.getBuffer(); - assertEquals(Long.MAX_VALUE, buffer.getMaxBufferSize()); - assertEquals(tmpdir, buffer.getFileBackupDir()); - assertEquals("packed_forward", buffer.bufferFormatType()); - assertEquals(19 * 1000, buffer.getChunkRetentionTimeMillis()); - assertEquals(2f, buffer.getChunkExpandRatio()); - assertEquals(7 * 1024 * 1024, buffer.getChunkInitialSize()); - assertEquals(13 * 1024 * 1024, buffer.getChunkRetentionSize()); - assertTrue(buffer.getJvmHeapBufferMode()); - - Flusher flusher = fluency.getFlusher(); - assertFalse(flusher.isTerminated()); - assertEquals(200, flusher.getFlushAttemptIntervalMillis()); - assertEquals(42, flusher.getWaitUntilBufferFlushed()); - assertEquals(24, flusher.getWaitUntilTerminated()); - - assertEquals(TreasureDataSender.class, flusher.getIngester().getSender().getClass()); - TreasureDataSender sender = (TreasureDataSender) flusher.getIngester().getSender(); - assertEquals(1234, sender.getRetryInternalMs()); - assertEquals(345678, sender.getMaxRetryInternalMs()); - assertEquals(3.14f, sender.getRetryFactor()); - assertEquals(17, sender.getRetryMax()); - assertEquals(123456, sender.getWorkBufSize()); - } + } + + @Test + void buildWithAllCustomConfig() throws IOException { + String tmpdir = System.getProperty("java.io.tmpdir"); + assertNotNull(tmpdir); + + FluencyBuilderForTreasureData builder = new FluencyBuilderForTreasureData(); + builder.setFlushAttemptIntervalMillis(200); + builder.setMaxBufferSize(Long.MAX_VALUE); + builder.setBufferChunkInitialSize(7 * 1024 * 1024); + builder.setBufferChunkRetentionSize(13 * 1024 * 1024); + builder.setBufferChunkRetentionTimeMillis(19 * 1000); + builder.setJvmHeapBufferMode(true); + builder.setWaitUntilBufferFlushed(42); + builder.setWaitUntilFlusherTerminated(24); + builder.setFileBackupDir(tmpdir); + builder.setSenderRetryIntervalMillis(1234); + builder.setSenderMaxRetryIntervalMillis(345678); + builder.setSenderRetryFactor(3.14f); + builder.setSenderRetryMax(17); + builder.setSenderWorkBufSize(123456); + ; + + try (Fluency fluency = builder.build(APIKEY)) { + assertEquals(Buffer.class, fluency.getBuffer().getClass()); + Buffer buffer = fluency.getBuffer(); + assertEquals(Long.MAX_VALUE, buffer.getMaxBufferSize()); + assertEquals(tmpdir, buffer.getFileBackupDir()); + assertEquals("packed_forward", buffer.bufferFormatType()); + assertEquals(19 * 1000, buffer.getChunkRetentionTimeMillis()); + assertEquals(2f, buffer.getChunkExpandRatio()); + assertEquals(7 * 1024 * 1024, buffer.getChunkInitialSize()); + assertEquals(13 * 1024 * 1024, buffer.getChunkRetentionSize()); + assertTrue(buffer.getJvmHeapBufferMode()); + + Flusher flusher = fluency.getFlusher(); + assertFalse(flusher.isTerminated()); + assertEquals(200, flusher.getFlushAttemptIntervalMillis()); + assertEquals(42, flusher.getWaitUntilBufferFlushed()); + assertEquals(24, flusher.getWaitUntilTerminated()); + + assertEquals(TreasureDataSender.class, flusher.getIngester().getSender().getClass()); + TreasureDataSender sender = (TreasureDataSender) flusher.getIngester().getSender(); + assertEquals(1234, sender.getRetryInternalMs()); + assertEquals(345678, sender.getMaxRetryInternalMs()); + assertEquals(3.14f, sender.getRetryFactor()); + assertEquals(17, sender.getRetryMax()); + assertEquals(123456, sender.getWorkBufSize()); } + } } diff --git a/fluency-treasuredata/src/test/java/org/komamitsu/fluency/treasuredata/ingester/TreasureDataIngesterTest.java b/fluency-treasuredata/src/test/java/org/komamitsu/fluency/treasuredata/ingester/TreasureDataIngesterTest.java index 3d8c1fa4..7b45545f 100644 --- a/fluency-treasuredata/src/test/java/org/komamitsu/fluency/treasuredata/ingester/TreasureDataIngesterTest.java +++ b/fluency-treasuredata/src/test/java/org/komamitsu/fluency/treasuredata/ingester/TreasureDataIngesterTest.java @@ -16,15 +16,6 @@ package org.komamitsu.fluency.treasuredata.ingester; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.treasuredata.ingester.sender.TreasureDataSender; -import org.mockito.ArgumentCaptor; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; - import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.eq; @@ -32,47 +23,46 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -class TreasureDataIngesterTest -{ - private static final Charset CHARSET = Charset.forName("UTF-8"); - private static final String TAG = "foo.bar"; - private static final byte[] DATA = "hello, world".getBytes(CHARSET); - private TreasureDataSender treasureDataSender; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.treasuredata.ingester.sender.TreasureDataSender; +import org.mockito.ArgumentCaptor; + +class TreasureDataIngesterTest { + private static final Charset CHARSET = Charset.forName("UTF-8"); + private static final String TAG = "foo.bar"; + private static final byte[] DATA = "hello, world".getBytes(CHARSET); + private TreasureDataSender treasureDataSender; - @BeforeEach - void setUp() - throws Exception - { - treasureDataSender = mock(TreasureDataSender.class); - } + @BeforeEach + void setUp() throws Exception { + treasureDataSender = mock(TreasureDataSender.class); + } - @Test - void ingest() - throws IOException - { - TreasureDataIngester ingester = new TreasureDataIngester(treasureDataSender); - ingester.ingest(TAG, ByteBuffer.wrap(DATA)); - ArgumentCaptor byteBufferArgumentCaptor = ArgumentCaptor.forClass(ByteBuffer.class); - verify(treasureDataSender, times(1)).send(eq(TAG), byteBufferArgumentCaptor.capture()); + @Test + void ingest() throws IOException { + TreasureDataIngester ingester = new TreasureDataIngester(treasureDataSender); + ingester.ingest(TAG, ByteBuffer.wrap(DATA)); + ArgumentCaptor byteBufferArgumentCaptor = ArgumentCaptor.forClass(ByteBuffer.class); + verify(treasureDataSender, times(1)).send(eq(TAG), byteBufferArgumentCaptor.capture()); - assertEquals(1, byteBufferArgumentCaptor.getAllValues().size()); - assertArrayEquals(DATA, byteBufferArgumentCaptor.getAllValues().get(0).array()); - } + assertEquals(1, byteBufferArgumentCaptor.getAllValues().size()); + assertArrayEquals(DATA, byteBufferArgumentCaptor.getAllValues().get(0).array()); + } - @Test - void getSender() - { - assertEquals(treasureDataSender, - new TreasureDataIngester(treasureDataSender).getSender()); - } + @Test + void getSender() { + assertEquals(treasureDataSender, new TreasureDataIngester(treasureDataSender).getSender()); + } - @Test - void close() - throws IOException - { - TreasureDataIngester ingester = new TreasureDataIngester(treasureDataSender); - ingester.close(); + @Test + void close() throws IOException { + TreasureDataIngester ingester = new TreasureDataIngester(treasureDataSender); + ingester.close(); - verify(treasureDataSender, times(1)).close(); - } + verify(treasureDataSender, times(1)).close(); + } } diff --git a/fluency-treasuredata/src/test/java/org/komamitsu/fluency/treasuredata/ingester/sender/TreasureDataSenderTest.java b/fluency-treasuredata/src/test/java/org/komamitsu/fluency/treasuredata/ingester/sender/TreasureDataSenderTest.java index ed1893f1..b94927eb 100644 --- a/fluency-treasuredata/src/test/java/org/komamitsu/fluency/treasuredata/ingester/sender/TreasureDataSenderTest.java +++ b/fluency-treasuredata/src/test/java/org/komamitsu/fluency/treasuredata/ingester/sender/TreasureDataSenderTest.java @@ -16,24 +16,6 @@ package org.komamitsu.fluency.treasuredata.ingester.sender; -import com.treasuredata.client.TDClient; -import com.treasuredata.client.TDClientHttpConflictException; -import com.treasuredata.client.TDClientHttpNotFoundException; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.NonRetryableException; -import org.mockito.ArgumentCaptor; - -import java.io.DataInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.zip.GZIPInputStream; - import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -48,174 +30,191 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -class TreasureDataSenderTest -{ - private final static Charset CHARSET = Charset.forName("UTF-8"); - private final static String DB = "foodb"; - private final static String TABLE = "bartbl"; - private final static String DB_AND_TABLE = "foodb.bartbl"; - private final static byte[] DATA = "hello, world".getBytes(CHARSET); - private TDClient client; - private TreasureDataSender sender; - - @BeforeEach - void setUp() - { - client = mock(TDClient.class); - sender = new TreasureDataSender(new TreasureDataSender.Config()) - { - @Override - protected TDClient buildClient() - { - return client; - } +import com.treasuredata.client.TDClient; +import com.treasuredata.client.TDClientHttpConflictException; +import com.treasuredata.client.TDClientHttpNotFoundException; +import java.io.DataInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.zip.GZIPInputStream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.NonRetryableException; +import org.mockito.ArgumentCaptor; + +class TreasureDataSenderTest { + private static final Charset CHARSET = Charset.forName("UTF-8"); + private static final String DB = "foodb"; + private static final String TABLE = "bartbl"; + private static final String DB_AND_TABLE = "foodb.bartbl"; + private static final byte[] DATA = "hello, world".getBytes(CHARSET); + private TDClient client; + private TreasureDataSender sender; + + @BeforeEach + void setUp() { + client = mock(TDClient.class); + sender = + new TreasureDataSender(new TreasureDataSender.Config()) { + @Override + protected TDClient buildClient() { + return client; + } }; + } + + private void assertImportedFile(File file) throws IOException { + try (FileInputStream fileInputStream = new FileInputStream(file); + GZIPInputStream gzipInputStream = new GZIPInputStream(fileInputStream); + DataInputStream dataInputStream = new DataInputStream(gzipInputStream)) { + byte[] data = new byte[DATA.length]; + dataInputStream.readFully(data); + assertArrayEquals(DATA, data); } - - private void assertImportedFile(File file) - throws IOException - { - try (FileInputStream fileInputStream = new FileInputStream(file); - GZIPInputStream gzipInputStream = new GZIPInputStream(fileInputStream); - DataInputStream dataInputStream = new DataInputStream(gzipInputStream)) { - byte[] data = new byte[DATA.length]; - dataInputStream.readFully(data); - assertArrayEquals(DATA, data); - } + } + + @Test + void send() throws IOException { + doAnswer( + invocation -> { + assertImportedFile(invocation.getArgument(2)); + return null; + }) + .when(client) + .importFile(anyString(), anyString(), any(File.class), anyString()); + + sender.send(DB_AND_TABLE, ByteBuffer.wrap(DATA)); + ArgumentCaptor uniqueIdArgumentCaptor = ArgumentCaptor.forClass(String.class); + verify(client, times(1)) + .importFile(eq(DB), eq(TABLE), any(File.class), uniqueIdArgumentCaptor.capture()); + verify(client, times(0)).createDatabase(anyString()); + verify(client, times(0)).createTable(anyString(), anyString()); + UUID.fromString(uniqueIdArgumentCaptor.getValue()); + } + + @Test + void sendWithCreatingTable() throws IOException { + AtomicInteger importToTableCalls = new AtomicInteger(); + doAnswer( + invocation -> { + if (importToTableCalls.getAndIncrement() == 0) { + throw new TDClientHttpNotFoundException("Not Found!!!!"); + } + assertImportedFile(invocation.getArgument(2)); + return null; + }) + .when(client) + .importFile(anyString(), anyString(), any(File.class), anyString()); + + sender.send(DB_AND_TABLE, ByteBuffer.wrap(DATA)); + ArgumentCaptor uniqueIdArgumentCaptor = ArgumentCaptor.forClass(String.class); + verify(client, times(2)) + .importFile(eq(DB), eq(TABLE), any(File.class), uniqueIdArgumentCaptor.capture()); + verify(client, times(0)).createDatabase(anyString()); + verify(client, times(1)).createTable(eq(DB), eq(TABLE)); + UUID.fromString(uniqueIdArgumentCaptor.getValue()); + } + + @Test + void sendWithCreatingDatabase() throws IOException { + AtomicInteger importToTableCalls = new AtomicInteger(); + doAnswer( + invocation -> { + if (importToTableCalls.getAndIncrement() == 0) { + throw new TDClientHttpNotFoundException("Not Found!!!!"); + } + assertImportedFile(invocation.getArgument(2)); + return null; + }) + .when(client) + .importFile(anyString(), anyString(), any(File.class), anyString()); + + AtomicInteger createTableCalls = new AtomicInteger(); + doAnswer( + invocation -> { + if (createTableCalls.getAndIncrement() == 0) { + throw new TDClientHttpNotFoundException("Not Found!!!!"); + } + return null; + }) + .when(client) + .createTable(anyString(), anyString()); + + sender.send(DB_AND_TABLE, ByteBuffer.wrap(DATA)); + ArgumentCaptor uniqueIdArgumentCaptor = ArgumentCaptor.forClass(String.class); + verify(client, times(2)) + .importFile(eq(DB), eq(TABLE), any(File.class), uniqueIdArgumentCaptor.capture()); + verify(client, times(1)).createDatabase(eq(DB)); + verify(client, times(2)).createTable(eq(DB), eq(TABLE)); + UUID.fromString(uniqueIdArgumentCaptor.getValue()); + } + + @Test + public void sendWithLackOfPermissionOnDatabase() throws IOException { + doThrow(new TDClientHttpNotFoundException("Not Found!!!!")) + .when(client) + .importFile(anyString(), anyString(), any(File.class), anyString()); + + doThrow(new TDClientHttpNotFoundException("Not Found!!!!")) + .when(client) + .createTable(anyString(), anyString()); + + doThrow(new TDClientHttpConflictException("Conflict!!!!")) + .when(client) + .createDatabase(anyString()); + + doReturn(false).when(client).existsDatabase(anyString()); + + try { + sender.send(DB_AND_TABLE, ByteBuffer.wrap(DATA)); + fail(); + } catch (NonRetryableException e) { + assertTrue(true); } - - @Test - void send() - throws IOException + ArgumentCaptor uniqueIdArgumentCaptor = ArgumentCaptor.forClass(String.class); + verify(client, times(1)) + .importFile(eq(DB), eq(TABLE), any(File.class), uniqueIdArgumentCaptor.capture()); + verify(client, times(4)).createDatabase(eq(DB)); + verify(client, times(4)).existsDatabase(eq(DB)); + verify(client, times(1)).createTable(eq(DB), eq(TABLE)); + UUID.fromString(uniqueIdArgumentCaptor.getValue()); + } + + @Test + void validateConfig() { { - doAnswer(invocation -> { - assertImportedFile(invocation.getArgument(2)); - return null; - }).when(client).importFile(anyString(), anyString(), any(File.class), anyString()); - - sender.send(DB_AND_TABLE, ByteBuffer.wrap(DATA)); - ArgumentCaptor uniqueIdArgumentCaptor = ArgumentCaptor.forClass(String.class); - verify(client, times(1)).importFile( - eq(DB), eq(TABLE), any(File.class), uniqueIdArgumentCaptor.capture()); - verify(client, times(0)).createDatabase(anyString()); - verify(client, times(0)).createTable(anyString(), anyString()); - UUID.fromString(uniqueIdArgumentCaptor.getValue()); + TreasureDataSender.Config config = new TreasureDataSender.Config(); + config.setRetryIntervalMs(9); + assertThrows(IllegalArgumentException.class, () -> new TreasureDataSender(config)); } - @Test - void sendWithCreatingTable() - throws IOException { - AtomicInteger importToTableCalls = new AtomicInteger(); - doAnswer(invocation -> { - if (importToTableCalls.getAndIncrement() == 0) { - throw new TDClientHttpNotFoundException("Not Found!!!!"); - } - assertImportedFile(invocation.getArgument(2)); - return null; - }).when(client).importFile(anyString(), anyString(), any(File.class), anyString()); - - sender.send(DB_AND_TABLE, ByteBuffer.wrap(DATA)); - ArgumentCaptor uniqueIdArgumentCaptor = ArgumentCaptor.forClass(String.class); - verify(client, times(2)).importFile( - eq(DB), eq(TABLE), any(File.class), uniqueIdArgumentCaptor.capture()); - verify(client, times(0)).createDatabase(anyString()); - verify(client, times(1)).createTable(eq(DB), eq(TABLE)); - UUID.fromString(uniqueIdArgumentCaptor.getValue()); + TreasureDataSender.Config config = new TreasureDataSender.Config(); + config.setMaxRetryIntervalMs(9); + assertThrows(IllegalArgumentException.class, () -> new TreasureDataSender(config)); } - @Test - void sendWithCreatingDatabase() - throws IOException { - AtomicInteger importToTableCalls = new AtomicInteger(); - doAnswer(invocation -> { - if (importToTableCalls.getAndIncrement() == 0) { - throw new TDClientHttpNotFoundException("Not Found!!!!"); - } - assertImportedFile(invocation.getArgument(2)); - return null; - }).when(client).importFile(anyString(), anyString(), any(File.class), anyString()); - - AtomicInteger createTableCalls = new AtomicInteger(); - doAnswer(invocation -> { - if (createTableCalls.getAndIncrement() == 0) { - throw new TDClientHttpNotFoundException("Not Found!!!!"); - } - return null; - }).when(client).createTable(anyString(), anyString()); - - sender.send(DB_AND_TABLE, ByteBuffer.wrap(DATA)); - ArgumentCaptor uniqueIdArgumentCaptor = ArgumentCaptor.forClass(String.class); - verify(client, times(2)).importFile( - eq(DB), eq(TABLE), any(File.class), uniqueIdArgumentCaptor.capture()); - verify(client, times(1)).createDatabase(eq(DB)); - verify(client, times(2)).createTable(eq(DB), eq(TABLE)); - UUID.fromString(uniqueIdArgumentCaptor.getValue()); + TreasureDataSender.Config config = new TreasureDataSender.Config(); + config.setRetryMax(-1); + assertThrows(IllegalArgumentException.class, () -> new TreasureDataSender(config)); } - @Test - public void sendWithLackOfPermissionOnDatabase() - throws IOException { - doThrow(new TDClientHttpNotFoundException("Not Found!!!!")) - .when(client).importFile(anyString(), anyString(), any(File.class), anyString()); - - doThrow(new TDClientHttpNotFoundException("Not Found!!!!")) - .when(client).createTable(anyString(), anyString()); - - doThrow(new TDClientHttpConflictException("Conflict!!!!")) - .when(client).createDatabase(anyString()); - - doReturn(false).when(client).existsDatabase(anyString()); - - try { - sender.send(DB_AND_TABLE, ByteBuffer.wrap(DATA)); - fail(); - } - catch (NonRetryableException e) { - assertTrue(true); - } - ArgumentCaptor uniqueIdArgumentCaptor = ArgumentCaptor.forClass(String.class); - verify(client, times(1)).importFile( - eq(DB), eq(TABLE), any(File.class), uniqueIdArgumentCaptor.capture()); - verify(client, times(4)).createDatabase(eq(DB)); - verify(client, times(4)).existsDatabase(eq(DB)); - verify(client, times(1)).createTable(eq(DB), eq(TABLE)); - UUID.fromString(uniqueIdArgumentCaptor.getValue()); + TreasureDataSender.Config config = new TreasureDataSender.Config(); + config.setRetryFactor(0.9f); + assertThrows(IllegalArgumentException.class, () -> new TreasureDataSender(config)); } - @Test - void validateConfig() { - { - TreasureDataSender.Config config = new TreasureDataSender.Config(); - config.setRetryIntervalMs(9); - assertThrows(IllegalArgumentException.class, () -> new TreasureDataSender(config)); - } - - { - TreasureDataSender.Config config = new TreasureDataSender.Config(); - config.setMaxRetryIntervalMs(9); - assertThrows(IllegalArgumentException.class, () -> new TreasureDataSender(config)); - } - - { - TreasureDataSender.Config config = new TreasureDataSender.Config(); - config.setRetryMax(-1); - assertThrows(IllegalArgumentException.class, () -> new TreasureDataSender(config)); - } - - { - TreasureDataSender.Config config = new TreasureDataSender.Config(); - config.setRetryFactor(0.9f); - assertThrows(IllegalArgumentException.class, () -> new TreasureDataSender(config)); - } - - { - TreasureDataSender.Config config = new TreasureDataSender.Config(); - config.setWorkBufSize(1023); - assertThrows(IllegalArgumentException.class, () -> new TreasureDataSender(config)); - } + TreasureDataSender.Config config = new TreasureDataSender.Config(); + config.setWorkBufSize(1023); + assertThrows(IllegalArgumentException.class, () -> new TreasureDataSender(config)); } -} \ No newline at end of file + } +} diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index e708b1c0..7f93135c 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index e411586a..df97d72b 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-bin.zip +networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 4f906e0c..1aa94a42 100755 --- a/gradlew +++ b/gradlew @@ -1,7 +1,7 @@ -#!/usr/bin/env sh +#!/bin/sh # -# Copyright 2015 the original author or authors. +# Copyright © 2015-2021 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,67 +17,99 @@ # ############################################################################## -## -## Gradle start up script for UN*X -## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# ############################################################################## # Attempt to set APP_HOME + # Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" +MAX_FD=maximum warn () { echo "$*" -} +} >&2 die () { echo echo "$*" echo exit 1 -} +} >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar @@ -87,9 +119,9 @@ CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACMD=$JAVA_HOME/jre/sh/java else - JAVACMD="$JAVA_HOME/bin/java" + JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME @@ -98,88 +130,120 @@ Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else - JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac fi -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. # For Cygwin or MSYS, switch paths to Windows format before running java -if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) fi - i=`expr $i + 1` + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg done - case $i in - 0) set -- ;; - 1) set -- "$args0" ;; - 2) set -- "$args0" "$args1" ;; - 3) set -- "$args0" "$args1" "$args2" ;; - 4) set -- "$args0" "$args1" "$args2" "$args3" ;; - 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac fi -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=`save "$@"` -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat index ac1b06f9..6689b85b 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -14,7 +14,7 @@ @rem limitations under the License. @rem -@if "%DEBUG%" == "" @echo off +@if "%DEBUG%"=="" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @@ -25,7 +25,8 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @@ -40,7 +41,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto execute +if %ERRORLEVEL% equ 0 goto execute echo. echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. @@ -75,13 +76,15 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar :end @rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd +if %ERRORLEVEL% equ 0 goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% :mainEnd if "%OS%"=="Windows_NT" endlocal